Merge branch 'master' into show_storage_policy

.asf.yaml

@@ -56,8 +56,7 @@ github:
           - cloud_p0 (Doris Cloud Regression)
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
-          - Build Broker
-          - ShellCheck          
+          - Build Broker      
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - Build Third Party Libraries (macOS-arm64)
@@ -80,7 +79,6 @@ github:
           - Clang Formatter
           - CheckStyle
           - Build Broker
-          - ShellCheck
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - FE UT (Doris FE UT)
@@ -103,7 +101,6 @@ github:
           - Clang Formatter
           - CheckStyle
           - Build Broker
-          - ShellCheck
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
           - COMPILE (DORIS_COMPILE)

README.md

@@ -177,15 +177,10 @@ In terms of optimizers, Doris uses a combination of CBO and RBO. RBO supports co
 
 **Apache Doris has graduated from Apache incubator successfully and become a Top-Level Project in June 2022**. 
 
-Currently, the Apache Doris community has gathered more than 400 contributors from nearly 200 companies in different industries, and the number of active contributors is close to 100 per month.
-
-
-[![Monthly Active Contributors](https://contributor-overtime-api.apiseven.com/contributors-svg?chart=contributorMonthlyActivity&repo=apache/doris)](https://www.apiseven.com/en/contributor-graph?chart=contributorMonthlyActivity&repo=apache/doris)
-
-[![Contributor over time](https://contributor-overtime-api.apiseven.com/contributors-svg?chart=contributorOverTime&repo=apache/doris)](https://www.apiseven.com/en/contributor-graph?chart=contributorOverTime&repo=apache/doris)
-
 We deeply appreciate 🔗[community contributors](https://github.com/apache/doris/graphs/contributors) for their contribution to Apache Doris.
 
+[![contrib graph](https://contrib.rocks/image?repo=apache/doris)](https://github.com/apache/doris/graphs/contributors)
+
 ## 👨‍👩‍👧‍👦 Users
 
 Apache Doris now has a wide user base in China and around the world, and as of today, **Apache Doris is used in production environments in thousands of companies worldwide.** More than 80% of the top 50 Internet companies in China in terms of market capitalization or valuation have been using Apache Doris for a long time, including Baidu, Meituan, Xiaomi, Jingdong, Bytedance, Tencent, NetEase, Kwai, Sina, 360, Mihoyo, and Ke Holdings. It is also widely used in some traditional industries such as finance, energy, manufacturing, and telecommunications.

be/src/cloud/cloud_cumulative_compaction.cpp

@@ -393,12 +393,9 @@ Status CloudCumulativeCompaction::process_old_version_delete_bitmap() {
                 rowset->rowset_id().to_string();
                 DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
                 DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id, pre_max_version};
-                DeleteBitmap::BitmapKey before_end {rowset->rowset_id(), seg_id,
-                                                    pre_max_version - 1};
                 auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
                         {rowset->rowset_id(), seg_id, pre_max_version});
-                to_remove_vec.emplace_back(
-                        std::make_tuple(_tablet->tablet_id(), start, before_end));
+                to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
                 if (d->isEmpty()) {
                     continue;
                 }

be/src/cloud/cloud_delete_bitmap_action.cpp

@@ -33,6 +33,7 @@
 #include <thread>
 #include <utility>
 
+#include "cloud/cloud_meta_mgr.h"
 #include "cloud/cloud_tablet.h"
 #include "cloud/cloud_tablet_mgr.h"
 #include "common/logging.h"
@@ -78,8 +79,8 @@ static Status _check_param(HttpRequest* req, uint64_t* tablet_id) {
     return Status::OK();
 }
 
-Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* req,
-                                                                 std::string* json_result) {
+Status CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest* req,
+                                                                       std::string* json_result) {
     uint64_t tablet_id = 0;
     // check & retrieve tablet_id from req if it contains
     RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed");
@@ -95,6 +96,50 @@ Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re
     auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
     auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
     auto size = tablet->tablet_meta()->delete_bitmap().get_size();
+    LOG(INFO) << "show_local_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count
+              << ",cardinality=" << cardinality << ",size=" << size;
+
+    rapidjson::Document root;
+    root.SetObject();
+    root.AddMember("delete_bitmap_count", count, root.GetAllocator());
+    root.AddMember("cardinality", cardinality, root.GetAllocator());
+    root.AddMember("size", size, root.GetAllocator());
+
+    // to json string
+    rapidjson::StringBuffer strbuf;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+    root.Accept(writer);
+    *json_result = std::string(strbuf.GetString());
+
+    return Status::OK();
+}
+
+Status CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req,
+                                                                    std::string* json_result) {
+    uint64_t tablet_id = 0;
+    // check & retrieve tablet_id from req if it contains
+    RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param failed");
+    if (tablet_id == 0) {
+        return Status::InternalError("check param failed: missing tablet_id");
+    }
+    TabletMetaSharedPtr tablet_meta;
+    auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta);
+    if (!st.ok()) {
+        LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id
+                     << ", st=" << st.to_string();
+        return st;
+    }
+    auto tablet = std::make_shared<CloudTablet>(_engine, std::move(tablet_meta));
+    st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true, true);
+    if (!st.ok()) {
+        LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st;
+        return st;
+    }
+    auto count = tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
+    auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
+    auto size = tablet->tablet_meta()->delete_bitmap().get_size();
+    LOG(INFO) << "show_ms_delete_bitmap_count,tablet_id=" << tablet_id << ",count=" << count
+              << ",cardinality=" << cardinality << ",size=" << size;
 
     rapidjson::Document root;
     root.SetObject();
@@ -113,9 +158,17 @@ Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re
 
 void CloudDeleteBitmapAction::handle(HttpRequest* req) {
     req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data());
-    if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_INFO) {
+    if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) {
+        std::string json_result;
+        Status st = _handle_show_local_delete_bitmap_count(req, &json_result);
+        if (!st.ok()) {
+            HttpChannel::send_reply(req, HttpStatus::OK, st.to_json());
+        } else {
+            HttpChannel::send_reply(req, HttpStatus::OK, json_result);
+        }
+    } else if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_MS) {
         std::string json_result;
-        Status st = _handle_show_delete_bitmap_count(req, &json_result);
+        Status st = _handle_show_ms_delete_bitmap_count(req, &json_result);
         if (!st.ok()) {
             HttpChannel::send_reply(req, HttpStatus::OK, st.to_json());
         } else {

be/src/cloud/cloud_delete_bitmap_action.h

@@ -31,7 +31,7 @@ class HttpRequest;
 
 class ExecEnv;
 
-enum class DeleteBitmapActionType { COUNT_INFO = 1 };
+enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 };
 
 /// This action is used for viewing the delete bitmap status
 class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
@@ -45,7 +45,8 @@ class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
     void handle(HttpRequest* req) override;
 
 private:
-    Status _handle_show_delete_bitmap_count(HttpRequest* req, std::string* json_result);
+    Status _handle_show_local_delete_bitmap_count(HttpRequest* req, std::string* json_result);
+    Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string* json_result);
 
 private:
     CloudStorageEngine& _engine;

be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp

@@ -75,7 +75,7 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
         for (size_t i = 0; i < partition.tablet_ids.size(); i++) {
             auto tablet_id = partition.tablet_ids[i];
             auto tablet_calc_delete_bitmap_ptr = std::make_shared<CloudTabletCalcDeleteBitmapTask>(
-                    _engine, this, tablet_id, transaction_id, version);
+                    _engine, this, tablet_id, transaction_id, version, partition.sub_txn_ids);
             if (has_compaction_stats) {
                 tablet_calc_delete_bitmap_ptr->set_compaction_stats(
                         partition.base_compaction_cnts[i], partition.cumulative_compaction_cnts[i],
@@ -107,12 +107,13 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
 
 CloudTabletCalcDeleteBitmapTask::CloudTabletCalcDeleteBitmapTask(
         CloudStorageEngine& engine, CloudEngineCalcDeleteBitmapTask* engine_task, int64_t tablet_id,
-        int64_t transaction_id, int64_t version)
+        int64_t transaction_id, int64_t version, const std::vector<int64_t>& sub_txn_ids)
         : _engine(engine),
           _engine_calc_delete_bitmap_task(engine_task),
           _tablet_id(tablet_id),
           _transaction_id(transaction_id),
-          _version(version) {
+          _version(version),
+          _sub_txn_ids(sub_txn_ids) {
     _mem_tracker = MemTrackerLimiter::create_shared(
             MemTrackerLimiter::Type::OTHER,
             fmt::format("CloudTabletCalcDeleteBitmapTask#_transaction_id={}", _transaction_id));
@@ -189,6 +190,60 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
         return error_st;
     }
 
+    int64_t t3 = MonotonicMicros();
+    Status status;
+    if (_sub_txn_ids.empty()) {
+        status = _handle_rowset(tablet, _version);
+    } else {
+        std::stringstream ss;
+        for (const auto& sub_txn_id : _sub_txn_ids) {
+            ss << sub_txn_id << ", ";
+        }
+        LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id << ", sub_txn_ids=["
+                  << ss.str() << "], table_id=" << tablet->table_id()
+                  << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
+                  << ", start_version=" << _version;
+        std::vector<RowsetSharedPtr> invisible_rowsets;
+        DeleteBitmapPtr tablet_delete_bitmap =
+                std::make_shared<DeleteBitmap>(tablet->tablet_meta()->delete_bitmap());
+        for (int i = 0; i < _sub_txn_ids.size(); ++i) {
+            int64_t sub_txn_id = _sub_txn_ids[i];
+            int64_t version = _version + i;
+            LOG(INFO) << "start calc delete bitmap for txn_id=" << _transaction_id
+                      << ", sub_txn_id=" << sub_txn_id << ", table_id=" << tablet->table_id()
+                      << ", partition_id=" << tablet->partition_id() << ", tablet_id=" << _tablet_id
+                      << ", start_version=" << _version << ", cur_version=" << version;
+            status = _handle_rowset(tablet, version, sub_txn_id, &invisible_rowsets,
+                                    tablet_delete_bitmap);
+            if (!status.ok()) {
+                LOG(INFO) << "failed to calculate delete bitmap on tablet"
+                          << ", table_id=" << tablet->table_id()
+                          << ", transaction_id=" << _transaction_id << ", sub_txn_id=" << sub_txn_id
+                          << ", tablet_id=" << tablet->tablet_id() << ", start version=" << _version
+                          << ", cur_version=" << version << ", status=" << status;
+                return status;
+            }
+            DCHECK(invisible_rowsets.size() == i + 1);
+        }
+    }
+    auto total_update_delete_bitmap_time_us = MonotonicMicros() - t3;
+    LOG(INFO) << "calculate delete bitmap successfully on tablet"
+              << ", table_id=" << tablet->table_id() << ", transaction_id=" << _transaction_id
+              << ", tablet_id=" << tablet->tablet_id()
+              << ", get_tablet_time_us=" << get_tablet_time_us
+              << ", sync_rowset_time_us=" << sync_rowset_time_us
+              << ", total_update_delete_bitmap_time_us=" << total_update_delete_bitmap_time_us
+              << ", res=" << status;
+    return status;
+}
+
+Status CloudTabletCalcDeleteBitmapTask::_handle_rowset(
+        std::shared_ptr<CloudTablet> tablet, int64_t version, int64_t sub_txn_id,
+        std::vector<RowsetSharedPtr>* invisible_rowsets,
+        DeleteBitmapPtr tablet_delete_bitmap) const {
+    int64_t transaction_id = sub_txn_id == -1 ? _transaction_id : sub_txn_id;
+    std::string txn_str = "txn_id=" + std::to_string(_transaction_id) +
+                          (sub_txn_id == -1 ? "" : ", sub_txn_id=" + std::to_string(sub_txn_id));
     RowsetSharedPtr rowset;
     DeleteBitmapPtr delete_bitmap;
     RowsetIdUnorderedSet rowset_ids;
@@ -197,59 +252,76 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
     int64_t txn_expiration;
     TxnPublishInfo previous_publish_info;
     Status status = _engine.txn_delete_bitmap_cache().get_tablet_txn_info(
-            _transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration,
+            transaction_id, _tablet_id, &rowset, &delete_bitmap, &rowset_ids, &txn_expiration,
             &partial_update_info, &publish_status, &previous_publish_info);
     if (status != Status::OK()) {
-        LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id
-                     << ", txn_id=" << _transaction_id << ", status=" << status;
+        LOG(WARNING) << "failed to get tablet txn info. tablet_id=" << _tablet_id << ", " << txn_str
+                     << ", status=" << status;
         _engine_calc_delete_bitmap_task->add_error_tablet_id(_tablet_id, status);
         return status;
     }
 
     int64_t t3 = MonotonicMicros();
-    rowset->set_version(Version(_version, _version));
+    rowset->set_version(Version(version, version));
     TabletTxnInfo txn_info;
     txn_info.rowset = rowset;
     txn_info.delete_bitmap = delete_bitmap;
     txn_info.rowset_ids = rowset_ids;
     txn_info.partial_update_info = partial_update_info;
     txn_info.publish_status = publish_status;
-    txn_info.publish_info = {.publish_version = _version,
+    txn_info.publish_info = {.publish_version = version,
                              .base_compaction_cnt = _ms_base_compaction_cnt,
                              .cumulative_compaction_cnt = _ms_cumulative_compaction_cnt,
                              .cumulative_point = _ms_cumulative_point};
     auto update_delete_bitmap_time_us = 0;
     if (txn_info.publish_status && (*(txn_info.publish_status) == PublishStatus::SUCCEED) &&
-        _version == previous_publish_info.publish_version &&
+        version == previous_publish_info.publish_version &&
         _ms_base_compaction_cnt == previous_publish_info.base_compaction_cnt &&
         _ms_cumulative_compaction_cnt == previous_publish_info.cumulative_compaction_cnt &&
         _ms_cumulative_point == previous_publish_info.cumulative_point) {
         // if version or compaction stats can't match, it means that this is a retry and there are
         // compaction or other loads finished successfully on the same tablet. So the previous publish
         // is stale and we should re-calculate the delete bitmap
-        LOG(INFO) << "tablet=" << _tablet_id << ",txn=" << _transaction_id
+        LOG(INFO) << "tablet=" << _tablet_id << ", " << txn_str
                   << ",publish_status=SUCCEED,not need to recalculate and update delete_bitmap.";
     } else {
-        status = CloudTablet::update_delete_bitmap(tablet, &txn_info, _transaction_id,
-                                                   txn_expiration);
+        if (invisible_rowsets == nullptr) {
+            status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id,
+                                                       txn_expiration);
+        } else {
+            txn_info.is_txn_load = true;
+            txn_info.invisible_rowsets = *invisible_rowsets;
+            txn_info.lock_id = _transaction_id;
+            txn_info.next_visible_version = _version;
+            status = CloudTablet::update_delete_bitmap(tablet, &txn_info, transaction_id,
+                                                       txn_expiration, tablet_delete_bitmap);
+        }
         update_delete_bitmap_time_us = MonotonicMicros() - t3;
     }
     if (status != Status::OK()) {
         LOG(WARNING) << "failed to calculate delete bitmap. rowset_id=" << rowset->rowset_id()
-                     << ", tablet_id=" << _tablet_id << ", txn_id=" << _transaction_id
-                     << ", status=" << status;
+                     << ", tablet_id=" << _tablet_id << ", " << txn_str << ", status=" << status;
         _engine_calc_delete_bitmap_task->add_error_tablet_id(_tablet_id, status);
         return status;
     }
 
     _engine_calc_delete_bitmap_task->add_succ_tablet_id(_tablet_id);
     LOG(INFO) << "calculate delete bitmap successfully on tablet"
-              << ", table_id=" << tablet->table_id() << ", transaction_id=" << _transaction_id
+              << ", table_id=" << tablet->table_id() << ", " << txn_str
               << ", tablet_id=" << tablet->tablet_id() << ", num_rows=" << rowset->num_rows()
-              << ", get_tablet_time_us=" << get_tablet_time_us
-              << ", sync_rowset_time_us=" << sync_rowset_time_us
               << ", update_delete_bitmap_time_us=" << update_delete_bitmap_time_us
               << ", res=" << status;
+    if (invisible_rowsets != nullptr) {
+        invisible_rowsets->push_back(rowset);
+        // see CloudTablet::save_delete_bitmap
+        auto dm = txn_info.delete_bitmap->delete_bitmap;
+        for (auto it = dm.begin(); it != dm.end(); ++it) {
+            if (std::get<1>(it->first) != DeleteBitmap::INVALID_SEGMENT_ID) {
+                tablet_delete_bitmap->merge(
+                        {std::get<0>(it->first), std::get<1>(it->first), version}, it->second);
+            }
+        }
+    }
     return status;
 }
 

be/src/cloud/cloud_engine_calc_delete_bitmap_task.h

@@ -34,7 +34,8 @@ class CloudTabletCalcDeleteBitmapTask {
 public:
     CloudTabletCalcDeleteBitmapTask(CloudStorageEngine& engine,
                                     CloudEngineCalcDeleteBitmapTask* engine_task, int64_t tablet_id,
-                                    int64_t transaction_id, int64_t version);
+                                    int64_t transaction_id, int64_t version,
+                                    const std::vector<int64_t>& sub_txn_ids);
     ~CloudTabletCalcDeleteBitmapTask() = default;
 
     void set_compaction_stats(int64_t ms_base_compaction_cnt, int64_t ms_cumulative_compaction_cnt,
@@ -43,12 +44,18 @@ class CloudTabletCalcDeleteBitmapTask {
     Status handle() const;
 
 private:
+    Status _handle_rowset(std::shared_ptr<CloudTablet> tablet, int64_t version,
+                          int64_t sub_txn_id = -1,
+                          std::vector<RowsetSharedPtr>* invisible_rowsets = nullptr,
+                          DeleteBitmapPtr tablet_delete_bitmap = nullptr) const;
+
     CloudStorageEngine& _engine;
     CloudEngineCalcDeleteBitmapTask* _engine_calc_delete_bitmap_task;
 
     int64_t _tablet_id;
     int64_t _transaction_id;
     int64_t _version;
+    std::vector<int64_t> _sub_txn_ids;
 
     int64_t _ms_base_compaction_cnt {-1};
     int64_t _ms_cumulative_compaction_cnt {-1};