2

apache · Oct 12, 2024 · 4ecce69 · 4ecce69
1 parent 8391753
commit 4ecce69
Show file tree

Hide file tree

Showing 6 changed files with 103 additions and 6 deletions.
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
@@ -50,6 +50,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_prepare_partition("ms", "prepare_partition"
 BvarLatencyRecorderWithTag g_bvar_ms_commit_partition("ms", "commit_partition");
 BvarLatencyRecorderWithTag g_bvar_ms_drop_partition("ms", "drop_partition");
 BvarLatencyRecorderWithTag g_bvar_ms_get_tablet_stats("ms", "get_tablet_stats");
+BvarLatencyRecorderWithTag g_bvar_ms_fix_tablet_stats("ms", "fix_tablet_stats");
 BvarLatencyRecorderWithTag g_bvar_ms_get_obj_store_info("ms", "get_obj_store_info");
 BvarLatencyRecorderWithTag g_bvar_ms_alter_obj_store_info("ms", "alter_obj_store_info");
 BvarLatencyRecorderWithTag g_bvar_ms_alter_storage_vault("ms", "alter_storage_vault");

diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
@@ -148,6 +148,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_prepare_partition;
 extern BvarLatencyRecorderWithTag g_bvar_ms_commit_partition;
 extern BvarLatencyRecorderWithTag g_bvar_ms_drop_partition;
 extern BvarLatencyRecorderWithTag g_bvar_ms_get_tablet_stats;
+extern BvarLatencyRecorderWithTag g_bvar_ms_fix_tablet_stats;
 extern BvarLatencyRecorderWithTag g_bvar_ms_get_obj_store_info;
 extern BvarLatencyRecorderWithTag g_bvar_ms_alter_obj_store_info;
 extern BvarLatencyRecorderWithTag g_bvar_ms_alter_storage_vault;

diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
@@ -2188,9 +2188,102 @@ std::pair<MetaServiceCode, std::string> MetaServiceImpl::get_instance_info(
     return {code, std::move(msg)};
 }
 
-void MetaServiceImpl::fix_tablet_stats(const FixTabletStatsRequest* req,
-                                       const FixTabletStatsResponse* resp) {
-
+void MetaServiceImpl::fix_tablet_stats(::google::protobuf::RpcController* controller,
+                                       const FixTabletStatsRequest* request,
+                                       FixTabletStatsResponse* response,
+                                       ::google::protobuf::Closure* done) {
+    RPC_PREPROCESS(fix_tablet_stats);
+    instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
+    if (instance_id.empty()) {
+        code = MetaServiceCode::INVALID_ARGUMENT;
+        msg = "empty instance_id";
+        LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+        return;
+    }
+    RPC_RATE_LIMIT(fix_tablet_stats)
+
+    std::unique_ptr<Transaction> txn;
+    for (const auto& i : request->table_id()) {
+        int64_t table_id = i;
+        TxnErrorCode err = txn_kv_->create_txn(&txn);
+        if (err != TxnErrorCode::TXN_OK) {
+            code = cast_as<ErrCategory::CREATE>(err);
+            msg = fmt::format("failed to create txn, table_id={}", table_id);
+            return;
+        }
+
+        std::string key, val;
+        int64_t start = 0;
+        int64_t end = std::numeric_limits<int64_t>::max() - 1;
+        auto begin_key = stats_tablet_key({instance_id, table_id, start, start, start});
+        auto end_key = stats_tablet_key({instance_id, table_id, end, end, end});
+        std::vector<std::pair<std::string, std::string>> stats_kvs;
+    stats_kvs.reserve(5); // aggregate + data_size + num_rows + num_rowsets + num_segments
+
+    std::unique_ptr<RangeGetIterator> it;
+    do {
+        TxnErrorCode err = txn->get(begin_key, end_key, &it, true);
+        if (err != TxnErrorCode::TXN_OK) {
+            code = cast_as<ErrCategory::READ>(err);
+            msg = fmt::format("failed to get tablet stats, err={} table_id={}", err,
+                              table_id);
+            return;
+        }
+        while (it->has_next()) {
+            auto [k, v] = it->next();
+            stats_kvs.emplace_back(std::string {k.data(), k.size()},
+                                   std::string {v.data(), v.size()});
+        }
+        begin_key = it->next_begin_key();
+    } while (it->more());
+
+    if (stats_kvs.empty()) {
+        code = MetaServiceCode::TABLET_NOT_FOUND;
+        msg = fmt::format("tablet stats not found, table_id={}", table_id);
+        return;
+    }
+
+    auto& [first_stats_key, v] = stats_kvs[0];
+    // First key MUST be tablet stats key, the original non-detached one
+    DCHECK(first_stats_key == begin_key_check)
+            << hex(first_stats_key) << " vs " << hex(begin_key_check);
+    if (!stats.ParseFromArray(v.data(), v.size())) {
+        code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+        msg = fmt::format("marformed tablet stats value, key={}", hex(first_stats_key));
+        return;
+    }
+    // Parse split tablet stats
+    int ret = get_detached_tablet_stats(stats_kvs, detached_stats);
+
+    if (ret != 0) {
+        code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+        msg = fmt::format("marformed splitted tablet stats kv, key={}", hex(first_stats_key));
+        return;
+    }
+
+        TabletIndexPB idx;
+        get_tablet_idx(code, msg, txn.get(), instance_id, tablet_id, idx);
+        if (code != MetaServiceCode::OK) {
+            return;
+        }
+        TabletStatsPB tablet_stat;
+        internal_get_tablet_stats(code, msg, txn.get(), instance_id, idx, tablet_stat, true);
+        if (code != MetaServiceCode::OK) {
+            break;
+        }
+
+        GetRowsetResponse resp;
+        internal_get_rowset(txn.get(), start, end, instance_id, tablet_id, code, msg, &resp);
+        if (code != MetaServiceCode::OK) {
+            return;
+        }
+
+        int64_t total_disk_size = 0;
+        for (const auto& rs_meta : resp.rowset_meta()) {
+            total_disk_size += rs_meta.total_disk_size();
+        }
+        tablet_stat.set_data_size(total_disk_size);
+    }
 }
 
 } // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
@@ -298,7 +298,9 @@ class MetaServiceImpl : public cloud::MetaService {
                                                               const std::string& cloud_unique_id,
                                                               InstanceInfoPB* instance);
 
-    void fix_tablet_stats(const FixTabletStatsRequest* req, const FixTabletStatsResponse* resp);
+    void fix_tablet_stats(::google::protobuf::RpcController* controller,
+                          const FixTabletStatsRequest* request, FixTabletStatsResponse* response,
+                          ::google::protobuf::Closure* done);
 
 private:
     std::pair<MetaServiceCode, std::string> alter_instance(

diff --git a/cloud/src/meta-service/meta_service_http.cpp b/cloud/src/meta-service/meta_service_http.cpp
@@ -472,7 +472,7 @@ static HttpResponse process_fix_tablet_stats(MetaServiceImpl* service, brpc::Con
     FixTabletStatsRequest req;
     PARSE_MESSAGE_OR_RETURN(ctrl, req);
     FixTabletStatsResponse resp;
-    service->fix_tablet_stats(&req, &resp);
+    service->fix_tablet_stats(ctrl, &req, &resp, nullptr);
 
     std::string body;
     if (resp.status().code() == MetaServiceCode::OK) {

diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
@@ -1123,7 +1123,7 @@ message GetTabletStatsResponse {
 
 message FixTabletStatsRequest {
     optional string cloud_unique_id = 1;
-    repeated int64 tablet_id = 2;
+    repeated int64 table_id = 2;
 }
 
 message FixTabletStatsResponse {