feat: databend-meta adds more metrics about raft-log (#16884)

- "raft_log_cache_items"                 : number of items in raft log cache;
- "raft_log_cache_used_size"             : size of used space in raft log cache;
- "raft_log_wal_open_chunk_size"         : size of open chunk in raft log wal;
- "raft_log_wal_offset"                  : global offset of raft log WAL;
- "raft_log_wal_closed_chunk_count"      : number of closed chunks in raft log WAL;
- "raft_log_wal_closed_chunk_total_size" : total size of closed chunks in raft log WAL;

Example output of these new metrics:

```text
metasrv_server_raft_log_cache_items                 35496
metasrv_server_raft_log_cache_used_size             659024620
metasrv_server_raft_log_wal_open_chunk_size         803
metasrv_server_raft_log_wal_offset                  3536913654
metasrv_server_raft_log_wal_closed_chunk_count      3
metasrv_server_raft_log_wal_closed_chunk_total_size 584045971
metasrv_server_raft_log_size                        584046774
```

Cargo.toml

@@ -405,7 +405,7 @@ prost = { version = "0.13" }
 prost-build = { version = "0.13" }
 prqlc = "0.11.3"
 quanta = "0.11.1"
-raft-log = { version = "0.2.3" }
+raft-log = { version = "0.2.5" }
 rand = { version = "0.8.5", features = ["small_rng"] }
 rayon = "1.9.0"
 recursive = "0.1.1"

src/meta/raft-store/src/raft_log_v004/mod.rs

@@ -27,6 +27,7 @@ pub const TREE_RAFT_LOG: &str = "raft_log";
 
 pub type RaftLogV004 = raft_log::RaftLog<RaftLogTypes>;
 pub type RaftLogConfig = raft_log::Config;
+pub type RaftLogStat = raft_log::Stat<RaftLogTypes>;
 
 pub use callback::Callback;
 pub use callback_data::CallbackData;

src/meta/service/src/meta_service/meta_node.rs

@@ -32,6 +32,7 @@ use databend_common_meta_client::reply_to_api_result;
 use databend_common_meta_client::RequestFor;
 use databend_common_meta_raft_store::config::RaftConfig;
 use databend_common_meta_raft_store::ondisk::DATA_VERSION;
+use databend_common_meta_raft_store::raft_log_v004::RaftLogStat;
 use databend_common_meta_sled_store::openraft;
 use databend_common_meta_sled_store::openraft::ChangeMembers;
 use databend_common_meta_stoerr::MetaStorageError;
@@ -446,6 +447,11 @@ impl MetaNode {
                 server_metrics::set_proposals_applied(mm.last_applied.unwrap_or_default().index);
                 server_metrics::set_last_seq(meta_node.get_last_seq().await);
 
+                {
+                    let st = meta_node.get_raft_log_stat().await;
+                    server_metrics::set_raft_log_stat(st);
+                }
+
                 // metrics about server storage
                 server_metrics::set_raft_log_size(meta_node.get_raft_log_size().await);
                 server_metrics::set_snapshot_key_count(meta_node.get_snapshot_key_count().await);
@@ -822,6 +828,10 @@ impl MetaNode {
         self.sto.log.read().await.on_disk_size()
     }
 
+    async fn get_raft_log_stat(&self) -> RaftLogStat {
+        self.sto.log.read().await.stat()
+    }
+
     async fn get_snapshot_key_count(&self) -> u64 {
         self.sto
             .try_get_snapshot_key_count()

src/meta/service/src/metrics/meta_metrics.rs

@@ -33,6 +33,7 @@ use prometheus_client::encoding::text::encode as prometheus_encode;
 pub mod server_metrics {
     use std::sync::LazyLock;
 
+    use databend_common_meta_raft_store::raft_log_v004::RaftLogStat;
     use databend_common_meta_types::raft_types::NodeId;
     use prometheus_client::metrics::counter::Counter;
     use prometheus_client::metrics::family::Family;
@@ -53,6 +54,14 @@ pub mod server_metrics {
         leader_changes: Counter,
         applying_snapshot: Gauge,
         snapshot_key_count: Gauge,
+
+        raft_log_cache_items: Gauge,
+        raft_log_cache_used_size: Gauge,
+        raft_log_wal_open_chunk_size: Gauge,
+        raft_log_wal_offset: Gauge,
+        raft_log_wal_closed_chunk_count: Gauge,
+        raft_log_wal_closed_chunk_total_size: Gauge,
+
         raft_log_size: Gauge,
         last_log_index: Gauge,
         last_seq: Gauge,
@@ -74,6 +83,12 @@ pub mod server_metrics {
                 leader_changes: Counter::default(),
                 applying_snapshot: Gauge::default(),
                 snapshot_key_count: Gauge::default(),
+                raft_log_cache_items: Gauge::default(),
+                raft_log_cache_used_size: Gauge::default(),
+                raft_log_wal_open_chunk_size: Gauge::default(),
+                raft_log_wal_offset: Gauge::default(),
+                raft_log_wal_closed_chunk_count: Gauge::default(),
+                raft_log_wal_closed_chunk_total_size: Gauge::default(),
                 raft_log_size: Gauge::default(),
                 last_log_index: Gauge::default(),
                 last_seq: Gauge::default(),
@@ -113,6 +128,38 @@ pub mod server_metrics {
                 "number of keys in the last snapshot",
                 metrics.snapshot_key_count.clone(),
             );
+
+            registry.register(
+                key!("raft_log_cache_items"),
+                "number of items in raft log cache",
+                metrics.raft_log_cache_items.clone(),
+            );
+            registry.register(
+                key!("raft_log_cache_used_size"),
+                "size of used space in raft log cache",
+                metrics.raft_log_cache_used_size.clone(),
+            );
+            registry.register(
+                key!("raft_log_wal_open_chunk_size"),
+                "size of open chunk in raft log wal",
+                metrics.raft_log_wal_open_chunk_size.clone(),
+            );
+            registry.register(
+                key!("raft_log_wal_offset"),
+                "global offset of raft log WAL",
+                metrics.raft_log_wal_offset.clone(),
+            );
+            registry.register(
+                key!("raft_log_wal_closed_chunk_count"),
+                "number of closed chunks in raft log WAL",
+                metrics.raft_log_wal_closed_chunk_count.clone(),
+            );
+            registry.register(
+                key!("raft_log_wal_closed_chunk_total_size"),
+                "total size of closed chunks in raft log WAL",
+                metrics.raft_log_wal_closed_chunk_total_size.clone(),
+            );
+
             registry.register(
                 key!("raft_log_size"),
                 "the size in bytes of the on disk data of raft log",
@@ -182,6 +229,27 @@ pub mod server_metrics {
         SERVER_METRICS.snapshot_key_count.set(n as i64);
     }
 
+    pub fn set_raft_log_stat(st: RaftLogStat) {
+        SERVER_METRICS
+            .raft_log_cache_items
+            .set(st.payload_cache_item_count as i64);
+        SERVER_METRICS
+            .raft_log_cache_used_size
+            .set(st.payload_cache_size as i64);
+        SERVER_METRICS
+            .raft_log_wal_open_chunk_size
+            .set(st.open_chunk.size as i64);
+        SERVER_METRICS
+            .raft_log_wal_offset
+            .set(st.open_chunk.global_end as i64);
+        SERVER_METRICS
+            .raft_log_wal_closed_chunk_count
+            .set(st.closed_chunks.len() as i64);
+        SERVER_METRICS
+            .raft_log_wal_closed_chunk_total_size
+            .set(st.closed_chunks.iter().map(|v| v.size).sum::<u64>() as i64);
+    }
+
     pub fn set_raft_log_size(raft_log_size: u64) {
         SERVER_METRICS.raft_log_size.set(raft_log_size as i64);
     }