refactor(statedb_cmd): Add deep check functionality to UTXO:Ords map …

…index (#2088)

The UTXO:Ords map index function now features an option for a 'deep check'. This thorough validation confirms whether the provided UtXO:Ords map database and the ones stored in the table match. If not, it raises a panic error. The print statement has also been modified to include the start time of the indexing process.

crates/rooch/src/commands/statedb/commands/genesis_ord.rs

@@ -15,6 +15,7 @@ use anyhow::Result;
 use bitcoin::hashes::Hash;
 use bitcoin::{OutPoint, PublicKey, ScriptBuf};
 use bitcoin_move::natives::ord::inscription_id::InscriptionId;
+use chrono::{DateTime, Local};
 use clap::Parser;
 use move_core_types::account_address::AccountAddress;
 use redb::Database;
@@ -47,7 +48,7 @@ use crate::commands::statedb::commands::genesis_utxo::{
 };
 use crate::commands::statedb::commands::import::{apply_fields, apply_nodes};
 use crate::commands::statedb::commands::{
-    init_genesis_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
+    get_ord_by_outpoint, init_genesis_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
 };
 
 pub const ADDRESS_UNBOUND: &str = "unbound";
@@ -109,6 +110,12 @@ pub struct GenesisOrdCommand {
     pub ord_batch_size: Option<usize>,
     #[clap(long, help = "utxo:ords map db path, will create new one if not exist")]
     pub utxo_ord_map: PathBuf,
+    #[clap(
+        long,
+        help = "deep check utxo:ords map db integrity",
+        default_value = "false"
+    )]
+    pub deep_check_utxo_ord_map: Option<bool>,
 
     #[clap(long = "data-dir", short = 'd')]
     /// Path to data dir, this dir is base dir, the final data_dir is base_dir/chain_network_name
@@ -135,17 +142,14 @@ impl GenesisOrdCommand {
             init_genesis_job(self.base_data_dir.clone(), self.chain_id.clone());
         let pre_root_state_root = H256::from(root.state_root.into_bytes());
 
-        let utxo_ord_map_existed = self.utxo_ord_map.exists();
-        if utxo_ord_map_existed {
-            println!("utxo:ords indexed in: {:?}", self.utxo_ord_map.clone());
-        }
-        // check utxo_ord_map_index file exist or not
-        let utxo_ord_map_db = Database::create(self.utxo_ord_map.clone()).unwrap();
+        let utxo_ord_map_existed = self.utxo_ord_map.exists(); // check if utxo:ords map db existed before create db
+        let utxo_ord_map_db = Database::create(self.utxo_ord_map.clone()).unwrap(); // create db if not existed
         let utxo_ord_map = Arc::new(utxo_ord_map_db);
         index_utxo_ords(
             self.ord_source.clone(),
             utxo_ord_map.clone(),
             utxo_ord_map_existed,
+            self.deep_check_utxo_ord_map.unwrap(),
         );
 
         let moveos_store = Arc::new(moveos_store);
@@ -191,14 +195,30 @@ impl GenesisOrdCommand {
     }
 }
 
-fn index_utxo_ords(input_path: PathBuf, utxo_ord_map: Arc<Database>, utxo_ord_map_existed: bool) {
-    if utxo_ord_map_existed {
+// indexing steps:
+// 1. load all ords for ord_src_path (may cost 10GiB memory)
+// 2. sort merge ords by utxo
+// 3. insert utxo:ords into db
+fn index_utxo_ords(
+    ord_src_path: PathBuf,
+    utxo_ord_map: Arc<Database>,
+    utxo_ord_map_existed: bool,
+    deep_check: bool,
+) {
+    if !deep_check && utxo_ord_map_existed {
+        println!("utxo:ords map db existed, skip indexing");
         return;
     }
 
     let start_time = SystemTime::now();
+    let datetime: DateTime<Local> = start_time.into();
+
+    println!("indexing utxo:ords started at: {}", datetime);
+
+    let read_txn = utxo_ord_map.clone().begin_read().unwrap();
+    let read_table = Some(Arc::new(read_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap()));
 
-    let mut reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(input_path).unwrap());
+    let mut reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(ord_src_path).unwrap());
     let mut is_title_line = true;
 
     let mut ord_count: u64 = 0;
@@ -222,33 +242,45 @@ fn index_utxo_ords(input_path: PathBuf, utxo_ord_map: Arc<Database>, utxo_ord_ma
 
         utxo_ords.push(UTXOOrds {
             utxo: satpoint_output,
-            ords: vec![obj_id],
+            ords: vec![obj_id.clone()], // only one ord for one utxo at most time
         });
         ord_count += 1;
     }
 
-    let utxo_count = sort_merge_utxo_ords(&mut utxo_ords);
-
-    let write_txn = utxo_ord_map.clone().begin_write().unwrap();
-    {
-        let mut table = write_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap();
-        for utxo_ord in utxo_ords {
-            table
-                .insert(
-                    bcs::to_bytes(&utxo_ord.utxo).unwrap().as_slice(),
-                    bcs::to_bytes(&utxo_ord.ords).unwrap().as_slice(),
-                )
-                .unwrap();
+    let utxo_count = sort_merge_utxo_ords(&mut utxo_ords) as u64;
+
+    if deep_check && utxo_ord_map_existed {
+        for utxo_ord in utxo_ords.iter() {
+            let ords_in_db = get_ord_by_outpoint(read_table.clone(), utxo_ord.utxo).unwrap();
+            if ords_in_db != utxo_ord.ords {
+                panic!(
+                    "failed to deep check: utxo: {} ords not match, expected: {:?}, actual: {:?}",
+                    utxo_ord.utxo, utxo_ord.ords, ords_in_db
+                );
+            }
+        }
+        println!("deep check passed");
+    } else {
+        let write_txn = utxo_ord_map.clone().begin_write().unwrap();
+        {
+            let mut table = write_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap();
+            for utxo_ord in utxo_ords {
+                table
+                    .insert(
+                        bcs::to_bytes(&utxo_ord.utxo).unwrap().as_slice(),
+                        bcs::to_bytes(&utxo_ord.ords).unwrap().as_slice(),
+                    )
+                    .unwrap();
+            }
         }
+        write_txn.commit().unwrap();
     }
-    write_txn.commit().unwrap();
 
     println!(
-        "{} utxo : {} ords indexed in: {:?} (started at: {:?})",
+        "{} utxo : {} ords indexed in: {:?}",
         utxo_count,
         ord_count,
         start_time.elapsed().unwrap(),
-        start_time
     );
 }