Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(statedb_cmd): Add deep check functionality to UTXO:Ords map index #2088

Merged
merged 2 commits into from
Jul 6, 2024
+58 −26
Merged

refactor(statedb_cmd): Add deep check functionality to UTXO:Ords map index #2088

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5493449
refactor(statedb_cmd): Add deep check functionality to UTXO:Ords map …
popcnt1 Jul 6, 2024
2b79f09
Merge branch 'main' into feat.ord_index_check
popcnt1 Jul 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 58 additions & 26 deletions crates/rooch/src/commands/statedb/commands/genesis_ord.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ use anyhow::Result;
use bitcoin::hashes::Hash;
use bitcoin::{OutPoint, PublicKey, ScriptBuf};
use bitcoin_move::natives::ord::inscription_id::InscriptionId;
use chrono::{DateTime, Local};
use clap::Parser;
use move_core_types::account_address::AccountAddress;
use redb::Database;
Expand Down Expand Up @@ -47,7 +48,7 @@ use crate::commands::statedb::commands::genesis_utxo::{
};
use crate::commands::statedb::commands::import::{apply_fields, apply_nodes};
use crate::commands::statedb::commands::{
init_genesis_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
get_ord_by_outpoint, init_genesis_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
};

pub const ADDRESS_UNBOUND: &str = "unbound";
Expand Down Expand Up @@ -109,6 +110,12 @@ pub struct GenesisOrdCommand {
pub ord_batch_size: Option<usize>,
#[clap(long, help = "utxo:ords map db path, will create new one if not exist")]
pub utxo_ord_map: PathBuf,
#[clap(
long,
help = "deep check utxo:ords map db integrity",
default_value = "false"
)]
pub deep_check_utxo_ord_map: Option<bool>,

#[clap(long = "data-dir", short = 'd')]
/// Path to data dir, this dir is base dir, the final data_dir is base_dir/chain_network_name
Expand All @@ -135,17 +142,14 @@ impl GenesisOrdCommand {
init_genesis_job(self.base_data_dir.clone(), self.chain_id.clone());
let pre_root_state_root = H256::from(root.state_root.into_bytes());

let utxo_ord_map_existed = self.utxo_ord_map.exists();
if utxo_ord_map_existed {
println!("utxo:ords indexed in: {:?}", self.utxo_ord_map.clone());
}
// check utxo_ord_map_index file exist or not
let utxo_ord_map_db = Database::create(self.utxo_ord_map.clone()).unwrap();
let utxo_ord_map_existed = self.utxo_ord_map.exists(); // check if utxo:ords map db existed before create db
let utxo_ord_map_db = Database::create(self.utxo_ord_map.clone()).unwrap(); // create db if not existed
let utxo_ord_map = Arc::new(utxo_ord_map_db);
index_utxo_ords(
self.ord_source.clone(),
utxo_ord_map.clone(),
utxo_ord_map_existed,
self.deep_check_utxo_ord_map.unwrap(),
);

let moveos_store = Arc::new(moveos_store);
Expand Down Expand Up @@ -191,14 +195,30 @@ impl GenesisOrdCommand {
}
}

fn index_utxo_ords(input_path: PathBuf, utxo_ord_map: Arc<Database>, utxo_ord_map_existed: bool) {
if utxo_ord_map_existed {
// indexing steps:
// 1. load all ords for ord_src_path (may cost 10GiB memory)
// 2. sort merge ords by utxo
// 3. insert utxo:ords into db
fn index_utxo_ords(
ord_src_path: PathBuf,
utxo_ord_map: Arc<Database>,
utxo_ord_map_existed: bool,
deep_check: bool,
) {
if !deep_check && utxo_ord_map_existed {
println!("utxo:ords map db existed, skip indexing");
return;
}

let start_time = SystemTime::now();
let datetime: DateTime<Local> = start_time.into();

println!("indexing utxo:ords started at: {}", datetime);

let read_txn = utxo_ord_map.clone().begin_read().unwrap();
let read_table = Some(Arc::new(read_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap()));

let mut reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(input_path).unwrap());
let mut reader = BufReader::with_capacity(8 * 1024 * 1024, File::open(ord_src_path).unwrap());
let mut is_title_line = true;

let mut ord_count: u64 = 0;
Expand All @@ -222,33 +242,45 @@ fn index_utxo_ords(input_path: PathBuf, utxo_ord_map: Arc<Database>, utxo_ord_ma

utxo_ords.push(UTXOOrds {
utxo: satpoint_output,
ords: vec![obj_id],
ords: vec![obj_id.clone()], // only one ord for one utxo at most time
});
ord_count += 1;
}

let utxo_count = sort_merge_utxo_ords(&mut utxo_ords);

let write_txn = utxo_ord_map.clone().begin_write().unwrap();
{
let mut table = write_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap();
for utxo_ord in utxo_ords {
table
.insert(
bcs::to_bytes(&utxo_ord.utxo).unwrap().as_slice(),
bcs::to_bytes(&utxo_ord.ords).unwrap().as_slice(),
)
.unwrap();
let utxo_count = sort_merge_utxo_ords(&mut utxo_ords) as u64;

if deep_check && utxo_ord_map_existed {
for utxo_ord in utxo_ords.iter() {
let ords_in_db = get_ord_by_outpoint(read_table.clone(), utxo_ord.utxo).unwrap();
if ords_in_db != utxo_ord.ords {
panic!(
"failed to deep check: utxo: {} ords not match, expected: {:?}, actual: {:?}",
utxo_ord.utxo, utxo_ord.ords, ords_in_db
);
}
}
println!("deep check passed");
} else {
let write_txn = utxo_ord_map.clone().begin_write().unwrap();
{
let mut table = write_txn.open_table(UTXO_ORD_MAP_TABLE).unwrap();
for utxo_ord in utxo_ords {
table
.insert(
bcs::to_bytes(&utxo_ord.utxo).unwrap().as_slice(),
bcs::to_bytes(&utxo_ord.ords).unwrap().as_slice(),
)
.unwrap();
}
}
write_txn.commit().unwrap();
}
write_txn.commit().unwrap();

println!(
"{} utxo : {} ords indexed in: {:?} (started at: {:?})",
"{} utxo : {} ords indexed in: {:?}",
utxo_count,
ord_count,
start_time.elapsed().unwrap(),
start_time
);
}

Expand Down
Loading