Skip to content

Commit

Permalink
fix(statedb): fix export genesis error caused by object struct changes (
Browse files Browse the repository at this point in the history 
#2244)

## Summary

### Original Error

```
Error: CSV error: found record with 1 fields, but the previous record has 2 fields
```

### Fix

The issue was fixed by exporting `(FieldKey, ObjectState)`.

### Refine Codes

1. general init_job() used, simplifies both of the initialization of genesis jobs and export tasks
2. utilize instant time instead of system time for recording time cost
  • Loading branch information
@popcnt1
popcnt1 authored Jul 22, 2024
1 parent 3647a12 commit 8d0efa3
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 149 deletions.
86 changes: 23 additions & 63 deletions crates/rooch/src/commands/statedb/commands/export.rs
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,30 @@
// Copyright (c) RoochNetwork
// SPDX-License-Identifier: Apache-2.0

use crate::cli_types::WalletContextOptions;
use crate::commands::statedb::commands::{
BATCH_SIZE, GLOBAL_STATE_TYPE_FIELD, GLOBAL_STATE_TYPE_OBJECT, GLOBAL_STATE_TYPE_ROOT,
};
use std::path::PathBuf;
use std::str::FromStr;

use anyhow::Result;
use clap::Parser;
use csv::Writer;
use serde::{Deserialize, Serialize};

use moveos_store::MoveOSStore;
use moveos_types::h256::H256;
use moveos_types::moveos_std::object::ObjectID;
use moveos_types::state::{FieldKey, ObjectState};
use moveos_types::state_resolver::StatelessResolver;
use rooch_config::{RoochOpt, R_OPT_NET_HELP};
use rooch_db::RoochDB;
use rooch_config::R_OPT_NET_HELP;
use rooch_types::bitcoin::ord::InscriptionStore;
use rooch_types::bitcoin::utxo::BitcoinUTXOStore;
use rooch_types::error::{RoochError, RoochResult};
use rooch_types::framework::address_mapping::RoochToBitcoinAddressMapping;
use rooch_types::rooch_network::RoochChainID;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use std::str::FromStr;
use std::time::SystemTime;

use crate::cli_types::WalletContextOptions;
use crate::commands::statedb::commands::{
init_job, GLOBAL_STATE_TYPE_FIELD, GLOBAL_STATE_TYPE_OBJECT, GLOBAL_STATE_TYPE_ROOT,
};

/// Export statedb
Expand Down Expand Up @@ -157,17 +159,9 @@ pub struct ExportCommand {

impl ExportCommand {
pub async fn execute(self) -> RoochResult<()> {
println!("Start statedb export task, batch_size: {:?}", BATCH_SIZE);
let opt = RoochOpt::new_with_default(self.base_data_dir, self.chain_id, None)?;
let rooch_db = RoochDB::init(opt.store_config())?;
let root = rooch_db.latest_root()?.ok_or_else(|| {
RoochError::from(anyhow::Error::msg(
"The statedb is empty, please init genesis first.",
))
})?;
println!("root object: {:?}", root);
let (root, moveos_store, start_time) =
init_job(self.base_data_dir.clone(), self.chain_id.clone());

let mut _start_time = SystemTime::now();
let file_name = self.output.display().to_string();
let mut writer_builder = csv::WriterBuilder::new();
let writer_builder = writer_builder.delimiter(b',').double_quote(false);
Expand All @@ -179,7 +173,7 @@ impl ExportCommand {
let mode = ExportMode::try_from(self.mode.unwrap_or(ExportMode::Genesis.to_num()))?;
match mode {
ExportMode::Genesis => {
Self::export_genesis(&rooch_db.moveos_store, root_state_root, &mut writer)?;
Self::export_genesis(&moveos_store, root_state_root, &mut writer)?;
}
ExportMode::Full => {
todo!()
Expand All @@ -188,17 +182,17 @@ impl ExportCommand {
todo!()
}
ExportMode::Indexer => {
Self::export_indexer(&rooch_db.moveos_store, root_state_root, &mut writer)?;
Self::export_indexer(&moveos_store, root_state_root, &mut writer)?;
}
ExportMode::Object => {
let obj_id = self
.object_id
.expect("Object id should exist in object mode");
Self::export_object(&rooch_db.moveos_store, root_state_root, obj_id, &mut writer)?;
Self::export_object(&moveos_store, root_state_root, obj_id, &mut writer)?;
}
}

println!("Finish export task.");
log::info!("Done in {:?}.", start_time.elapsed(),);
Ok(())
}

Expand All @@ -211,66 +205,32 @@ impl ExportCommand {
let utxo_store_id = BitcoinUTXOStore::object_id();
let inscription_store_id = InscriptionStore::object_id();
let rooch_to_bitcoin_address_mapping_id = RoochToBitcoinAddressMapping::object_id();
println!("export_genesis utxo_store_id: {:?}", utxo_store_id);
println!(
"export_genesis inscription_store_id: {:?}",
inscription_store_id
);
println!(
"export_genesis rooch_to_bitcoin_address_mapping_id: {:?}",
rooch_to_bitcoin_address_mapping_id
);

let genesis_object_ids = vec![
utxo_store_id.clone(),
inscription_store_id.clone(),
rooch_to_bitcoin_address_mapping_id,
];

//let mut genesis_objects = vec![];
let mut genesis_states = vec![];
let mut genesis_states: Vec<(FieldKey, ObjectState)> = vec![];
for object_id in genesis_object_ids.into_iter() {
let state = moveos_store
.get_field_at(root_state_root, &object_id.field_key())?
.expect("state should exist.");
//TODO
//let object = state.clone().as_raw_object()?;
genesis_states.push((object_id.field_key(), state));
//genesis_objects.push(object);
}

// write csv field states
// for obj in genesis_objects.into_iter() {
// Self::export_field_states(
// moveos_store,
// H256::from(obj.state_root.into_bytes()),
// root_state_root,
// obj.id,
// false,
// true,
// writer,
// )?;
// }

// write csv object states.
// write root state
{
let root_export_id =
ExportID::new(ObjectID::root(), root_state_root, root_state_root, 0);
writer.write_field(GLOBAL_STATE_TYPE_ROOT)?;
writer.write_field(root_export_id.to_string())?;
writer.write_record(None::<&[u8]>)?;
writer.write_record([GLOBAL_STATE_TYPE_ROOT, root_export_id.to_string().as_str()])?;
}
for (k, _v) in genesis_states.into_iter() {
writer.write_field(k.to_string())?;
//TODO write ObjectState csv
//writer.write_field(v.to_string())?;
writer.write_record(None::<&[u8]>)?;
for (k, v) in genesis_states.into_iter() {
writer.write_record([k.to_string().as_str(), v.to_string().as_str()])?;
}

// flush csv writer
writer.flush()?;
println!("export_genesis root state_root: {:?}", root_state_root);

Ok(())
}

Expand Down
8 changes: 4 additions & 4 deletions crates/rooch/src/commands/statedb/commands/genesis_ord.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use std::str::FromStr;
use std::sync::mpsc::{Receiver, SyncSender};
use std::sync::{mpsc, Arc};
use std::thread;
use std::time::SystemTime;
use std::time::{Instant, SystemTime};

use anyhow::Result;
use bitcoin::OutPoint;
Expand Down Expand Up @@ -47,7 +47,7 @@ use crate::commands::statedb::commands::genesis_utxo::{
};
use crate::commands::statedb::commands::import::{apply_fields, apply_nodes};
use crate::commands::statedb::commands::{
get_ord_by_outpoint, init_genesis_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
get_ord_by_outpoint, init_job, sort_merge_utxo_ords, UTXOOrds, UTXO_ORD_MAP_TABLE,
};

pub const ADDRESS_UNBOUND: &str = "unbound";
Expand Down Expand Up @@ -137,7 +137,7 @@ impl GenesisOrdCommand {
pub async fn execute(self) -> RoochResult<()> {
// 1. init import job
let (root, moveos_store, start_time) =
init_genesis_job(self.base_data_dir.clone(), self.chain_id.clone());
init_job(self.base_data_dir.clone(), self.chain_id.clone());
let pre_root_state_root = root.state_root();

let utxo_ord_map_existed = self.utxo_ord_map.exists(); // check if utxo:ords map db existed before create db
Expand Down Expand Up @@ -295,7 +295,7 @@ fn import_utxo(
startup_update_set: UpdateSet<FieldKey, ObjectState>,
root_size: u64,
root_state_root: H256,
startup_time: SystemTime,
startup_time: Instant,
) {
let (tx, rx) = mpsc::sync_channel(2);
let produce_updates_thread = thread::spawn(move || {
Expand Down
80 changes: 28 additions & 52 deletions crates/rooch/src/commands/statedb/commands/genesis_utxo.rs
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,35 @@
// Copyright (c) RoochNetwork
// SPDX-License-Identifier: Apache-2.0

use crate::cli_types::WalletContextOptions;
use crate::commands::statedb::commands::import::{apply_fields, apply_nodes};
use crate::commands::statedb::commands::{
drive_bitcoin_address, get_ord_by_outpoint, SCRIPT_TYPE_NON_STANDARD, SCRIPT_TYPE_P2MS,
SCRIPT_TYPE_P2PK, UTXO_ORD_MAP_TABLE, UTXO_SEAL_INSCRIPTION_PROTOCOL,
};
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, HashMap};
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::mpsc::{Receiver, SyncSender};
use std::sync::{mpsc, Arc};
use std::thread;
use std::time::{Instant, SystemTime};

use anyhow::{Error, Result};
use bitcoin::{OutPoint, Txid};
use chrono::{DateTime, Local};
use clap::Parser;
use move_core_types::account_address::AccountAddress;
use redb::{Database, ReadOnlyTable};
use serde::{Deserialize, Serialize};

use moveos_store::MoveOSStore;
use moveos_types::h256::H256;
use moveos_types::move_std::string::MoveString;
use moveos_types::moveos_std::object::{
ObjectEntity, ObjectID, ObjectMeta, GENESIS_STATE_ROOT, SHARED_OBJECT_FLAG_MASK,
SYSTEM_OWNER_ADDRESS,
ObjectEntity, ObjectID, GENESIS_STATE_ROOT, SHARED_OBJECT_FLAG_MASK, SYSTEM_OWNER_ADDRESS,
};
use moveos_types::moveos_std::simple_multimap::{Element, SimpleMultiMap};
use moveos_types::startup_info::StartupInfo;
use moveos_types::state::{FieldKey, ObjectState};
use redb::{Database, ReadOnlyTable};
use rooch_common::fs::file_cache::FileCacheManager;
use rooch_config::{RoochOpt, R_OPT_NET_HELP};
use rooch_db::RoochDB;
use rooch_config::R_OPT_NET_HELP;
use rooch_types::address::BitcoinAddress;
use rooch_types::addresses::BITCOIN_MOVE_ADDRESS;
use rooch_types::bitcoin::utxo::{BitcoinUTXOStore, UTXO};
Expand All @@ -34,18 +38,14 @@ use rooch_types::error::{RoochError, RoochResult};
use rooch_types::framework::address_mapping::RoochToBitcoinAddressMapping;
use rooch_types::into_address::IntoAddress;
use rooch_types::rooch_network::RoochChainID;
use serde::{Deserialize, Serialize};
use smt::UpdateSet;
use std::collections::hash_map::Entry;
use std::collections::{BTreeMap, HashMap};
use std::fs::File;
use std::io::{BufRead, BufReader, Read};
use std::path::PathBuf;
use std::str::FromStr;
use std::sync::mpsc::{Receiver, SyncSender};
use std::sync::{mpsc, Arc};
use std::thread;
use std::time::SystemTime;

use crate::cli_types::WalletContextOptions;
use crate::commands::statedb::commands::import::{apply_fields, apply_nodes};
use crate::commands::statedb::commands::{
drive_bitcoin_address, get_ord_by_outpoint, init_job, SCRIPT_TYPE_NON_STANDARD,
SCRIPT_TYPE_P2MS, SCRIPT_TYPE_P2PK, UTXO_ORD_MAP_TABLE, UTXO_SEAL_INSCRIPTION_PROTOCOL,
};

/// Genesis Import UTXO
#[derive(Debug, Parser)]
Expand Down Expand Up @@ -77,7 +77,8 @@ impl GenesisUTXOCommand {
pub async fn execute(self) -> RoochResult<()> {
let input_path = self.input.clone();
let batch_size = self.batch_size.unwrap();
let (root, moveos_store, start_time) = self.init();
let (root, moveos_store, start_time) =
init_job(self.base_data_dir.clone(), self.chain_id.clone());
let pre_root_state_root = root.state_root();
let (tx, rx) = mpsc::sync_channel(2);
let moveos_store = Arc::new(moveos_store);
Expand All @@ -98,31 +99,6 @@ impl GenesisUTXOCommand {

Ok(())
}

fn init(self) -> (ObjectMeta, MoveOSStore, SystemTime) {
let start_time = SystemTime::now();
let datetime: DateTime<Local> = start_time.into();

let opt = RoochOpt::new_with_default(self.base_data_dir, self.chain_id, None).unwrap();
let rooch_db = RoochDB::init(opt.store_config()).unwrap();
let root = rooch_db.latest_root().unwrap().unwrap();

let utxo_store_id = BitcoinUTXOStore::object_id();
let address_mapping_id = RoochToBitcoinAddressMapping::object_id();

println!(
"task progress started at {}, batch_size: {}",
datetime,
self.batch_size.unwrap()
);
println!("root object: {:?}", root);
println!("utxo_store_id: {:?}", utxo_store_id);
println!(
"rooch to bitcoin address_mapping_id: {:?}",
address_mapping_id
);
(root, rooch_db.moveos_store, start_time)
}
}

#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
Expand Down Expand Up @@ -227,7 +203,7 @@ pub fn apply_utxo_updates_to_state(

startup_update_set: Option<UpdateSet<FieldKey, ObjectState>>,

task_start_time: SystemTime,
task_start_time: Instant,
) {
let moveos_store = &moveos_store.clone();
let mut utxo_count = 0;
Expand Down Expand Up @@ -315,7 +291,7 @@ fn finish_task(
utxo_store_state_root: H256,
rooch_to_bitcoin_address_mapping_state_root: H256,

task_start_time: SystemTime,
task_start_time: Instant,
startup_update_set: Option<UpdateSet<FieldKey, ObjectState>>,
) {
// Update UTXOStore Object
Expand Down Expand Up @@ -364,7 +340,7 @@ fn finish_task(
let startup_info = moveos_store.get_config_store().get_startup_info().unwrap();
println!(
"Done in {:?}. New startup_info: {:?}",
task_start_time.elapsed().unwrap(),
task_start_time.elapsed(),
startup_info
);
}
Expand Down
Loading

0 comments on commit 8d0efa3

Please sign in to comment.