From 64a8a3873d85521085ec9ee74948a4d6b77aaade Mon Sep 17 00:00:00 2001 From: stringhandler Date: Fri, 20 Dec 2024 12:26:06 +0200 Subject: [PATCH] feat: more lmdb optimizations --- src/server/http/stats/handlers.rs | 2 +- src/server/p2p/network.rs | 2 +- src/sharechain/in_memory.rs | 3 +- src/sharechain/p2chain.rs | 92 +++++++++++++++++++++---------- src/sharechain/p2chain_level.rs | 75 +++++++++++++++++-------- 5 files changed, 119 insertions(+), 55 deletions(-) diff --git a/src/server/http/stats/handlers.rs b/src/server/http/stats/handlers.rs index 630368d..ee1196d 100644 --- a/src/server/http/stats/handlers.rs +++ b/src/server/http/stats/handlers.rs @@ -11,7 +11,7 @@ use axum::{ use log::{error, info}; use serde::Serialize; use tari_core::proof_of_work::PowAlgorithm; -use tari_utilities::{epoch_time::EpochTime, hex::Hex}; +use tari_utilities::{encoding::Base58, epoch_time::EpochTime, hex::Hex}; use tokio::sync::oneshot; use super::MAX_ACCEPTABLE_HTTP_TIMEOUT; diff --git a/src/server/p2p/network.rs b/src/server/p2p/network.rs index 610324a..027f98f 100644 --- a/src/server/p2p/network.rs +++ b/src/server/p2p/network.rs @@ -59,7 +59,7 @@ use tari_common::configuration::Network; use tari_common_types::types::FixedHash; use tari_core::proof_of_work::PowAlgorithm; use tari_shutdown::ShutdownSignal; -use tari_utilities::{epoch_time::EpochTime, hex::Hex}; +use tari_utilities::{encoding::Base58, epoch_time::EpochTime, hex::Hex}; use tokio::{ select, sync::{ diff --git a/src/sharechain/in_memory.rs b/src/sharechain/in_memory.rs index 317bf8e..f87ffa0 100644 --- a/src/sharechain/in_memory.rs +++ b/src/sharechain/in_memory.rs @@ -18,7 +18,7 @@ use tari_core::{ PowAlgorithm, }, }; -use tari_utilities::{epoch_time::EpochTime, hex::Hex}; +use tari_utilities::{encoding::Base58, epoch_time::EpochTime, hex::Hex}; use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use super::{ @@ -76,6 +76,7 @@ impl InMemoryShareChain { Ok(Self { p2_chain: Arc::new(RwLock::new(P2Chain::new_empty( + pow_algo, config.share_window * 2, config.share_window, config.block_time, diff --git a/src/sharechain/p2chain.rs b/src/sharechain/p2chain.rs index f317343..d74cad3 100644 --- a/src/sharechain/p2chain.rs +++ b/src/sharechain/p2chain.rs @@ -30,11 +30,12 @@ use std::{ use itertools::Itertools; use log::*; +use minotari_app_grpc::tari_rpc::PowAlgo; use tari_common_types::types::FixedHash; -use tari_core::proof_of_work::{lwma_diff::LinearWeightedMovingAverage, AccumulatedDifficulty}; +use tari_core::proof_of_work::{lwma_diff::LinearWeightedMovingAverage, AccumulatedDifficulty, PowAlgorithm}; use tari_utilities::hex::Hex; -use super::lmdb_block_storage::BlockCache; +use super::{lmdb_block_storage::BlockCache, p2chain_level::P2BlockHeader}; use crate::sharechain::{ error::ShareChainError, in_memory::MAX_UNCLE_AGE, @@ -125,6 +126,7 @@ impl Display for ChainAddResult { } pub struct P2Chain { + pub algo: PowAlgorithm, pub block_time: u64, block_cache: Arc, pub cached_shares: Option)>>, @@ -164,6 +166,11 @@ impl P2Chain { level.get(hash) } + pub fn get_block_header_at_height(&self, height: u64, hash: &FixedHash) -> Option { + let level = self.level_at_height(height)?; + level.get_header(hash) + } + pub fn block_exists(&self, height: u64, hash: &FixedHash) -> bool { self.level_at_height(height).map_or(false, |level| level.contains(hash)) } @@ -174,11 +181,12 @@ impl P2Chain { level.get(&level.chain_block()) } - pub fn new_empty(total_size: u64, share_window: u64, block_time: u64, block_cache: T) -> Self { + pub fn new_empty(algo: PowAlgorithm, total_size: u64, share_window: u64, block_time: u64, block_cache: T) -> Self { let levels = HashMap::new(); let lwma = LinearWeightedMovingAverage::new(DIFFICULTY_ADJUSTMENT_WINDOW, block_time).expect("Failed to create LWMA"); Self { + algo, block_cache: Arc::new(block_cache), block_time, cached_shares: None, @@ -259,22 +267,26 @@ impl P2Chain { trace!(target: LOG_TARGET, "Trying to verify new block to add: {}:{}", new_block_height, &hash.to_hex()[0..8]); // we should validate what we can if a block is invalid, we should delete it. let mut new_tip = ChainAddResult::default(); - let block = self - .get_block_at_height(new_block_height, &hash) - .ok_or(ShareChainError::BlockNotFound)? - .clone(); - let algo = block.original_header.pow.pow_algo; + let block_prev_hash = self + .get_parent_of(new_block_height, &hash) + .ok_or(ShareChainError::BlockNotFound)?; + // let block = self + // .get_block_at_height(new_block_height, &hash) + // .ok_or(ShareChainError::BlockNotFound)? + // .clone(); + // let algo = block.original_header.pow.pow_algo; // do we know of the parent // we should not check the chain start for parents - if block.height != 0 { - if !self.block_exists(new_block_height.saturating_sub(1), &block.prev_hash) { + if new_block_height != 0 { + if !self.block_exists(new_block_height.saturating_sub(1), &block_prev_hash) { // we dont know the parent new_tip .missing_blocks - .insert(block.prev_hash, new_block_height.saturating_sub(1)); + .insert(block_prev_hash, new_block_height.saturating_sub(1)); } // now lets check the uncles - for uncle in &block.uncles { + + for uncle in &self.get_uncles(new_block_height, &hash) { if let Some(uncle_parent_hash) = self.get_parent_of(uncle.0, &uncle.1) { if !self.block_exists(uncle.0.saturating_sub(1), &uncle_parent_hash) { new_tip @@ -295,10 +307,12 @@ impl P2Chain { self.verify_block(hash, new_block_height)?; // we have to reload the block to check if verified is set to true now let block = self - .get_block_at_height(new_block_height, &hash) + .get_block_header_at_height(new_block_height, &hash) .ok_or(ShareChainError::BlockNotFound)? .clone(); + let algo = self.algo; + // edge case for chain start if self.get_tip().is_none() && new_block_height == 0 { self.set_new_tip(new_block_height, hash)?; @@ -349,18 +363,21 @@ impl P2Chain { if current_counting_block.height == 0 { break; } - if let Some(parent) = self.get_parent_block(¤t_counting_block) { + if let Some(parent) = self.get_block_header_at_height( + current_counting_block.height.saturating_sub(1), + ¤t_counting_block.prev_hash, + ) { if !parent.verified { all_blocks_verified = false; // so this block is unverified, we cannot count it but lets see if it just misses some blocks so // we can ask for them - if self.get_parent_block(&parent).is_none() { + if !self.block_exists(parent.height.saturating_sub(1), &parent.prev_hash) { new_tip .missing_blocks .insert(parent.prev_hash, parent.height.saturating_sub(1)); } for uncle in &parent.uncles { - if self.get_block_at_height(uncle.0, &uncle.1).is_none() { + if !self.block_exists(uncle.0, &uncle.1) { new_tip.missing_blocks.insert(uncle.1, uncle.0); } } @@ -385,7 +402,12 @@ impl P2Chain { break; } // we can unwrap as we now the parent exists - current_counting_block = self.get_parent_block(¤t_counting_block).unwrap().clone(); + current_counting_block = self + .get_block_header_at_height( + current_counting_block.height.saturating_sub(1), + ¤t_counting_block.prev_hash, + ) + .ok_or(ShareChainError::BlockNotFound)?; } if !all_blocks_verified { @@ -397,13 +419,13 @@ impl P2Chain { let next_level_data = self.calculate_next_level_data(new_block_height, hash); return Ok((new_tip, next_level_data)); } - if block.total_pow() > self.total_accumulated_tip_difficulty() { + if block.total_pow > self.total_accumulated_tip_difficulty() { new_tip.set_new_tip(hash, new_block_height); // we need to reorg the chain // lets start by resetting the lwma self.lwma = LinearWeightedMovingAverage::new(DIFFICULTY_ADJUSTMENT_WINDOW, self.block_time) .expect("Failed to create LWMA"); - self.lwma.add_front(block.timestamp, block.target_difficulty()); + self.lwma.add_front(block.timestamp, block.target_difficulty); let chain_height = self .level_at_height(block.height) .ok_or(ShareChainError::BlockLevelNotFound)?; @@ -426,13 +448,13 @@ impl P2Chain { let parent_level = self.level_at_height(current_block.height.saturating_sub(1)).unwrap(); if current_block.prev_hash != parent_level.chain_block() { // safety check - let nextblock = parent_level.get(¤t_block.prev_hash); + let nextblock = parent_level.get_header(¤t_block.prev_hash); if nextblock.is_none() { error!(target: LOG_TARGET, "FATAL: Reorging (block in chain) failed because parent block was not found and chain data is corrupted."); panic!( "FATAL: Reorging (block in chain) failed because parent block was not found and chain \ data is corrupted. current_block: {:?}, current tip: {:?}", - current_block, + current_block.height, self.get_tip().map(|t| t.height()) ); } @@ -441,17 +463,17 @@ impl P2Chain { mut_parent_level.set_chain_block(current_block.prev_hash); current_block = nextblock.unwrap().clone(); self.lwma - .add_front(current_block.timestamp, current_block.target_difficulty()); + .add_front(current_block.timestamp, current_block.target_difficulty); } else if !self.lwma.is_full() { // we still need more blocks to fill up the lwma - let nextblock = parent_level.get(¤t_block.prev_hash); + let nextblock = parent_level.get_header(¤t_block.prev_hash); if nextblock.is_none() { error!(target: LOG_TARGET, "FATAL: Reorging (block not in chain) failed because parent block was not found and chain data is corrupted."); panic!( "FATAL: Could not calculate LMWA while reorging (block in chain) failed because \ parent block was not found and chain data is corrupted. current_block: {:?}, current \ tip: {:?}", - current_block, + current_block.height, self.get_tip().map(|t| t.height()) ); } @@ -459,7 +481,7 @@ impl P2Chain { current_block = nextblock.unwrap().clone(); self.lwma - .add_front(current_block.timestamp, current_block.target_difficulty()); + .add_front(current_block.timestamp, current_block.target_difficulty); } else { break; } @@ -494,16 +516,23 @@ impl P2Chain { next_level_data } - // this assumes it has no missing parents - fn verify_block(&mut self, hash: FixedHash, height: u64) -> Result<(), ShareChainError> { + fn is_verified(&self, hash: &FixedHash, height: u64) -> Result { let level = self .level_at_height(height) .ok_or(ShareChainError::BlockLevelNotFound)?; - let block = level.get(&hash).ok_or(ShareChainError::BlockNotFound)?; - if block.verified { + Ok(level.is_verified(hash)) + } + + // this assumes it has no missing parents + fn verify_block(&mut self, hash: FixedHash, height: u64) -> Result<(), ShareChainError> { + if self.is_verified(&hash, height)? { return Ok(()); } let verified = true; + let level = self + .level_at_height(height) + .ok_or(ShareChainError::BlockLevelNotFound)?; + let block = level.get(&hash).ok_or(ShareChainError::BlockNotFound)?; // lets check the total accumulated difficulty let mut total_work = AccumulatedDifficulty::from_u128(u128::from(block.target_difficulty().as_u64())) @@ -592,6 +621,11 @@ impl P2Chain { level.get_prev_hash(hash) } + pub fn get_uncles(&self, height: u64, hash: &FixedHash) -> Vec<(u64, FixedHash)> { + let level = self.level_at_height(height).unwrap(); + level.get_uncles(hash) + } + pub fn get_parent_block(&self, block: &P2Block) -> Option> { let parent_height = match block.height.checked_sub(1) { Some(height) => height, diff --git a/src/sharechain/p2chain_level.rs b/src/sharechain/p2chain_level.rs index bc29f18..8a6519e 100644 --- a/src/sharechain/p2chain_level.rs +++ b/src/sharechain/p2chain_level.rs @@ -21,18 +21,28 @@ // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH // DAMAGE. -use std::sync::{Arc, RwLock}; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; use tari_common_types::types::{BlockHash, FixedHash}; +use tari_core::proof_of_work::{AccumulatedDifficulty, Difficulty}; +use tari_utilities::epoch_time::EpochTime; use super::lmdb_block_storage::BlockCache; use crate::sharechain::{error::ShareChainError, p2block::P2Block}; -struct P2BlockHeader { - height: u64, - hash: FixedHash, - prev_hash: FixedHash, - uncles: Vec<(u64, FixedHash)>, +#[derive(Clone)] +pub(crate) struct P2BlockHeader { + pub height: u64, + pub hash: FixedHash, + pub prev_hash: FixedHash, + pub timestamp: EpochTime, + pub target_difficulty: Difficulty, + pub total_pow: AccumulatedDifficulty, + pub verified: bool, + pub uncles: Vec<(u64, FixedHash)>, } /// A collection of blocks with the same height. pub struct P2ChainLevel { @@ -40,7 +50,7 @@ pub struct P2ChainLevel { block_cache: Arc, height: u64, chain_block: RwLock, - block_headers: RwLock>, + block_headers: RwLock>, } impl P2ChainLevel { @@ -54,14 +64,21 @@ impl P2ChainLevel { hash: block.hash, prev_hash: block.prev_hash, uncles: block.uncles.clone(), + timestamp: block.timestamp, + target_difficulty: block.target_difficulty(), + total_pow: block.total_pow(), + verified: block.verified, }; + let mut block_headers = HashMap::new(); + block_headers.insert(block.hash, header); + block_cache.insert(block.hash, block); Self { block_cache, height, chain_block, - block_headers: RwLock::new(vec![header]), + block_headers: RwLock::new(block_headers), } } @@ -69,7 +86,7 @@ impl P2ChainLevel { let mut res = vec![]; // TODO: Optimize let lock = self.block_headers.read().expect("could not lock"); - for block in lock.iter() { + for block in lock.values() { for uncles in &block.uncles { if &uncles.1 == hash { res.push((block.height, block.hash)); @@ -84,12 +101,17 @@ impl P2ChainLevel { pub fn get_prev_hash(&self, hash: &FixedHash) -> Option { let lock = self.block_headers.read().expect("could not lock"); - for block in lock.iter() { - if &block.hash == hash { - return Some(block.prev_hash); - } - } - None + lock.get(hash).map(|b| b.prev_hash) + } + + pub fn get_uncles(&self, hash: &FixedHash) -> Vec<(u64, FixedHash)> { + let lock = self.block_headers.read().expect("could not lock"); + lock.get(hash).map(|b| b.uncles.clone()).unwrap_or_default() + } + + pub fn is_verified(&self, hash: &FixedHash) -> bool { + let lock = self.block_headers.read().expect("could not lock"); + lock.get(hash).map(|b| b.verified).unwrap_or(false) } pub fn height(&self) -> u64 { @@ -116,8 +138,15 @@ impl P2ChainLevel { hash: block.hash, prev_hash: block.prev_hash, uncles: block.uncles.clone(), + timestamp: block.timestamp, + target_difficulty: block.target_difficulty(), + total_pow: block.total_pow(), + verified: block.verified, }; - self.block_headers.write().expect("could not lock").push(header); + self.block_headers + .write() + .expect("could not lock") + .insert(block.hash, header); self.block_cache.insert(block.hash, block); Ok(()) } @@ -130,20 +159,20 @@ impl P2ChainLevel { self.block_cache.get(hash) } + pub fn get_header(&self, hash: &BlockHash) -> Option { + self.block_headers.read().expect("could not lock").get(hash).cloned() + } + pub fn contains(&self, hash: &BlockHash) -> bool { - self.block_headers - .read() - .expect("could not lock") - .iter() - .any(|b| b.hash == *hash) + self.block_headers.read().expect("could not lock").contains_key(hash) } pub fn all_blocks(&self) -> Vec> { self.block_headers .read() .expect("could not lock") - .iter() - .filter_map(|header| self.block_cache.get(&header.hash)) + .keys() + .filter_map(|hash| self.block_cache.get(&hash)) .collect() } }