From f37ad839d9c6baa0e9f85bb21700e52e055d77c5 Mon Sep 17 00:00:00 2001 From: Odysseas Gabrielides Date: Thu, 15 Aug 2024 12:20:57 +0700 Subject: [PATCH 01/19] verify references after applying batch ops --- grovedb/src/batch/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7f9c119e..d1f86462 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -59,9 +59,7 @@ use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, Storage, StorageBatch, StorageContext, }; -use grovedb_version::{ - check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, -}; +use grovedb_version::{check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, TryIntoVersioned}; use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; @@ -2060,6 +2058,9 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, Some(tx)) .map_err(|e| e.into()) ); + + let issues = self.visualize_verify_grovedb(true, &Default::default()).unwrap(); + println!("tx_issues are {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); } else { cost_return_on_error!( &mut cost, @@ -2087,6 +2088,9 @@ impl GroveDb { .commit_multi_context_batch(storage_batch, None) .map_err(|e| e.into()) ); + + let issues = self.visualize_verify_grovedb(true, &Default::default()).unwrap(); + println!("non_tx_issues are {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); } Ok(()).wrap_with_cost(cost) } From 66e7611cf5811b437f28946c2e2c38b1f4fc446d Mon Sep 17 00:00:00 2001 From: Odysseas Gabrielides Date: Thu, 15 Aug 2024 12:39:03 +0700 Subject: [PATCH 02/19] fix --- grovedb/src/batch/mod.rs | 12 ++++++++---- grovedb/src/lib.rs | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index d1f86462..a41e75ee 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -2059,8 +2059,10 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self.visualize_verify_grovedb(true, &Default::default()).unwrap(); - println!("tx_issues are {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + let issues = self.visualize_verify_grovedb(Some(tx), true, &Default::default()).unwrap(); + if issues.len() > 0 { + println!("tx_issues: {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + } } else { cost_return_on_error!( &mut cost, @@ -2089,8 +2091,10 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self.visualize_verify_grovedb(true, &Default::default()).unwrap(); - println!("non_tx_issues are {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + let issues = self.visualize_verify_grovedb(None, true, &Default::default()).unwrap(); + if issues.len() > 0 { + println!("non_tx_issues: {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + } } Ok(()).wrap_with_cost(cost) } diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 96c2d0de..1665b6cf 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -899,11 +899,12 @@ impl GroveDb { /// Method to visualize hash mismatch after verification pub fn visualize_verify_grovedb( &self, + transaction: TransactionArg, verify_references: bool, grove_version: &GroveVersion, ) -> Result, Error> { Ok(self - .verify_grovedb(None, verify_references, grove_version)? + .verify_grovedb(transaction, verify_references, grove_version)? .iter() .map(|(path, (root_hash, expected, actual))| { ( From 9c186b04efa43af80469f1f207b905b450155f5c Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Fri, 16 Aug 2024 10:13:08 +0700 Subject: [PATCH 03/19] merge_qualified_paths --- grovedb/src/batch/batch_structure.rs | 29 +-- .../estimated_costs/average_case_costs.rs | 29 +-- .../batch/estimated_costs/worst_case_costs.rs | 29 +-- grovedb/src/batch/mod.rs | 234 +++++++++++------- grovedb/src/lib.rs | 16 +- grovedb/src/operations/delete/mod.rs | 6 +- 6 files changed, 198 insertions(+), 145 deletions(-) diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index eb00f1e8..64f36992 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -16,12 +16,13 @@ use nohash_hasher::IntMap; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath, Op, TreeCache}, + batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath, GroveOp, TreeCache}, Element, ElementFlags, Error, }; +use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] -pub type OpsByPath = BTreeMap>; +pub type OpsByPath = BTreeMap>; /// Level, path, key, op #[cfg(feature = "full")] pub type OpsByLevelPath = IntMap; @@ -32,7 +33,7 @@ pub(super) struct BatchStructure { /// Operations by level path pub(super) ops_by_level_paths: OpsByLevelPath, /// This is for references - pub(super) ops_by_qualified_paths: BTreeMap>, Op>, + pub(super) ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp>, /// Merk trees /// Very important: the type of run mode we are in is contained in this /// cache @@ -112,15 +113,15 @@ where let mut current_last_level: u32 = 0; // qualified paths meaning path + key - let mut ops_by_qualified_paths: BTreeMap>, Op> = BTreeMap::new(); + let mut ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp> = BTreeMap::new(); for op in ops.into_iter() { let mut path = op.path.clone(); path.push(op.key.clone()); - ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone()); + ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone().into()); let op_cost = OperationCost::default(); let op_result = match &op.op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -128,10 +129,10 @@ where } Ok(()) } - Op::RefreshReference { .. } | Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { Ok(()) } - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => { + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", )) @@ -144,16 +145,16 @@ where let level = op.path.len(); if let Some(ops_on_level) = ops_by_level_paths.get_mut(&level) { if let Some(ops_on_path) = ops_on_level.get_mut(&op.path) { - ops_on_path.insert(op.key, op.op); + ops_on_path.insert(op.key, op.op.into()); } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); - ops_on_path.insert(op.key, op.op); + let mut ops_on_path: BTreeMap = BTreeMap::new(); + ops_on_path.insert(op.key, op.op.into()); ops_on_level.insert(op.path.clone(), ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); - ops_on_path.insert(op.key, op.op); - let mut ops_on_level: BTreeMap> = + let mut ops_on_path: BTreeMap = BTreeMap::new(); + ops_on_path.insert(op.key, op.op.into()); + let mut ops_on_level: BTreeMap> = BTreeMap::new(); ops_on_level.insert(op.path, ops_on_path); ops_by_level_paths.insert(level, ops_on_level); diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 2f50186b..a31ed530 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -26,14 +26,15 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, GroveOp, TreeCache, }, Error, GroveDb, }; +use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] -impl Op { +impl GroveOp { /// Get the estimated average case cost of the op. Calls a lower level /// function to calculate the estimate based on the type of op. Returns /// CostResult. @@ -53,14 +54,14 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::average_case_merk_replace_tree( key, layer_element_estimates, sum.is_some(), propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => { + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { GroveDb::average_case_merk_insert_tree( key, flags, @@ -70,14 +71,14 @@ impl Op { grove_version, ) } - Op::Insert { element } => GroveDb::average_case_merk_insert_element( + GroveOp::Insert { element } => GroveDb::average_case_merk_insert_element( key, element, in_tree_using_sums, propagate_if_input(), grove_version, ), - Op::RefreshReference { + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -93,14 +94,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::average_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, in_tree_using_sums, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes, } => GroveDb::average_case_merk_patch_element( @@ -111,20 +112,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::average_case_merk_delete_element( + GroveOp::Delete => GroveDb::average_case_merk_delete_element( key, layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::average_case_merk_delete_tree( key, false, layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::average_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::average_case_merk_delete_tree( key, true, layer_element_estimates, @@ -184,8 +185,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -247,7 +248,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op.average_case_cost(&key, layer_element_estimates, false, grove_version) + op.op().average_case_cost(&key, layer_element_estimates, false, grove_version) ); } diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 1b1d42e7..170cf020 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -25,14 +25,15 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, Op, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, GroveOp, TreeCache, }, Error, GroveDb, }; +use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] -impl Op { +impl GroveOp { fn worst_case_cost( &self, key: &KeyInfo, @@ -49,7 +50,7 @@ impl Op { } }; match self { - Op::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( + GroveOp::ReplaceTreeRootKey { sum, .. } => GroveDb::worst_case_merk_replace_tree( key, sum.is_some(), is_in_parent_sum_tree, @@ -57,7 +58,7 @@ impl Op { propagate, grove_version, ), - Op::InsertTreeWithRootHash { flags, sum, .. } => GroveDb::worst_case_merk_insert_tree( + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => GroveDb::worst_case_merk_insert_tree( key, flags, sum.is_some(), @@ -65,14 +66,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Insert { element } => GroveDb::worst_case_merk_insert_element( + GroveOp::Insert { element } => GroveDb::worst_case_merk_insert_element( key, element, is_in_parent_sum_tree, propagate_if_input(), grove_version, ), - Op::RefreshReference { + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -88,14 +89,14 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Replace { element } => GroveDb::worst_case_merk_replace_element( + GroveOp::Replace { element } => GroveDb::worst_case_merk_replace_element( key, element, is_in_parent_sum_tree, propagate_if_input(), grove_version, ), - Op::Patch { + GroveOp::Patch { element, change_in_bytes: _, } => GroveDb::worst_case_merk_replace_element( @@ -105,20 +106,20 @@ impl Op { propagate_if_input(), grove_version, ), - Op::Delete => GroveDb::worst_case_merk_delete_element( + GroveOp::Delete => GroveDb::worst_case_merk_delete_element( key, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteTree => GroveDb::worst_case_merk_delete_tree( key, false, worst_case_layer_element_estimates, propagate, grove_version, ), - Op::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( + GroveOp::DeleteSumTree => GroveDb::worst_case_merk_delete_tree( key, true, worst_case_layer_element_estimates, @@ -178,8 +179,8 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -214,7 +215,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op.worst_case_cost( + op.op().worst_case_cost( &key, false, worst_case_layer_element_estimates, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index a41e75ee..2a62a5cf 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -46,14 +46,10 @@ use grovedb_costs::{ }, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{ - tree::{ - kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, - }, - CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, -}; +use grovedb_merk::{tree::{ + kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + value_hash, NULL_HASH, +}, CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, Op}; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, @@ -79,9 +75,61 @@ use crate::{ Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; +/// An operation with extra information about merge status +/// Merging happens for some operations that will could change the element +/// with just in time element flag updates. +/// +/// If already merged is true and merged_op doesn't exist that means that there is no +/// point to try to merge again. +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct MergeQualifiedGroveOp { + unmerged_op: GroveOp, + merged_op: Option, + already_merged: bool, +} + +impl MergeQualifiedGroveOp { + #[inline] + pub fn op(&self) -> &GroveOp { + if let Some(op) = &self.merged_op { + op + } else { + &self.unmerged_op + } + } + + #[inline] + pub fn op_mut(&mut self) -> &mut GroveOp { + if let Some(op) = &mut self.merged_op { + op + } else { + &mut self.unmerged_op + } + } + + #[inline] + pub fn take_op(self) -> GroveOp { + if let Some(op) = self.merged_op { + op + } else { + self.unmerged_op + } + } +} + +impl From for MergeQualifiedGroveOp { + fn from(value: GroveOp) -> Self { + MergeQualifiedGroveOp { + unmerged_op: value, + merged_op: None, + already_merged: false, + } + } +} + /// Operations #[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub enum Op { +pub enum GroveOp { /// Replace tree root key ReplaceTreeRootKey { /// Hash @@ -139,29 +187,29 @@ pub enum Op { DeleteSumTree, } -impl Op { +impl GroveOp { fn to_u8(&self) -> u8 { match self { - Op::DeleteTree => 0, - Op::DeleteSumTree => 1, - Op::Delete => 2, - Op::InsertTreeWithRootHash { .. } => 3, - Op::ReplaceTreeRootKey { .. } => 4, - Op::RefreshReference { .. } => 5, - Op::Replace { .. } => 6, - Op::Patch { .. } => 7, - Op::Insert { .. } => 8, + GroveOp::DeleteTree => 0, + GroveOp::DeleteSumTree => 1, + GroveOp::Delete => 2, + GroveOp::InsertTreeWithRootHash { .. } => 3, + GroveOp::ReplaceTreeRootKey { .. } => 4, + GroveOp::RefreshReference { .. } => 5, + GroveOp::Replace { .. } => 6, + GroveOp::Patch { .. } => 7, + GroveOp::Insert { .. } => 8, } } } -impl PartialOrd for Op { +impl PartialOrd for GroveOp { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Ord for Op { +impl Ord for GroveOp { fn cmp(&self, other: &Self) -> Ordering { self.to_u8().cmp(&other.to_u8()) } @@ -340,7 +388,7 @@ pub struct GroveDbOp { /// Key of an element in the subtree pub key: KeyInfo, /// Operation to perform on the key - pub op: Op, + pub op: GroveOp, } impl fmt::Debug for GroveDbOp { @@ -353,10 +401,10 @@ impl fmt::Debug for GroveDbOp { self.key.visualize(key_drawer).unwrap(); let op_dbg = match &self.op { - Op::Insert { element } => format!("Insert {:?}", element), - Op::Replace { element } => format!("Replace {:?}", element), - Op::Patch { element, .. } => format!("Patch {:?}", element), - Op::RefreshReference { + GroveOp::Insert { element } => format!("Insert {:?}", element), + GroveOp::Replace { element } => format!("Replace {:?}", element), + GroveOp::Patch { element, .. } => format!("Patch {:?}", element), + GroveOp::RefreshReference { reference_path_type, max_reference_hop, trust_refresh_reference, @@ -367,11 +415,11 @@ impl fmt::Debug for GroveDbOp { reference_path_type, max_reference_hop, trust_refresh_reference ) } - Op::Delete => "Delete".to_string(), - Op::DeleteTree => "Delete Tree".to_string(), - Op::DeleteSumTree => "Delete Sum Tree".to_string(), - Op::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), - Op::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), + GroveOp::Delete => "Delete".to_string(), + GroveOp::DeleteTree => "Delete Tree".to_string(), + GroveOp::DeleteSumTree => "Delete Sum Tree".to_string(), + GroveOp::ReplaceTreeRootKey { .. } => "Replace Tree Hash and Root Key".to_string(), + GroveOp::InsertTreeWithRootHash { .. } => "Insert Tree Hash and Root Key".to_string(), }; f.debug_struct("GroveDbOp") @@ -389,7 +437,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Insert { element }, + op: GroveOp::Insert { element }, } } @@ -398,7 +446,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Insert { element }, + op: GroveOp::Insert { element }, } } @@ -408,7 +456,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -417,7 +465,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Replace { element }, + op: GroveOp::Replace { element }, } } @@ -432,7 +480,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -449,7 +497,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Patch { + op: GroveOp::Patch { element, change_in_bytes, }, @@ -469,7 +517,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::RefreshReference { + op: GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -484,7 +532,7 @@ impl GroveDbOp { Self { path, key: KnownKey(key), - op: Op::Delete, + op: GroveOp::Delete, } } @@ -495,9 +543,9 @@ impl GroveDbOp { path, key: KnownKey(key), op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } @@ -507,7 +555,7 @@ impl GroveDbOp { Self { path, key, - op: Op::Delete, + op: GroveOp::Delete, } } @@ -517,9 +565,9 @@ impl GroveDbOp { path, key, op: if is_sum_tree { - Op::DeleteSumTree + GroveOp::DeleteSumTree } else { - Op::DeleteTree + GroveOp::DeleteTree }, } } @@ -562,7 +610,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !doubled_ops.is_empty() { doubled_ops.push(op.op.clone()); same_path_key_ops.push((op.path.clone(), op.key.clone(), doubled_ops)); @@ -572,7 +620,7 @@ impl GroveDbOp { let inserts = ops .iter() .filter_map(|current_op| match current_op.op { - Op::Insert { .. } | Op::Replace { .. } => Some(current_op.clone()), + GroveOp::Insert { .. } | GroveOp::Replace { .. } => Some(current_op.clone()), _ => None, }) .collect::>(); @@ -580,7 +628,7 @@ impl GroveDbOp { let deletes = ops .iter() .filter_map(|current_op| { - if let Op::Delete = current_op.op { + if let GroveOp::Delete = current_op.op { Some(current_op.clone()) } else { None @@ -627,7 +675,7 @@ impl GroveDbOp { #[derive(Debug)] pub struct GroveDbOpConsistencyResults { repeated_ops: Vec<(GroveDbOp, u16)>, // the u16 is count - same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, + same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, insert_ops_below_deleted_ops: Vec<(GroveDbOp, Vec)>, /* the deleted op first, * then inserts under */ } @@ -662,8 +710,8 @@ trait TreeCache { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -726,7 +774,7 @@ where fn process_reference<'a>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, recursions_allowed: u8, intermediate_reference_info: Option<&'a ReferencePathType>, grove_version: &GroveVersion, @@ -865,7 +913,7 @@ where fn follow_reference_get_value_hash<'a>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, Op>, + ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, recursions_allowed: u8, grove_version: &GroveVersion, ) -> CostResult { @@ -875,14 +923,16 @@ where } // If the element being referenced changes in the same batch // we need to set the value_hash based on the new change and not the old state. - if let Some(op) = ops_by_qualified_paths.get(qualified_path) { + + // However the operation might either be merged or unmerged, if it is unmerged we need to merge it with the state first + if let Some(qualified_op) = ops_by_qualified_paths.get(qualified_path) { // the path is being modified, inserted or deleted in the batch of operations - match op { - Op::ReplaceTreeRootKey { .. } | Op::InsertTreeWithRootHash { .. } => Err( + match qualified_op.op() { + GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => Err( Error::InvalidBatchOperation("references can not point to trees being updated"), ) .wrap_with_cost(cost), - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { match element { Element::Item(..) | Element::SumItem(..) => { let serialized = cost_return_on_error_no_add!( @@ -915,7 +965,7 @@ where } } } - Op::RefreshReference { + GroveOp::RefreshReference { reference_path_type, trust_refresh_reference, .. @@ -934,7 +984,7 @@ where grove_version, ) } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { Err(Error::InvalidBatchOperation( "references can not point to something currently being deleted", )) @@ -1000,8 +1050,8 @@ where fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, Op>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1020,10 +1070,10 @@ where let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); let is_sum_tree = merk.is_sum_tree; - let mut batch_operations: Vec<(Vec, _)> = vec![]; + let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { - match op { - Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { + match op.take_op() { + GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { match &element { Element::Reference(path_reference, element_max_reference_hop, _) => { let merk_feature_type = cost_return_on_error!( @@ -1126,7 +1176,7 @@ where } } } - Op::RefreshReference { + GroveOp::RefreshReference { reference_path_type, max_reference_hop, flags, @@ -1212,7 +1262,7 @@ where ) ); } - Op::Delete => { + GroveOp::Delete => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1225,7 +1275,7 @@ where ) ); } - Op::DeleteTree => { + GroveOp::DeleteTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1237,7 +1287,7 @@ where ) ); } - Op::DeleteSumTree => { + GroveOp::DeleteSumTree => { cost_return_on_error!( &mut cost, Element::delete_into_batch_operations( @@ -1249,7 +1299,7 @@ where ) ); } - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, @@ -1267,7 +1317,7 @@ where ) ); } - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash, root_key, flags, @@ -1499,16 +1549,16 @@ impl GroveDb { { match ops_on_path.entry(key.clone()) { Entry::Vacant(vacant_entry) => { - vacant_entry.insert(Op::ReplaceTreeRootKey { + vacant_entry.insert(GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }); + }.into()); } Entry::Occupied(occupied_entry) => { let mutable_occupied_entry = occupied_entry.into_mut(); - match mutable_occupied_entry { - Op::ReplaceTreeRootKey { + match mutable_occupied_entry.op_mut() { + GroveOp::ReplaceTreeRootKey { hash, root_key, sum, @@ -1517,33 +1567,33 @@ impl GroveDb { *root_key = calculated_root_key; *sum = sum_value; } - Op::InsertTreeWithRootHash { .. } => { + GroveOp::InsertTreeWithRootHash { .. } => { return Err(Error::CorruptedCodeExecution( "we can not do this operation twice", )) .wrap_with_cost(cost); } - Op::Insert { element } - | Op::Replace { element } - | Op::Patch { element, .. } => { + GroveOp::Insert { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(_, flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: None, - }; + }.into(); } else if let Element::SumTree(.., flags) = element { *mutable_occupied_entry = - Op::InsertTreeWithRootHash { + GroveOp::InsertTreeWithRootHash { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), sum: sum_value, - }; + }.into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", @@ -1551,14 +1601,14 @@ impl GroveDb { .wrap_with_cost(cost); } } - Op::RefreshReference { .. } => { + GroveOp::RefreshReference { .. } => { return Err(Error::InvalidBatchOperation( "insertion of element under a refreshed \ reference", )) .wrap_with_cost(cost); } - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -1571,28 +1621,28 @@ impl GroveDb { } } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }, + }.into(), ); ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), - Op::ReplaceTreeRootKey { + GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }, + }.into(), ); - let mut ops_on_level: BTreeMap> = + let mut ops_on_level: BTreeMap> = BTreeMap::new(); ops_on_level.insert(KeyInfoPath(parent_path.to_vec()), ops_on_path); ops_by_level_paths.insert(current_level - 1, ops_on_level); @@ -1721,7 +1771,7 @@ impl GroveDb { let mut cost = OperationCost::default(); for op in ops.into_iter() { match op.op { - Op::Insert { element } | Op::Replace { element } => { + GroveOp::Insert { element } | GroveOp::Replace { element } => { // TODO: paths in batches is something to think about let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); @@ -1737,7 +1787,7 @@ impl GroveDb { ) ); } - Op::Delete => { + GroveOp::Delete => { let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); cost_return_on_error!( diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 1665b6cf..059ff9b9 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -985,7 +985,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1027,7 +1027,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1054,7 +1054,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1073,7 +1073,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - true, + false, None, grove_version, ) @@ -1123,7 +1123,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1167,7 +1167,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1194,7 +1194,7 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - true, + false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1213,7 +1213,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - true, + false, Some(transaction), grove_version, ) diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 31d96b85..967b68e7 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -33,7 +33,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::{ - batch::{GroveDbOp, Op}, + batch::{GroveDbOp, GroveOp}, util::storage_context_with_parent_optional_tx, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -565,7 +565,7 @@ impl GroveDb { let batch_deleted_keys = current_batch_operations .iter() .filter_map(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => { + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { // todo: to_path clones (best to figure out how to compare without // cloning) if op.path.to_path() == subtree_merk_path_vec { @@ -595,7 +595,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - Op::Delete | Op::DeleteTree | Op::DeleteSumTree => false, + GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); From 04431074387c7c0113a9c87742b2beb6dca2c8e9 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sun, 18 Aug 2024 01:30:47 +0700 Subject: [PATCH 04/19] working in theory --- grovedb/src/batch/batch_structure.rs | 50 +- .../estimated_costs/average_case_costs.rs | 22 +- .../batch/estimated_costs/worst_case_costs.rs | 37 +- grovedb/src/batch/just_in_time_cost_tests.rs | 136 ++- grovedb/src/batch/mod.rs | 870 ++++++++++++------ grovedb/src/batch/multi_insert_cost_tests.rs | 26 +- grovedb/src/batch/single_insert_cost_tests.rs | 26 +- .../single_sum_item_insert_cost_tests.rs | 24 +- grovedb/src/element/helpers.rs | 1 + grovedb/src/lib.rs | 4 +- grovedb/src/operations/insert/mod.rs | 1 - grovedb/src/tests/mod.rs | 8 +- grovedb/src/tests/query_tests.rs | 26 +- grovedb/src/tests/sum_tree_tests.rs | 26 +- merk/benches/merk.rs | 12 + 15 files changed, 883 insertions(+), 386 deletions(-) diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 64f36992..b07c6b89 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -14,12 +14,12 @@ use grovedb_visualize::{DebugByteVectors, DebugBytes}; #[cfg(feature = "full")] use nohash_hasher::IntMap; +use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath, GroveOp, TreeCache}, + batch::{key_info::KeyInfo, GroveDbOp, GroveOp, KeyInfoPath, TreeCache}, Element, ElementFlags, Error, }; -use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] pub type OpsByPath = BTreeMap>; @@ -113,15 +113,43 @@ where let mut current_last_level: u32 = 0; // qualified paths meaning path + key - let mut ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp> = BTreeMap::new(); + let mut ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp> = + BTreeMap::new(); for op in ops.into_iter() { let mut path = op.path.clone(); path.push(op.key.clone()); - ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone().into()); let op_cost = OperationCost::default(); let op_result = match &op.op { - GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { + match element { + Element::Item(..) | Element::SumItem(..) => { + // let path = path.to_path_consume(); + // let previous_element = Element::get_from_storage() + // todo: path merges here + ops_by_qualified_paths + .insert(path.to_path_consume(), op.op.clone().into()); + } + Element::Reference(..) => { + ops_by_qualified_paths + .insert(path.to_path_consume(), op.op.clone().into()); + } + Element::Tree(..) => { + cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); + ops_by_qualified_paths + .insert(path.to_path_consume(), op.op.clone().into()); + } + Element::SumTree(..) => { + cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, true)); + ops_by_qualified_paths + .insert(path.to_path_consume(), op.op.clone().into()); + } + } + Ok(()) + } + GroveOp::InsertOnly { element } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -129,7 +157,11 @@ where } Ok(()) } - GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { + GroveOp::RefreshReference { .. } + | GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => { + ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone().into()); Ok(()) } GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { @@ -154,8 +186,10 @@ where } else { let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert(op.key, op.op.into()); - let mut ops_on_level: BTreeMap> = - BTreeMap::new(); + let mut ops_on_level: BTreeMap< + KeyInfoPath, + BTreeMap, + > = BTreeMap::new(); ops_on_level.insert(op.path, ops_on_path); ops_by_level_paths.insert(level, ops_on_level); if current_last_level < level { diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index a31ed530..457e6976 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -22,16 +22,15 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::Element; +use crate::{batch::MergeQualifiedGroveOp, Element}; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, GroveOp, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, GroveOp, KeyInfoPath, TreeCache, }, Error, GroveDb, }; -use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] impl GroveOp { @@ -71,7 +70,7 @@ impl GroveOp { grove_version, ) } - GroveOp::Insert { element } => GroveDb::average_case_merk_insert_element( + GroveOp::InsertOrReplace { element } => GroveDb::average_case_merk_insert_element( key, element, in_tree_using_sums, @@ -248,7 +247,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op.op().average_case_cost(&key, layer_element_estimates, false, grove_version) + op.op() + .average_case_cost(&key, layer_element_estimates, false, grove_version) ); } @@ -322,7 +322,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -391,7 +391,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -458,7 +458,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -531,7 +531,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -617,7 +617,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -775,7 +775,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 170cf020..72e13601 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -21,16 +21,15 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::Element; +use crate::{batch::MergeQualifiedGroveOp, Element}; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, KeyInfoPath, GroveOp, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, GroveOp, KeyInfoPath, TreeCache, }, Error, GroveDb, }; -use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] impl GroveOp { @@ -58,15 +57,17 @@ impl GroveOp { propagate, grove_version, ), - GroveOp::InsertTreeWithRootHash { flags, sum, .. } => GroveDb::worst_case_merk_insert_tree( - key, - flags, - sum.is_some(), - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), - GroveOp::Insert { element } => GroveDb::worst_case_merk_insert_element( + GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { + GroveDb::worst_case_merk_insert_tree( + key, + flags, + sum.is_some(), + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } + GroveOp::InsertOrReplace { element } => GroveDb::worst_case_merk_insert_element( key, element, is_in_parent_sum_tree, @@ -287,7 +288,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -342,7 +343,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -397,7 +398,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -463,7 +464,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -529,7 +530,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -593,7 +594,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index e1fddf5c..820992f0 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -5,11 +5,22 @@ mod tests { use std::option::Option::None; + use grovedb_costs::{ + storage_cost::{ + removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, + transition::OperationStorageTransitionType, + StorageCost, + }, + OperationCost, + }; use grovedb_version::version::GroveVersion; + use integer_encoding::VarInt; use crate::{ batch::GroveDbOp, - reference_path::ReferencePathType::UpstreamFromElementHeightReference, + reference_path::{ + ReferencePathType, ReferencePathType::UpstreamFromElementHeightReference, + }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -41,17 +52,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -175,17 +186,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -247,7 +258,7 @@ mod tests { .get(&0) .expect("expected to have root path"); assert_eq!(ops_by_root_path.len(), 1); - let new_ops = vec![GroveDbOp::insert_op( + let new_ops = vec![GroveDbOp::insert_or_replace_op( vec![b"balances".to_vec()], b"person".to_vec(), Element::new_sum_item_with_flags(1000, Some([0, 1].to_vec())), @@ -311,4 +322,113 @@ mod tests { assert_ne!(apply_root_hash, apply_partial_root_hash); } + + #[test] + fn test_batch_root_one_update_tree_bigger_flags_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + EMPTY_PATH, + b"refs", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![ + GroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + GroveDbOp::insert_only_op( + vec![b"refs".to_vec()], + b"ref_key".to_vec(), + Element::new_reference_with_hops( + ReferencePathType::AbsolutePathReference(vec![ + b"tree".to_vec(), + b"key1".to_vec(), + ]), + Some(1), + ), + ), + ]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 2a62a5cf..cae65be5 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -46,16 +46,22 @@ use grovedb_costs::{ }, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{tree::{ - kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, -}, CryptoHash, Error as MerkError, Merk, MerkType, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, Op}; +use grovedb_merk::{ + tree::{ + kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + value_hash, NULL_HASH, + }, + CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, +}; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, Storage, StorageBatch, StorageContext, }; -use grovedb_version::{check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, TryIntoVersioned}; +use grovedb_version::{ + check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, TryIntoVersioned, +}; use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; @@ -79,8 +85,8 @@ use crate::{ /// Merging happens for some operations that will could change the element /// with just in time element flag updates. /// -/// If already merged is true and merged_op doesn't exist that means that there is no -/// point to try to merge again. +/// If already merged is true and merged_op doesn't exist that means that there +/// is no point to try to merge again. #[derive(Debug, PartialEq, Eq, Hash, Clone)] pub struct MergeQualifiedGroveOp { unmerged_op: GroveOp, @@ -139,8 +145,13 @@ pub enum GroveOp { /// Sum sum: Option, }, - /// Insert - Insert { + /// Inserts an element that is known to not yet exist + InsertOnly { + /// Element + element: Element, + }, + /// Inserts or Replaces an element + InsertOrReplace { /// Element element: Element, }, @@ -198,7 +209,8 @@ impl GroveOp { GroveOp::RefreshReference { .. } => 5, GroveOp::Replace { .. } => 6, GroveOp::Patch { .. } => 7, - GroveOp::Insert { .. } => 8, + GroveOp::InsertOrReplace { .. } => 8, + GroveOp::InsertOnly { .. } => 9, } } } @@ -401,7 +413,8 @@ impl fmt::Debug for GroveDbOp { self.key.visualize(key_drawer).unwrap(); let op_dbg = match &self.op { - GroveOp::Insert { element } => format!("Insert {:?}", element), + GroveOp::InsertOrReplace { element } => format!("Insert Or Replace {:?}", element), + GroveOp::InsertOnly { element } => format!("Insert {:?}", element), GroveOp::Replace { element } => format!("Replace {:?}", element), GroveOp::Patch { element, .. } => format!("Patch {:?}", element), GroveOp::RefreshReference { @@ -432,12 +445,22 @@ impl fmt::Debug for GroveDbOp { impl GroveDbOp { /// An insert op using a known owned path and known key - pub fn insert_op(path: Vec>, key: Vec, element: Element) -> Self { + pub fn insert_only_op(path: Vec>, key: Vec, element: Element) -> Self { + let path = KeyInfoPath::from_known_owned_path(path); + Self { + path, + key: KnownKey(key), + op: GroveOp::InsertOnly { element }, + } + } + + /// An insert op using a known owned path and known key + pub fn insert_or_replace_op(path: Vec>, key: Vec, element: Element) -> Self { let path = KeyInfoPath::from_known_owned_path(path); Self { path, key: KnownKey(key), - op: GroveOp::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -446,7 +469,7 @@ impl GroveDbOp { Self { path, key, - op: GroveOp::Insert { element }, + op: GroveOp::InsertOrReplace { element }, } } @@ -620,7 +643,9 @@ impl GroveDbOp { let inserts = ops .iter() .filter_map(|current_op| match current_op.op { - GroveOp::Insert { .. } | GroveOp::Replace { .. } => Some(current_op.clone()), + GroveOp::InsertOrReplace { .. } | GroveOp::Replace { .. } => { + Some(current_op.clone()) + } _ => None, }) .collect::>(); @@ -771,14 +796,24 @@ where /// deserializing the referenced element. /// * `Error::InvalidBatchOperation`: If the referenced element points to a /// tree being updated. - fn process_reference<'a>( + fn process_reference<'a, G, SR>( &'a mut self, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, recursions_allowed: u8, intermediate_reference_info: Option<&'a ReferencePathType>, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked let reference_merk_wrapped = self @@ -832,6 +867,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } else { @@ -839,64 +876,197 @@ where // but the hop count is greater than 1, we can't just take the value hash from // the referenced element as an element further down in the chain might still // change in the batch. - let referenced_element = cost_return_on_error!( - &mut cost, - merk.get( - key.as_ref(), - true, - Some(Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map_err(|e| Error::CorruptedData(e.to_string())) - ); + self.process_reference_with_hop_count_greater_than_one( + key, + reference_path, + qualified_path, + ops_by_qualified_paths, + recursions_allowed, + flags_update, + split_removal_bytes, + grove_version, + ) + } + } - let referenced_element = cost_return_on_error_no_add!( - &cost, - referenced_element.ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference to path:`{}` key:`{}` in batch is missing", - reference_string, - hex::encode(key) - )) - }) - ); + /// Retrieves and deserializes the referenced element from the Merk tree. + /// + /// This function is responsible for fetching the referenced element using the + /// provided key and reference path, deserializing it into an `Element`. It handles + /// potential errors that can occur during these operations, such as missing references + /// or corrupted data. + /// + /// # Arguments + /// + /// * `key` - The key associated with the referenced element within the Merk tree. + /// * `reference_path` - The path to the referenced element, used to locate it in the Merk tree. + /// * `grove_version` - The current version of the GroveDB being used for serialization + /// and deserialization operations. + /// + /// # Returns + /// + /// * `Ok(Element)` - Returns the deserialized `Element` if the retrieval and deserialization + /// are successful, wrapped in the associated cost. + /// * `Err(Error)` - Returns an error if any issue occurs during the retrieval or deserialization + /// of the referenced element. + /// + /// # Errors + /// + /// This function may return the following errors: + /// + /// * `Error::MissingReference` - If the referenced element is missing from the Merk tree. + /// * `Error::CorruptedData` - If the referenced element cannot be deserialized due to corrupted data. + fn get_and_deserialize_referenced_element<'a>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + grove_version: &GroveVersion, + ) -> CostResult { + let mut cost = OperationCost::default(); - let element = cost_return_on_error_no_add!( - &cost, - Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize element")) - }) - ); + let reference_merk_wrapped = self + .merks + .remove(reference_path) + .map(|x| Ok(x).wrap_with_cost(Default::default())) + .unwrap_or_else(|| (self.get_merk_fn)(reference_path, false)); - match element { - Element::Item(..) | Element::SumItem(..) => { - let serialized = - cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } - Element::Reference(path, ..) => { - let path = cost_return_on_error_no_add!( - &cost, - path_from_reference_qualified_path_type(path, qualified_path) - ); - self.follow_reference_get_value_hash( - path.as_slice(), - ops_by_qualified_paths, - recursions_allowed - 1, - grove_version, - ) - } - Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost), + let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); + + let referenced_element = cost_return_on_error!( + &mut cost, + merk.get( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value), + grove_version + ) + .map_err(|e| Error::CorruptedData(e.to_string())) + ); + + let referenced_element = cost_return_on_error_no_add!( + &cost, + referenced_element.ok_or({ + let reference_string = reference_path + .iter() + .map(hex::encode) + .collect::>() + .join("/"); + Error::MissingReference(format!( + "reference to path:`{}` key:`{}` in batch is missing", + reference_string, + hex::encode(key) + )) + }) + ); + + let element = cost_return_on_error_no_add!( + &cost, + Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { + Error::CorruptedData(String::from("unable to deserialize element")) + }) + ); + + Ok(element).wrap_with_cost(cost) + } + + /// Processes a reference with a hop count greater than one, handling the + /// retrieval and further processing of the referenced element. + /// + /// This function is used when the hop count is greater than 1, meaning that + /// the reference points to another element that may also be a reference. + /// It handles retrieving the referenced element, deserializing it, and + /// determining the appropriate action based on the type of the element. + /// + /// # Arguments + /// + /// * `key` - The key corresponding to the referenced element in the current + /// Merk tree. + /// * `reference_path` - The path to the referenced element within the + /// current batch of operations. + /// * `qualified_path` - The fully qualified path to the reference, already + /// validated as a valid path. + /// * `ops_by_qualified_paths` - A map of qualified paths to their + /// corresponding batch operations. Used to track and manage updates + /// within the batch. + /// * `recursions_allowed` - The maximum allowed hop count to reach the + /// final target element. Each recursive call reduces this by one. + /// * `flags_update` - A mutable closure that handles updating element flags + /// during the processing of the reference. + /// * `split_removal_bytes` - A mutable closure that handles splitting and + /// managing the removal of bytes during the processing of the reference. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. + /// + /// # Returns + /// + /// * `Ok(CryptoHash)` - Returns the crypto hash of the referenced element + /// if successful, wrapped in the associated cost. + /// * `Err(Error)` - Returns an error if there is an issue with the + /// operation, such as a missing reference, corrupted data, or an invalid + /// batch operation. + /// + /// # Errors + /// + /// This function will return `Err(Error)` if any issues are encountered + /// during the processing of the reference. Possible errors include: + /// + /// * `Error::MissingReference` - If a direct or indirect reference to the + /// target element is missing in the batch. + /// * `Error::CorruptedData` - If there is an issue while retrieving or + /// deserializing the referenced element. + /// * `Error::InvalidBatchOperation` - If the referenced element points to a + /// tree being updated, which is not allowed. + fn process_reference_with_hop_count_greater_than_one<'a, G, SR>( + &'a mut self, + key: &[u8], + reference_path: &[Vec], + qualified_path: &[Vec], + ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, + recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, + grove_version: &GroveVersion, + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { + let mut cost = OperationCost::default(); + + let element = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element(key, reference_path, grove_version) + ); + + match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path, qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), } } @@ -910,13 +1080,23 @@ where /// insert ref_3 and another operation to change something in the /// reference chain in the same batch. /// All these has to be taken into account. - fn follow_reference_get_value_hash<'a>( + fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, recursions_allowed: u8, + flags_update: &mut G, + split_removal_bytes: &mut SR, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult + where + G: FnMut(&StorageCost, Option, &mut ElementFlags) -> Result, + SR: FnMut( + &mut ElementFlags, + u32, + u32, + ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + { let mut cost = OperationCost::default(); if recursions_allowed == 0 { return Err(Error::ReferenceLimit).wrap_with_cost(cost); @@ -924,7 +1104,8 @@ where // If the element being referenced changes in the same batch // we need to set the value_hash based on the new change and not the old state. - // However the operation might either be merged or unmerged, if it is unmerged we need to merge it with the state first + // However the operation might either be merged or unmerged, if it is unmerged + // we need to merge it with the state first if let Some(qualified_op) = ops_by_qualified_paths.get(qualified_path) { // the path is being modified, inserted or deleted in the batch of operations match qualified_op.op() { @@ -932,15 +1113,61 @@ where Error::InvalidBatchOperation("references can not point to trees being updated"), ) .wrap_with_cost(cost), - GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { + GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { match element { Element::Item(..) | Element::SumItem(..) => { - let serialized = cost_return_on_error_no_add!( - &cost, - element.serialize(grove_version) - ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) + + + if element.get_flags().is_none() { + // There are no storage flags, we can just hash new element + let serialized = cost_return_on_error_no_add!( + &cost, + element.serialize(grove_version) + ); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + let mut new_element = element.clone(); + let mut new_flags = new_element.get_flags_mut().as_mut().unwrap(); + // There are storage flags + let storage_costs = StorageCost {added_bytes: 2, replaced_bytes: 4, removed_bytes: StorageRemovedBytes::NoStorageRemoval}; + // it can be unmerged, let's get the value on disk + let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked + let old_element = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element(key, reference_path, grove_version)); + + let maybe_old_flags = old_element.get_flags_owned(); + let changed = cost_return_on_error_no_add!( + &cost,(flags_update)(&storage_costs, maybe_old_flags, new_flags) + .map_err(|e| match e { + Error::JustInTimeElementFlagsClientError(_) => { + MerkError::ClientCorruptionError(e.to_string()).into() + } + _ => MerkError::ClientCorruptionError( + "non client error".to_string(), + ).into(), + })); + if changed { + // There are no storage flags, we can just hash new element + let serialized = cost_return_on_error_no_add!( + &cost, + new_element.serialize(grove_version) + ); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { + // There are no storage flags, we can just hash new element + let serialized = cost_return_on_error_no_add!( + &cost, + element.serialize(grove_version) + ); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + } } Element::Reference(path, ..) => { let path = cost_return_on_error_no_add!( @@ -954,6 +1181,8 @@ where path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, + flags_update, + split_removal_bytes, grove_version, ) } @@ -965,6 +1194,32 @@ where } } } + GroveOp::InsertOnly { element } => match element { + Element::Item(..) | Element::SumItem(..) => { + let serialized = + cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } + Element::Reference(path, ..) => { + let path = cost_return_on_error_no_add!( + &cost, + path_from_reference_qualified_path_type(path.clone(), qualified_path) + ); + self.follow_reference_get_value_hash( + path.as_slice(), + ops_by_qualified_paths, + recursions_allowed - 1, + flags_update, + split_removal_bytes, + grove_version, + ) + } + Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), + }, GroveOp::RefreshReference { reference_path_type, trust_refresh_reference, @@ -981,6 +1236,8 @@ where ops_by_qualified_paths, recursions_allowed, reference_info, + flags_update, + split_removal_bytes, grove_version, ) } @@ -997,6 +1254,8 @@ where ops_by_qualified_paths, recursions_allowed, None, + flags_update, + split_removal_bytes, grove_version, ) } @@ -1073,109 +1332,112 @@ where let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { match op.take_op() { - GroveOp::Insert { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { - match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - let path_reference = cost_return_on_error!( - &mut cost, - path_from_reference_path_type( - path_reference.clone(), - path, - Some(key_info.as_slice()) - ) + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => match &element { + Element::Reference(path_reference, element_max_reference_hop, _) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) .wrap_with_cost(OperationCost::default()) - ); - if path_reference.is_empty() { - return Err(Error::InvalidBatchOperation( - "attempting to insert an empty reference", - )) - .wrap_with_cost(cost); - } + ); + let path_reference = cost_return_on_error!( + &mut cost, + path_from_reference_path_type( + path_reference.clone(), + path, + Some(key_info.as_slice()) + ) + .wrap_with_cost(OperationCost::default()) + ); + if path_reference.is_empty() { + return Err(Error::InvalidBatchOperation( + "attempting to insert an empty reference", + )) + .wrap_with_cost(cost); + } - let referenced_element_value_hash = cost_return_on_error!( - &mut cost, - self.follow_reference_get_value_hash( - path_reference.as_slice(), - ops_by_qualified_paths, - element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), - grove_version, - ) - ); + let referenced_element_value_hash = cost_return_on_error!( + &mut cost, + self.follow_reference_get_value_hash( + path_reference.as_slice(), + ops_by_qualified_paths, + element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, + grove_version, + ) + ); - cost_return_on_error!( + cost_return_on_error!( + &mut cost, + element.insert_reference_into_batch_operations( + key_info.get_key_clone(), + referenced_element_value_hash, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Tree(..) | Element::SumTree(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + cost_return_on_error!( + &mut cost, + element.insert_subtree_into_batch_operations( + key_info.get_key_clone(), + NULL_HASH, + false, + &mut batch_operations, + merk_feature_type, + grove_version, + ) + ); + } + Element::Item(..) | Element::SumItem(..) => { + let merk_feature_type = cost_return_on_error!( + &mut cost, + element + .get_feature_type(is_sum_tree) + .wrap_with_cost(OperationCost::default()) + ); + if batch_apply_options.validate_insertion_does_not_override { + let inserted = cost_return_on_error!( &mut cost, - element.insert_reference_into_batch_operations( - key_info.get_key_clone(), - referenced_element_value_hash, + element.insert_if_not_exists_into_batch_operations( + &mut merk, + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); - } - Element::Tree(..) | Element::SumTree(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); + if !inserted { + return Err(Error::InvalidBatchOperation( + "attempting to overwrite a tree", + )) + .wrap_with_cost(cost); + } + } else { cost_return_on_error!( &mut cost, - element.insert_subtree_into_batch_operations( - key_info.get_key_clone(), - NULL_HASH, - false, + element.insert_into_batch_operations( + key_info.get_key(), &mut batch_operations, merk_feature_type, grove_version, ) ); } - Element::Item(..) | Element::SumItem(..) => { - let merk_feature_type = cost_return_on_error!( - &mut cost, - element - .get_feature_type(is_sum_tree) - .wrap_with_cost(OperationCost::default()) - ); - if batch_apply_options.validate_insertion_does_not_override { - let inserted = cost_return_on_error!( - &mut cost, - element.insert_if_not_exists_into_batch_operations( - &mut merk, - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - if !inserted { - return Err(Error::InvalidBatchOperation( - "attempting to overwrite a tree", - )) - .wrap_with_cost(cost); - } - } else { - cost_return_on_error!( - &mut cost, - element.insert_into_batch_operations( - key_info.get_key(), - &mut batch_operations, - merk_feature_type, - grove_version, - ) - ); - } - } } - } + }, GroveOp::RefreshReference { reference_path_type, max_reference_hop, @@ -1247,6 +1509,8 @@ where path_reference.as_slice(), ops_by_qualified_paths, max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), + flags_update, + split_removal_bytes, grove_version ) ); @@ -1353,11 +1617,17 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { + println!( + "getting specialized cost key: {}, value {}", + key.len(), + value.len() + ); Element::specialized_costs_for_key_value(key, value, is_sum_tree, grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &mut |storage_costs, old_value, new_value| { + println!("updating to new value"); // todo: change the flags without full deserialization let old_element = Element::deserialize(old_value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; @@ -1549,11 +1819,14 @@ impl GroveDb { { match ops_on_path.entry(key.clone()) { Entry::Vacant(vacant_entry) => { - vacant_entry.insert(GroveOp::ReplaceTreeRootKey { - hash: root_hash, - root_key: calculated_root_key, - sum: sum_value, - }.into()); + vacant_entry.insert( + GroveOp::ReplaceTreeRootKey { + hash: root_hash, + root_key: calculated_root_key, + sum: sum_value, + } + .into(), + ); } Entry::Occupied(occupied_entry) => { let mutable_occupied_entry = occupied_entry.into_mut(); @@ -1573,7 +1846,8 @@ impl GroveDb { )) .wrap_with_cost(cost); } - GroveOp::Insert { element } + GroveOp::InsertOrReplace { element } + | GroveOp::InsertOnly { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { if let Element::Tree(_, flags) = element { @@ -1583,7 +1857,8 @@ impl GroveDb { root_key: calculated_root_key, flags: flags.clone(), sum: None, - }.into(); + } + .into(); } else if let Element::SumTree(.., flags) = element { @@ -1593,7 +1868,8 @@ impl GroveDb { root_key: calculated_root_key, flags: flags.clone(), sum: sum_value, - }.into(); + } + .into(); } else { return Err(Error::InvalidBatchOperation( "insertion of element under a non tree", @@ -1608,7 +1884,9 @@ impl GroveDb { )) .wrap_with_cost(cost); } - GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { + GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -1621,29 +1899,35 @@ impl GroveDb { } } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = + BTreeMap::new(); ops_on_path.insert( key.clone(), GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }.into(), + } + .into(), ); ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); + let mut ops_on_path: BTreeMap = + BTreeMap::new(); ops_on_path.insert( key.clone(), GroveOp::ReplaceTreeRootKey { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - }.into(), + } + .into(), ); - let mut ops_on_level: BTreeMap> = - BTreeMap::new(); + let mut ops_on_level: BTreeMap< + KeyInfoPath, + BTreeMap, + > = BTreeMap::new(); ops_on_level.insert(KeyInfoPath(parent_path.to_vec()), ops_on_path); ops_by_level_paths.insert(current_level - 1, ops_on_level); } @@ -1771,7 +2055,7 @@ impl GroveDb { let mut cost = OperationCost::default(); for op in ops.into_iter() { match op.op { - GroveOp::Insert { element } | GroveOp::Replace { element } => { + GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } => { // TODO: paths in batches is something to think about let path_slices: Vec<&[u8]> = op.path.iterator().map(|p| p.as_slice()).collect(); @@ -2109,9 +2393,18 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self.visualize_verify_grovedb(Some(tx), true, &Default::default()).unwrap(); + let issues = self + .visualize_verify_grovedb(Some(tx), true, &Default::default()) + .unwrap(); if issues.len() > 0 { - println!("tx_issues: {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + println!( + "tx_issues: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); } } else { cost_return_on_error!( @@ -2141,9 +2434,18 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self.visualize_verify_grovedb(None, true, &Default::default()).unwrap(); + let issues = self + .visualize_verify_grovedb(None, true, &Default::default()) + .unwrap(); if issues.len() > 0 { - println!("non_tx_issues: {}", issues.iter().map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)).collect::>().join(" | ")); + println!( + "non_tx_issues: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); } } Ok(()).wrap_with_cost(cost) @@ -2505,28 +2807,28 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2586,8 +2888,16 @@ mod tests { // No two operations should be the same let ops = vec![ - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), + GroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2598,12 +2908,16 @@ mod tests { // Can't perform 2 or more operations on the same node let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::insert_op(vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( + vec![b"a".to_vec()], + b"b".to_vec(), + Element::empty_tree(), + ), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2614,7 +2928,7 @@ mod tests { // Can't insert under a deleted path let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::new_item(vec![1]), @@ -2630,12 +2944,12 @@ mod tests { // Should allow invalid operations pass when disable option is set to true let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2680,28 +2994,28 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2779,28 +3093,28 @@ mod tests { let tx = db.start_transaction(); // let's start by inserting a tree structure let ops = vec![ - GroveDbOp::insert_op(vec![], b"1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"1".to_vec()], b"my_contract".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"0".to_vec(), Element::new_item(b"this is the contract".to_vec()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec(), b"1".to_vec()], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2810,7 +3124,7 @@ mod tests { b"0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2839,7 +3153,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2853,7 +3167,7 @@ mod tests { some_element_flags.clone(), ), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2864,7 +3178,7 @@ mod tests { b"my apples are safe".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2876,7 +3190,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2913,7 +3227,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2924,7 +3238,7 @@ mod tests { b"wisdom".to_vec(), Element::new_item_with_flags(b"Wisdom Ogwu".to_vec(), some_element_flags.clone()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2935,7 +3249,7 @@ mod tests { b"canteloupe!".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2947,7 +3261,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -2991,57 +3305,57 @@ mod tests { fn grove_db_ops_for_contract_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![], b"contract".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [0u8].to_vec(), Element::new_item(b"serialized_contract".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [1u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"domain".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"normalized_domain_label".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"unique_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"alias_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"preorder".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], b"salted_domain".to_vec(), Element::empty_tree(), @@ -3053,7 +3367,7 @@ mod tests { fn grove_db_ops_for_contract_document_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3064,7 +3378,7 @@ mod tests { Element::new_item(b"serialized_domain".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3075,7 +3389,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3087,7 +3401,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3100,7 +3414,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_op( + grove_db_ops.push(GroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3223,13 +3537,13 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3251,23 +3565,23 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element.clone(), ), - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3315,12 +3629,12 @@ mod tests { .expect("cannot insert a subtree"); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), @@ -3339,18 +3653,18 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3398,7 +3712,7 @@ mod tests { .expect("cannot insert value"); // Insertion into scalar is invalid - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3409,7 +3723,7 @@ mod tests { .is_err()); // Insertion into a tree is correct - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"valid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3454,8 +3768,8 @@ mod tests { // TEST_LEAF can not be overwritten let ops = vec![ - GroveDbOp::insert_op(vec![], TEST_LEAF.to_vec(), element2), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], TEST_LEAF.to_vec(), element2), + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key_subtree".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3482,7 +3796,7 @@ mod tests { // TEST_LEAF will be deleted so you can not insert underneath it let ops = vec![ GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3498,7 +3812,7 @@ mod tests { // validate_tree_insertion_does_not_override set to true let ops = vec![ GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3528,12 +3842,12 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![], TEST_LEAF.to_vec(), Element::new_item(b"ayy".to_vec()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3584,7 +3898,7 @@ mod tests { ) .unwrap() .expect("cannot insert an item"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_item(b"ayy2".to_vec()), @@ -3648,22 +3962,26 @@ mod tests { let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"key".to_vec(), element2.clone()), + GroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"key".to_vec(), + element2.clone(), + ), GroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) @@ -3750,7 +4068,7 @@ mod tests { } let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![GroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element.clone(), @@ -3759,7 +4077,11 @@ mod tests { .unwrap() .expect("cannot apply batch"); - let batch = vec![GroveDbOp::insert_op(acc_path, b"key".to_vec(), element)]; + let batch = vec![GroveDbOp::insert_or_replace_op( + acc_path, + b"key".to_vec(), + element, + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("cannot apply same batch twice"); @@ -3788,7 +4110,7 @@ mod tests { let root_hash = db.root_hash(None, grove_version).unwrap().unwrap(); let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![GroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element, @@ -3808,7 +4130,7 @@ mod tests { let grove_version = GroveVersion::latest(); // insert reference that points to non-existent item let db = make_test_grovedb(grove_version); - let batch = vec![GroveDbOp::insert_op( + let batch = vec![GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3825,7 +4147,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -3833,7 +4155,7 @@ mod tests { b"invalid_path".to_vec(), ])), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem.clone(), @@ -3866,7 +4188,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key2".to_vec(), Element::new_reference_with_hops( @@ -3877,7 +4199,7 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference_with_hops( @@ -3888,7 +4210,11 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_op(vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem), + GroveDbOp::insert_or_replace_op( + vec![TEST_LEAF.to_vec()], + b"invalid_path".to_vec(), + elem, + ), ]; assert!(matches!( db.apply_batch(batch, None, None, grove_version).unwrap(), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index ad171d6d..fe358081 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -46,8 +46,8 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), ]; let cost = db.apply_batch(ops, None, Some(&tx), grove_version).cost; assert_eq!( @@ -97,13 +97,13 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2).add(non_batch_cost_3); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![], b"key3".to_vec(), Element::new_reference(SiblingReference(b"key2".to_vec())), @@ -173,18 +173,18 @@ mod tests { .add(non_batch_cost_4); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -212,8 +212,8 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op(vec![], b"key2".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); @@ -268,8 +268,8 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + GroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index c025fb27..739de7ce 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -42,7 +42,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -57,7 +57,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -122,7 +122,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -200,7 +200,7 @@ mod tests { assert_eq!(cost.storage_cost.added_bytes, 143); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -293,7 +293,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -375,7 +375,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -453,7 +453,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 59].to_vec()), @@ -516,7 +516,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 60].to_vec()), @@ -601,7 +601,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![1])), @@ -667,7 +667,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), @@ -756,7 +756,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), @@ -821,7 +821,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), @@ -911,7 +911,7 @@ mod tests { .expect("expected to insert item"); // We are adding 1 byte to the flags - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_tree_with_flags(None, Some(vec![0, 1, 1])), diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index 0ba3da44..65786eb9 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -42,7 +42,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item(150), @@ -57,7 +57,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -134,7 +134,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -220,7 +220,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -306,7 +306,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -396,7 +396,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -487,7 +487,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 42].to_vec())), @@ -562,7 +562,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 43].to_vec())), @@ -648,7 +648,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, None), @@ -714,7 +714,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, Some(vec![1])), @@ -780,7 +780,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, None), @@ -846,7 +846,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, Some(vec![1])), diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 2d2db076..598241f6 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -251,6 +251,7 @@ impl Element { ); // todo: we actually don't need to deserialize the whole element let element = Element::deserialize(value, grove_version)?; + println!("element is {}", element); let cost = match element { Element::Tree(_, flags) => { let flags_len = flags.map_or(0, |flags| { diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 059ff9b9..8c010e28 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1065,7 +1065,7 @@ impl GroveDb { ))?; let referenced_value_hash = { - let mut full_path = path_from_reference_path_type( + let full_path = path_from_reference_path_type( reference_path.clone(), &path.to_vec(), Some(&key), @@ -1205,7 +1205,7 @@ impl GroveDb { ))?; let referenced_value_hash = { - let mut full_path = path_from_reference_path_type( + let full_path = path_from_reference_path_type( reference_path.clone(), &path.to_vec(), Some(&key), diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 87f0f97b..af7629ae 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -7,7 +7,6 @@ use std::{collections::HashMap, option::Option::None}; use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::value_hash; #[cfg(feature = "full")] use grovedb_merk::{tree::NULL_HASH, Merk, MerkOptions}; use grovedb_path::SubtreePath; diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index f6b7b6d4..dfb4623c 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -4071,22 +4071,22 @@ mod tests { let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"value".to_vec(), Element::new_item(b"hello".to_vec()), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refc".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"value".to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refb".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refc".to_vec())), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refa".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refb".to_vec())), diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 48c358c6..5d80b994 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -1595,33 +1595,37 @@ mod tests { 77, 252, 86, 99, 107, 197, 226, 188, 54, 239, 64, 17, 37, ]; - let batch = vec![GroveDbOp::insert_op(vec![], vec![1], Element::empty_tree())]; + let batch = vec![GroveDbOp::insert_or_replace_op( + vec![], + vec![1], + Element::empty_tree(), + )]; db.apply_batch(batch, None, None, grove_version) .unwrap() .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![vec![1]], tree_name_slice.to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], vec![1], Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec(), vec![1]], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1631,7 +1635,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1647,7 +1651,7 @@ mod tests { .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1658,7 +1662,7 @@ mod tests { b"person_id_1".to_vec(), Element::new_item(vec![50]), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1669,7 +1673,7 @@ mod tests { b"cammi".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1681,7 +1685,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 92df7d73..78f73ed4 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -779,17 +779,17 @@ fn test_sum_tree_with_batches() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"a".to_vec(), Element::new_item(vec![214]), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"b".to_vec(), Element::new_sum_item(10), @@ -835,7 +835,7 @@ fn test_sum_tree_with_batches() { )); // Create new batch to use existing tree - let ops = vec![GroveDbOp::insert_op( + let ops = vec![GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"c".to_vec(), Element::new_sum_item(10), @@ -871,42 +871,42 @@ fn test_sum_tree_with_batches() { // Add a new sum tree with its own sum items, should affect sum of original // tree let ops = vec![ - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"d".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"first".to_vec(), Element::new_sum_item(4), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"e".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"first".to_vec(), Element::new_sum_item(12), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"third".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), @@ -916,7 +916,7 @@ fn test_sum_tree_with_batches() { b"a".to_vec(), Element::new_sum_item(5), ), - GroveDbOp::insert_op( + GroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), diff --git a/merk/benches/merk.rs b/merk/benches/merk.rs index e2d55219..46f83400 100644 --- a/merk/benches/merk.rs +++ b/merk/benches/merk.rs @@ -39,9 +39,11 @@ use grovedb_merk::{ use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; use rand::prelude::*; +use grovedb_version::version::GroveVersion; /// 1 million gets in 2k batches pub fn get(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; let num_batches = initial_size / batch_size; @@ -95,6 +97,7 @@ pub fn get(c: &mut Criterion) { /// 1 million sequential inserts in 2k batches pub fn insert_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -136,6 +139,7 @@ pub fn insert_1m_2k_seq(c: &mut Criterion) { /// 1 million random inserts in 2k batches pub fn insert_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -177,6 +181,7 @@ pub fn insert_1m_2k_rand(c: &mut Criterion) { /// 1 million sequential updates in 2k batches pub fn update_1m_2k_seq(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -237,6 +242,7 @@ pub fn update_1m_2k_seq(c: &mut Criterion) { /// 1 million random updates in 2k batches pub fn update_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -297,6 +303,7 @@ pub fn update_1m_2k_rand(c: &mut Criterion) { /// 1 million random deletes in 2k batches pub fn delete_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -382,6 +389,7 @@ pub fn delete_1m_2k_rand(c: &mut Criterion) { /// 1 million random proofs in 2k batches pub fn prove_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -434,6 +442,7 @@ pub fn prove_1m_2k_rand(c: &mut Criterion) { /// Build 1 million trunk chunks in 2k batches, random pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -477,6 +486,7 @@ pub fn build_trunk_chunk_1m_2k_rand(c: &mut Criterion) { /// Chunk producer random 1 million pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -518,6 +528,7 @@ pub fn chunkproducer_rand_1m_1_rand(c: &mut Criterion) { /// Chunk iter 1 million pub fn chunk_iter_1m_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let initial_size = 1_000_000; let batch_size = 2_000; @@ -565,6 +576,7 @@ pub fn chunk_iter_1m_1(c: &mut Criterion) { /// Restore merk of size 500 pub fn restore_500_1(c: &mut Criterion) { + let grove_version = GroveVersion::latest(); let merk_size = 500; let mut merk = TempMerk::new(grove_version); From aec9eeeea5f4fc6b1767f3c0b1c3f0367f84b363 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 04:33:20 +0700 Subject: [PATCH 05/19] more work --- grovedb/src/batch/batch_structure.rs | 69 +-- .../estimated_costs/average_case_costs.rs | 26 +- .../batch/estimated_costs/worst_case_costs.rs | 24 +- grovedb/src/batch/just_in_time_cost_tests.rs | 20 +- grovedb/src/batch/mod.rs | 502 +++++++-------- grovedb/src/batch/multi_insert_cost_tests.rs | 28 +- .../src/batch/single_deletion_cost_tests.rs | 20 +- grovedb/src/batch/single_insert_cost_tests.rs | 581 +++++++++++++++++- .../single_sum_item_deletion_cost_tests.rs | 8 +- .../single_sum_item_insert_cost_tests.rs | 26 +- grovedb/src/element/helpers.rs | 1 - grovedb/src/operations/delete/average_case.rs | 8 +- .../src/operations/delete/delete_up_tree.rs | 12 +- grovedb/src/operations/delete/mod.rs | 10 +- grovedb/src/operations/delete/worst_case.rs | 8 +- grovedb/src/tests/mod.rs | 10 +- grovedb/src/tests/query_tests.rs | 24 +- grovedb/src/tests/sum_tree_tests.rs | 28 +- merk/src/tree/mod.rs | 26 +- 19 files changed, 962 insertions(+), 469 deletions(-) diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index b07c6b89..4fc3a381 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -14,15 +14,14 @@ use grovedb_visualize::{DebugByteVectors, DebugBytes}; #[cfg(feature = "full")] use nohash_hasher::IntMap; -use crate::batch::MergeQualifiedGroveOp; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, GroveOp, KeyInfoPath, TreeCache}, + batch::{key_info::KeyInfo, QualifiedGroveDbOp, GroveOp, KeyInfoPath, TreeCache}, Element, ElementFlags, Error, }; #[cfg(feature = "full")] -pub type OpsByPath = BTreeMap>; +pub type OpsByPath = BTreeMap>; /// Level, path, key, op #[cfg(feature = "full")] pub type OpsByLevelPath = IntMap; @@ -33,7 +32,7 @@ pub(super) struct BatchStructure { /// Operations by level path pub(super) ops_by_level_paths: OpsByLevelPath, /// This is for references - pub(super) ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp>, + pub(super) ops_by_qualified_paths: BTreeMap>, GroveOp>, /// Merk trees /// Very important: the type of run mode we are in is contained in this /// cache @@ -85,7 +84,7 @@ where { /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn from_ops( - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, merk_tree_cache: C, @@ -102,7 +101,7 @@ where /// Create batch structure from a list of ops. Returns CostResult. pub(super) fn continue_from_ops( previous_ops: Option, - ops: Vec, + ops: Vec, update_element_flags_function: F, split_remove_bytes_function: SR, mut merk_tree_cache: C, @@ -113,43 +112,15 @@ where let mut current_last_level: u32 = 0; // qualified paths meaning path + key - let mut ops_by_qualified_paths: BTreeMap>, MergeQualifiedGroveOp> = - BTreeMap::new(); + let mut ops_by_qualified_paths: BTreeMap>, GroveOp> = BTreeMap::new(); for op in ops.into_iter() { let mut path = op.path.clone(); path.push(op.key.clone()); + ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone()); let op_cost = OperationCost::default(); let op_result = match &op.op { - GroveOp::InsertOrReplace { element } - | GroveOp::Replace { element } - | GroveOp::Patch { element, .. } => { - match element { - Element::Item(..) | Element::SumItem(..) => { - // let path = path.to_path_consume(); - // let previous_element = Element::get_from_storage() - // todo: path merges here - ops_by_qualified_paths - .insert(path.to_path_consume(), op.op.clone().into()); - } - Element::Reference(..) => { - ops_by_qualified_paths - .insert(path.to_path_consume(), op.op.clone().into()); - } - Element::Tree(..) => { - cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); - ops_by_qualified_paths - .insert(path.to_path_consume(), op.op.clone().into()); - } - Element::SumTree(..) => { - cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, true)); - ops_by_qualified_paths - .insert(path.to_path_consume(), op.op.clone().into()); - } - } - Ok(()) - } - GroveOp::InsertOnly { element } => { + GroveOp::InsertOnly { element } | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -157,11 +128,7 @@ where } Ok(()) } - GroveOp::RefreshReference { .. } - | GroveOp::Delete - | GroveOp::DeleteTree - | GroveOp::DeleteSumTree => { - ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone().into()); + GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { Ok(()) } GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { @@ -177,19 +144,17 @@ where let level = op.path.len(); if let Some(ops_on_level) = ops_by_level_paths.get_mut(&level) { if let Some(ops_on_path) = ops_on_level.get_mut(&op.path) { - ops_on_path.insert(op.key, op.op.into()); + ops_on_path.insert(op.key, op.op); } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); - ops_on_path.insert(op.key, op.op.into()); + let mut ops_on_path: BTreeMap = BTreeMap::new(); + ops_on_path.insert(op.key, op.op); ops_on_level.insert(op.path.clone(), ops_on_path); } } else { - let mut ops_on_path: BTreeMap = BTreeMap::new(); - ops_on_path.insert(op.key, op.op.into()); - let mut ops_on_level: BTreeMap< - KeyInfoPath, - BTreeMap, - > = BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); + ops_on_path.insert(op.key, op.op); + let mut ops_on_level: BTreeMap> = + BTreeMap::new(); ops_on_level.insert(op.path, ops_on_path); ops_by_level_paths.insert(level, ops_on_level); if current_last_level < level { @@ -206,6 +171,6 @@ where split_removal_bytes: split_remove_bytes_function, last_level: current_last_level, }) - .wrap_with_cost(cost) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 457e6976..b9a5b4be 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -22,11 +22,11 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::{batch::MergeQualifiedGroveOp, Element}; +use crate::{batch::GroveDBOp, Element}; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, GroveOp, KeyInfoPath, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, QualifiedGroveDbOp, GroveOp, KeyInfoPath, TreeCache, }, Error, GroveDb, @@ -165,7 +165,7 @@ impl fmt::Debug for AverageCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for AverageCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut average_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -184,8 +184,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, QualifiedGroveDbOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -310,7 +310,7 @@ mod tests { use crate::{ batch::{ estimated_costs::EstimatedCostsType::AverageCaseCostsType, key_info::KeyInfo, - GroveDbOp, KeyInfoPath, + QualifiedGroveDbOp, KeyInfoPath, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, @@ -322,7 +322,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -391,7 +391,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -458,7 +458,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -531,7 +531,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -617,7 +617,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -696,7 +696,7 @@ mod tests { #[test] fn test_batch_root_one_sum_item_replace_op_average_case_costs() { let grove_version = GroveVersion::latest(); - let ops = vec![GroveDbOp::replace_op( + let ops = vec![QualifiedGroveDbOp::replace_op( vec![vec![7]], hex::decode("46447a3b4c8939fd4cf8b610ba7da3d3f6b52b39ab2549bf91503b9b07814055") .unwrap(), @@ -775,7 +775,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 72e13601..a7e2edc5 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -21,11 +21,11 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::{batch::MergeQualifiedGroveOp, Element}; +use crate::{batch::QualifiedGroveDbOp, Element}; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveDbOp, GroveOp, KeyInfoPath, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, QualifiedGroveDbOp, GroveOp, KeyInfoPath, TreeCache, }, Error, GroveDb, @@ -161,7 +161,7 @@ impl fmt::Debug for WorstCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for WorstCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &GroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { let mut worst_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -180,8 +180,8 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, QualifiedGroveDbOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -275,7 +275,7 @@ mod tests { use crate::{ batch::{ - estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, GroveDbOp, + estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, QualifiedGroveDbOp, KeyInfoPath, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, @@ -288,7 +288,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -343,7 +343,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree_with_flags(Some(b"cat".to_vec())), @@ -398,7 +398,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -464,7 +464,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -530,7 +530,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -594,7 +594,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 820992f0..9c990d1b 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -17,7 +17,7 @@ mod tests { use integer_encoding::VarInt; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::{ ReferencePathType, ReferencePathType::UpstreamFromElementHeightReference, }, @@ -52,17 +52,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -186,17 +186,17 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"documents".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -258,7 +258,7 @@ mod tests { .get(&0) .expect("expected to have root path"); assert_eq!(ops_by_root_path.len(), 1); - let new_ops = vec![GroveDbOp::insert_or_replace_op( + let new_ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"balances".to_vec()], b"person".to_vec(), Element::new_sum_item_with_flags(1000, Some([0, 1].to_vec())), @@ -363,12 +363,12 @@ mod tests { // We are adding 2 bytes let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), ), - GroveDbOp::insert_only_op( + QualifiedGroveDbOp::insert_only_op( vec![b"refs".to_vec()], b"ref_key".to_vec(), Element::new_reference_with_hops( diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index cae65be5..b1dc0fa7 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -65,6 +65,8 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; +use grovedb_merk::tree::kv::KV; +use grovedb_merk::tree::TreeNode; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -80,58 +82,7 @@ use crate::{ }, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; - -/// An operation with extra information about merge status -/// Merging happens for some operations that will could change the element -/// with just in time element flag updates. -/// -/// If already merged is true and merged_op doesn't exist that means that there -/// is no point to try to merge again. -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub struct MergeQualifiedGroveOp { - unmerged_op: GroveOp, - merged_op: Option, - already_merged: bool, -} - -impl MergeQualifiedGroveOp { - #[inline] - pub fn op(&self) -> &GroveOp { - if let Some(op) = &self.merged_op { - op - } else { - &self.unmerged_op - } - } - - #[inline] - pub fn op_mut(&mut self) -> &mut GroveOp { - if let Some(op) = &mut self.merged_op { - op - } else { - &mut self.unmerged_op - } - } - - #[inline] - pub fn take_op(self) -> GroveOp { - if let Some(op) = self.merged_op { - op - } else { - self.unmerged_op - } - } -} - -impl From for MergeQualifiedGroveOp { - fn from(value: GroveOp) -> Self { - MergeQualifiedGroveOp { - unmerged_op: value, - merged_op: None, - already_merged: false, - } - } -} +use crate::operations::proof::util::hex_to_ascii; /// Operations #[derive(Debug, PartialEq, Eq, Hash, Clone)] @@ -394,7 +345,7 @@ impl KeyInfoPath { /// Batch operation #[derive(Clone, PartialEq, Eq)] -pub struct GroveDbOp { +pub struct QualifiedGroveDbOp { /// Path to a subtree - subject to an operation pub path: KeyInfoPath, /// Key of an element in the subtree @@ -403,7 +354,7 @@ pub struct GroveDbOp { pub op: GroveOp, } -impl fmt::Debug for GroveDbOp { +impl fmt::Debug for QualifiedGroveDbOp { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut path_out = Vec::new(); let path_drawer = Drawer::new(&mut path_out); @@ -443,7 +394,7 @@ impl fmt::Debug for GroveDbOp { } } -impl GroveDbOp { +impl QualifiedGroveDbOp { /// An insert op using a known owned path and known key pub fn insert_only_op(path: Vec>, key: Vec, element: Element) -> Self { let path = KeyInfoPath::from_known_owned_path(path); @@ -596,7 +547,7 @@ impl GroveDbOp { } /// Verify consistency of operations - pub fn verify_consistency_of_operations(ops: &[GroveDbOp]) -> GroveDbOpConsistencyResults { + pub fn verify_consistency_of_operations(ops: &[QualifiedGroveDbOp]) -> GroveDbOpConsistencyResults { let ops_len = ops.len(); // operations should not have any duplicates let mut repeated_ops = vec![]; @@ -648,7 +599,7 @@ impl GroveDbOp { } _ => None, }) - .collect::>(); + .collect::>(); let deletes = ops .iter() @@ -659,7 +610,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); let mut insert_ops_below_deleted_ops = vec![]; @@ -681,7 +632,7 @@ impl GroveDbOp { None } }) - .collect::>(); + .collect::>(); if !inserts_with_deleted_ops_above.is_empty() { insert_ops_below_deleted_ops .push((deleted_op.clone(), inserts_with_deleted_ops_above)); @@ -699,9 +650,9 @@ impl GroveDbOp { /// Results of a consistency check on an operation batch #[derive(Debug)] pub struct GroveDbOpConsistencyResults { - repeated_ops: Vec<(GroveDbOp, u16)>, // the u16 is count + repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, // the u16 is count same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, - insert_ops_below_deleted_ops: Vec<(GroveDbOp, Vec)>, /* the deleted op first, + insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, /* the deleted op first, * then inserts under */ } @@ -727,7 +678,7 @@ impl fmt::Debug for TreeCacheMerkByPath { } trait TreeCache { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error>; fn get_batch_run_mode(&self) -> BatchRunMode; @@ -735,8 +686,8 @@ trait TreeCache { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -799,7 +750,7 @@ where fn process_reference<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, intermediate_reference_info: Option<&'a ReferencePathType>, flags_update: &mut G, @@ -891,37 +842,42 @@ where /// Retrieves and deserializes the referenced element from the Merk tree. /// - /// This function is responsible for fetching the referenced element using the - /// provided key and reference path, deserializing it into an `Element`. It handles - /// potential errors that can occur during these operations, such as missing references - /// or corrupted data. + /// This function is responsible for fetching the referenced element using + /// the provided key and reference path, deserializing it into an + /// `Element`. It handles potential errors that can occur during these + /// operations, such as missing references or corrupted data. /// /// # Arguments /// - /// * `key` - The key associated with the referenced element within the Merk tree. - /// * `reference_path` - The path to the referenced element, used to locate it in the Merk tree. - /// * `grove_version` - The current version of the GroveDB being used for serialization - /// and deserialization operations. + /// * `key` - The key associated with the referenced element within the Merk + /// tree. + /// * `reference_path` - The path to the referenced element, used to locate + /// it in the Merk tree. + /// * `grove_version` - The current version of the GroveDB being used for + /// serialization and deserialization operations. /// /// # Returns /// - /// * `Ok(Element)` - Returns the deserialized `Element` if the retrieval and deserialization - /// are successful, wrapped in the associated cost. - /// * `Err(Error)` - Returns an error if any issue occurs during the retrieval or deserialization - /// of the referenced element. + /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` and the serialized counterpart + /// if the retrieval and deserialization are successful, wrapped in the associated cost. + /// Also returns if the merk of the element is a sum tree as a bool. + /// * `Err(Error)` - Returns an error if any issue occurs during the + /// retrieval or deserialization of the referenced element. /// /// # Errors /// /// This function may return the following errors: /// - /// * `Error::MissingReference` - If the referenced element is missing from the Merk tree. - /// * `Error::CorruptedData` - If the referenced element cannot be deserialized due to corrupted data. + /// * `Error::MissingReference` - If the referenced element is missing from + /// the Merk tree. + /// * `Error::CorruptedData` - If the referenced element cannot be + /// deserialized due to corrupted data. fn get_and_deserialize_referenced_element<'a>( &'a mut self, key: &[u8], reference_path: &[Vec], grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult, bool)>, Error> { let mut cost = OperationCost::default(); let reference_merk_wrapped = self @@ -933,40 +889,28 @@ where let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); let referenced_element = cost_return_on_error!( - &mut cost, - merk.get( - key.as_ref(), - true, - Some(Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map_err(|e| Error::CorruptedData(e.to_string())) - ); - - let referenced_element = cost_return_on_error_no_add!( - &cost, - referenced_element.ok_or({ - let reference_string = reference_path - .iter() - .map(hex::encode) - .collect::>() - .join("/"); - Error::MissingReference(format!( - "reference to path:`{}` key:`{}` in batch is missing", - reference_string, - hex::encode(key) - )) - }) - ); + &mut cost, + merk.get( + key.as_ref(), + true, + Some(Element::value_defined_cost_for_serialized_value), + grove_version + ) + .map_err(|e| Error::CorruptedData(e.to_string())) + ); - let element = cost_return_on_error_no_add!( - &cost, - Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize element")) - }) - ); + if let Some(referenced_element) = referenced_element { + let element = cost_return_on_error_no_add!( + &cost, + Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { + Error::CorruptedData(String::from("unable to deserialize element")) + }) + ); - Ok(element).wrap_with_cost(cost) + Ok(Some((element, referenced_element, merk.is_sum_tree))).wrap_with_cost(cost) + } else { + Ok(None).wrap_with_cost(cost) + } } /// Processes a reference with a hop count greater than one, handling the @@ -1021,7 +965,7 @@ where key: &[u8], reference_path: &[Vec], qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1037,10 +981,21 @@ where { let mut cost = OperationCost::default(); - let element = cost_return_on_error!( - &mut cost, - self.get_and_deserialize_referenced_element(key, reference_path, grove_version) - ); + let Some((element, _, _)) = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element(key, reference_path, grove_version) + ) else { + let reference_string = reference_path + .iter() + .map(hex::encode) + .collect::>() + .join("/"); + return Err(Error::MissingReference(format!( + "reference to path:`{}` key:`{}` in batch is missing", + reference_string, + hex::encode(key) + ))).wrap_with_cost(cost); + }; match element { Element::Item(..) | Element::SumItem(..) => { @@ -1083,7 +1038,7 @@ where fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, qualified_path: &[Vec], - ops_by_qualified_paths: &'a BTreeMap>, MergeQualifiedGroveOp>, + ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1106,9 +1061,9 @@ where // However the operation might either be merged or unmerged, if it is unmerged // we need to merge it with the state first - if let Some(qualified_op) = ops_by_qualified_paths.get(qualified_path) { + if let Some(op) = ops_by_qualified_paths.get(qualified_path) { // the path is being modified, inserted or deleted in the batch of operations - match qualified_op.op() { + match op { GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => Err( Error::InvalidBatchOperation("references can not point to trees being updated"), ) @@ -1118,56 +1073,72 @@ where | GroveOp::Patch { element, .. } => { match element { Element::Item(..) | Element::SumItem(..) => { - - - if element.get_flags().is_none() { - // There are no storage flags, we can just hash new element - let serialized = cost_return_on_error_no_add!( - &cost, - element.serialize(grove_version) - ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); - Ok(val_hash).wrap_with_cost(cost) - } else { + let serialized = cost_return_on_error_no_add!( + &cost, + element.serialize(grove_version) + ); + if element.get_flags().is_none() { + // There are no storage flags, we can just hash new element + let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) + } else { let mut new_element = element.clone(); let mut new_flags = new_element.get_flags_mut().as_mut().unwrap(); - // There are storage flags - let storage_costs = StorageCost {added_bytes: 2, replaced_bytes: 4, removed_bytes: StorageRemovedBytes::NoStorageRemoval}; - // it can be unmerged, let's get the value on disk - let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked - let old_element = cost_return_on_error!( - &mut cost, - self.get_and_deserialize_referenced_element(key, reference_path, grove_version)); + // it can be unmerged, let's get the value on disk + let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked + if let Some((old_element, old_serialized_element, is_in_sum_tree)) = cost_return_on_error!( + &mut cost, + self.get_and_deserialize_referenced_element( + key, + reference_path, + grove_version + ) + ) { let maybe_old_flags = old_element.get_flags_owned(); + + let old_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths(key.len() as u32, old_serialized_element.len() as u32, is_in_sum_tree); + let new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths(key.len() as u32, serialized.len() as u32, is_in_sum_tree); + + // There are storage flags + let storage_costs = TreeNode::storage_cost_for_update(new_storage_cost, old_storage_cost); + let changed = cost_return_on_error_no_add!( - &cost,(flags_update)(&storage_costs, maybe_old_flags, new_flags) + &cost, + (flags_update)(&storage_costs, maybe_old_flags, new_flags) .map_err(|e| match e { Error::JustInTimeElementFlagsClientError(_) => { - MerkError::ClientCorruptionError(e.to_string()).into() + MerkError::ClientCorruptionError(e.to_string()) + .into() } _ => MerkError::ClientCorruptionError( "non client error".to_string(), - ).into(), - })); + ) + .into(), + }) + ); if changed { // There are no storage flags, we can just hash new element let serialized = cost_return_on_error_no_add!( &cost, new_element.serialize(grove_version) ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } else { // There are no storage flags, we can just hash new element - let serialized = cost_return_on_error_no_add!( - &cost, - element.serialize(grove_version) - ); - let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); + + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } + } else { + let val_hash = + value_hash(&serialized).unwrap_add_cost(&mut cost); + Ok(val_hash).wrap_with_cost(cost) } + } } Element::Reference(path, ..) => { let path = cost_return_on_error_no_add!( @@ -1273,7 +1244,7 @@ where F: FnMut(&[Vec], bool) -> CostResult, Error>, S: StorageContext<'db>, { - fn insert(&mut self, op: &GroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, is_sum_tree: bool) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let mut inserted_path = op.path.to_path(); @@ -1309,8 +1280,8 @@ where fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - ops_by_qualified_paths: &BTreeMap>, MergeQualifiedGroveOp>, + ops_at_path_by_key: BTreeMap, + ops_by_qualified_paths: &BTreeMap>, GroveOp>, batch_apply_options: &BatchApplyOptions, flags_update: &mut G, split_removal_bytes: &mut SR, @@ -1331,7 +1302,7 @@ where let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { - match op.take_op() { + match op { GroveOp::InsertOnly { element } | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } @@ -1830,7 +1801,7 @@ impl GroveDb { } Entry::Occupied(occupied_entry) => { let mutable_occupied_entry = occupied_entry.into_mut(); - match mutable_occupied_entry.op_mut() { + match mutable_occupied_entry { GroveOp::ReplaceTreeRootKey { hash, root_key, @@ -1899,7 +1870,7 @@ impl GroveDb { } } } else { - let mut ops_on_path: BTreeMap = + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), @@ -1907,13 +1878,12 @@ impl GroveDb { hash: root_hash, root_key: calculated_root_key, sum: sum_value, - } - .into(), + }, ); ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), @@ -1926,7 +1896,7 @@ impl GroveDb { ); let mut ops_on_level: BTreeMap< KeyInfoPath, - BTreeMap, + BTreeMap, > = BTreeMap::new(); ops_on_level.insert(KeyInfoPath(parent_path.to_vec()), ops_on_path); ops_by_level_paths.insert(current_level - 1, ops_on_level); @@ -1949,7 +1919,7 @@ impl GroveDb { /// Then return the list of leftover operations fn apply_body<'db, S: StorageContext<'db>>( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -1994,7 +1964,7 @@ impl GroveDb { fn continue_partial_apply_body<'db, S: StorageContext<'db>>( &self, previous_leftover_operations: Option, - additional_ops: Vec, + additional_ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2040,7 +2010,7 @@ impl GroveDb { /// Applies operations on GroveDB without batching pub fn apply_operations_without_batching( &self, - ops: Vec, + ops: Vec, options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -2094,7 +2064,7 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, transaction: TransactionArg, grove_version: &GroveVersion, @@ -2121,12 +2091,12 @@ impl GroveDb { /// Applies batch on GroveDB pub fn apply_partial_batch( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, cost_based_add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -2190,8 +2160,8 @@ impl GroveDb { Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( |_| { Error::InvalidPath(format!( - "could not get key for parent of subtree for batch at path {}", - parent_path.to_vec().into_iter().map(hex::encode).join("/") + "could not get key for parent of subtree for batch at path [{}] for key {}", + parent_path.to_vec().into_iter().map(|v| hex_to_ascii(&v)).join("/"), hex_to_ascii(parent_key) )) } ) @@ -2301,7 +2271,7 @@ impl GroveDb { /// Applies batch of operations on GroveDB pub fn apply_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2341,7 +2311,7 @@ impl GroveDb { .unwrap_or(true); if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2457,7 +2427,7 @@ impl GroveDb { /// If it is not set we default to pausing at the root tree pub fn apply_partial_batch_with_element_flags_update( &self, - ops: Vec, + ops: Vec, batch_apply_options: Option, mut update_element_flags_function: impl FnMut( &StorageCost, @@ -2475,7 +2445,7 @@ impl GroveDb { mut add_on_operations: impl FnMut( &OperationCost, &Option, - ) -> Result, Error>, + ) -> Result, Error>, transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult<(), Error> { @@ -2505,7 +2475,7 @@ impl GroveDb { !batch_apply_options.disable_operation_consistency_check; if check_batch_operation_consistency { - let consistency_result = GroveDbOp::verify_consistency_of_operations(&ops); + let consistency_result = QualifiedGroveDbOp::verify_consistency_of_operations(&ops); if !consistency_result.is_empty() { return Err(Error::InvalidBatchOperation( "batch operations fail consistency checks", @@ -2706,7 +2676,7 @@ impl GroveDb { /// ops pub fn estimated_case_operations_for_batch( estimated_costs_type: EstimatedCostsType, - ops: Vec, + ops: Vec, batch_apply_options: Option, update_element_flags_function: impl FnMut( &StorageCost, @@ -2807,28 +2777,28 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -2888,12 +2858,12 @@ mod tests { // No two operations should be the same let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2908,12 +2878,12 @@ mod tests { // Can't perform 2 or more operations on the same node let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"a".to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2928,12 +2898,12 @@ mod tests { // Can't insert under a deleted path let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::new_item(vec![1]), ), - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), ]; assert!(matches!( db.apply_batch(ops, None, None, grove_version).unwrap(), @@ -2944,12 +2914,12 @@ mod tests { // Should allow invalid operations pass when disable option is set to true let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"b".to_vec(), Element::empty_tree(), @@ -2994,28 +2964,28 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element2.clone(), @@ -3093,28 +3063,28 @@ mod tests { let tx = db.start_transaction(); // let's start by inserting a tree structure let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec()], b"my_contract".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"0".to_vec(), Element::new_item(b"this is the contract".to_vec()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec()], b"1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"1".to_vec(), b"my_contract".to_vec(), b"1".to_vec()], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3124,7 +3094,7 @@ mod tests { b"0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3153,7 +3123,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3167,7 +3137,7 @@ mod tests { some_element_flags.clone(), ), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3178,7 +3148,7 @@ mod tests { b"my apples are safe".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3190,7 +3160,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3227,7 +3197,7 @@ mod tests { // now let's add an item let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3238,7 +3208,7 @@ mod tests { b"wisdom".to_vec(), Element::new_item_with_flags(b"Wisdom Ogwu".to_vec(), some_element_flags.clone()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3249,7 +3219,7 @@ mod tests { b"canteloupe!".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3261,7 +3231,7 @@ mod tests { b"0".to_vec(), Element::empty_tree_with_flags(some_element_flags.clone()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ b"1".to_vec(), b"my_contract".to_vec(), @@ -3302,60 +3272,60 @@ mod tests { .expect("successful batch apply"); } - fn grove_db_ops_for_contract_insert() -> Vec { + fn grove_db_ops_for_contract_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![], b"contract".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [0u8].to_vec(), Element::new_item(b"serialized_contract".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec()], [1u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"domain".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"normalized_domain_label".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"unique_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"domain".to_vec()], b"alias_records".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec()], b"preorder".to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], [0u8].to_vec(), Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![b"contract".to_vec(), [1u8].to_vec(), b"preorder".to_vec()], b"salted_domain".to_vec(), Element::empty_tree(), @@ -3364,10 +3334,10 @@ mod tests { grove_db_ops } - fn grove_db_ops_for_contract_document_insert() -> Vec { + fn grove_db_ops_for_contract_document_insert() -> Vec { let mut grove_db_ops = vec![]; - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3378,7 +3348,7 @@ mod tests { Element::new_item(b"serialized_domain".to_vec()), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3389,7 +3359,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3401,7 +3371,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3414,7 +3384,7 @@ mod tests { Element::empty_tree(), )); - grove_db_ops.push(GroveDbOp::insert_or_replace_op( + grove_db_ops.push(QualifiedGroveDbOp::insert_or_replace_op( vec![ b"contract".to_vec(), [1u8].to_vec(), @@ -3537,13 +3507,13 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3565,23 +3535,23 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"key2".to_vec(), element.clone(), ), - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), @@ -3629,17 +3599,17 @@ mod tests { .expect("cannot insert a subtree"); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; assert!(db .apply_batch(ops, None, None, grove_version) @@ -3653,23 +3623,23 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element, ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), + QualifiedGroveDbOp::delete_op(vec![b"key1".to_vec()], b"key2".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -3712,7 +3682,7 @@ mod tests { .expect("cannot insert value"); // Insertion into scalar is invalid - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"invalid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3723,7 +3693,7 @@ mod tests { .is_err()); // Insertion into a tree is correct - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"valid".to_vec()], b"key1".to_vec(), element.clone(), @@ -3768,8 +3738,8 @@ mod tests { // TEST_LEAF can not be overwritten let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], TEST_LEAF.to_vec(), element2), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], TEST_LEAF.to_vec(), element2), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key_subtree".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3795,8 +3765,8 @@ mod tests { // TEST_LEAF will be deleted so you can not insert underneath it let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3811,8 +3781,8 @@ mod tests { // We are testing with the batch apply option // validate_tree_insertion_does_not_override set to true let ops = vec![ - GroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::delete_op(vec![], TEST_LEAF.to_vec()), + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3842,12 +3812,12 @@ mod tests { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], TEST_LEAF.to_vec(), Element::new_item(b"ayy".to_vec()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -3898,7 +3868,7 @@ mod tests { ) .unwrap() .expect("cannot insert an item"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_item(b"ayy2".to_vec()), @@ -3962,27 +3932,27 @@ mod tests { let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), element.clone(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key".to_vec(), element2.clone(), ), - GroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), + QualifiedGroveDbOp::delete_op(vec![ANOTHER_TEST_LEAF.to_vec()], b"key1".to_vec()), ]; db.apply_batch(ops, None, None, grove_version) .unwrap() @@ -4068,7 +4038,7 @@ mod tests { } let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_or_replace_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element.clone(), @@ -4077,7 +4047,7 @@ mod tests { .unwrap() .expect("cannot apply batch"); - let batch = vec![GroveDbOp::insert_or_replace_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path, b"key".to_vec(), element, @@ -4110,7 +4080,7 @@ mod tests { let root_hash = db.root_hash(None, grove_version).unwrap().unwrap(); let element = Element::new_item(b"ayy".to_vec()); - let batch = vec![GroveDbOp::insert_or_replace_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( acc_path.clone(), b"key".to_vec(), element, @@ -4130,7 +4100,7 @@ mod tests { let grove_version = GroveVersion::latest(); // insert reference that points to non-existent item let db = make_test_grovedb(grove_version); - let batch = vec![GroveDbOp::insert_or_replace_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -4147,7 +4117,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ @@ -4155,7 +4125,7 @@ mod tests { b"invalid_path".to_vec(), ])), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem.clone(), @@ -4188,7 +4158,7 @@ mod tests { let db = make_test_grovedb(grove_version); let elem = Element::new_item(b"ayy".to_vec()); let batch = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key2".to_vec(), Element::new_reference_with_hops( @@ -4199,7 +4169,7 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::new_reference_with_hops( @@ -4210,7 +4180,7 @@ mod tests { Some(1), ), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"invalid_path".to_vec(), elem, diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index fe358081..8c60e9ec 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -11,7 +11,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType::{SiblingReference, UpstreamFromElementHeightReference}, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, @@ -46,8 +46,8 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), ]; let cost = db.apply_batch(ops, None, Some(&tx), grove_version).cost; assert_eq!( @@ -97,13 +97,13 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2).add(non_batch_cost_3); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key3".to_vec(), Element::new_reference(SiblingReference(b"key2".to_vec())), @@ -173,18 +173,18 @@ mod tests { .add(non_batch_cost_4); tx.rollback().expect("expected to rollback"); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::new_item_with_flags(b"pizza".to_vec(), Some([0, 1].to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key3".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key3".to_vec()], b"key4".to_vec(), Element::new_reference(UpstreamFromElementHeightReference( @@ -212,8 +212,8 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); @@ -268,8 +268,8 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - GroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), Element::empty_tree(), diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index fac9682f..d4c5d844 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -11,7 +11,7 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -72,7 +72,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -137,7 +137,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -212,7 +212,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -288,7 +288,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -357,7 +357,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -452,7 +452,7 @@ mod tests { )); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch_with_element_flags_update( ops, @@ -543,7 +543,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -623,7 +623,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -699,7 +699,7 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![GroveDbOp::delete_op(vec![], b"key1".to_vec())]; + let ops = vec![QualifiedGroveDbOp::delete_op(vec![], b"key1".to_vec())]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index 739de7ce..ac66c79b 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -20,10 +20,11 @@ mod tests { use intmap::IntMap; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; + use crate::reference_path::ReferencePathType::SiblingReference; #[test] fn test_batch_one_insert_costs_match_non_batch() { @@ -42,7 +43,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -57,7 +58,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -122,7 +123,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item(b"cat".to_vec()), @@ -200,7 +201,7 @@ mod tests { assert_eq!(cost.storage_cost.added_bytes, 143); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -293,7 +294,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_tree(), @@ -375,7 +376,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_tree(), @@ -453,7 +454,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 59].to_vec()), @@ -516,7 +517,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::new_item([0u8; 60].to_vec()), @@ -573,6 +574,136 @@ mod tests { ); } + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_below_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 128 + // 1 for the flag options + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 56 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 1 byte for the value_size (required space for 127) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 128 + 40 = 205 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 205, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + + #[test] + fn test_batch_root_one_insert_with_flags_cost_right_above_value_required_cost_of_2() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 0])), + )]; + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + let cost = cost_result.cost; + // Explanation for 243 storage_written_bytes + + // Key -> 37 bytes + // 32 bytes for the key prefix + // 4 bytes for the key + // 1 byte for key_size (required space for 36) + + // Value -> 130 + // 1 for the flag option + // 1 for flags size + // 2 for flag bytes + // 1 for the enum type + // 1 for the value size + // 60 bytes + // 32 for node hash + // 32 for value hash + // 1 for basic merk + // 2 byte for the value_size (required space for 128) + + // Parent Hook -> 40 + // Key Bytes 4 + // Hash Size 32 + // Key Length 1 + // Child Heights 2 + // Sum 1 + // Total 37 + 130 + 40 = 207 + + // Hash node calls + // 2 for the node hash + // 1 for the value hash (just under) + // 1 kv_digest_to_kv_hash + + // Seek Count + // 1 to load from root tree + // 1 to insert + // 1 to update root tree + assert_eq!( + cost, + OperationCost { + seek_count: 3, + storage_cost: StorageCost { + added_bytes: 207, + replaced_bytes: 0, + removed_bytes: NoStorageRemoval, + }, + storage_loaded_bytes: 0, + hash_node_calls: 4, + } + ); + } + #[test] fn test_batch_root_one_update_item_bigger_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -601,7 +732,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![1])), @@ -667,7 +798,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), @@ -728,6 +859,124 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 11, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 316, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 556, // todo: verify this + hash_node_calls: 17, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_item_smaller_cost_no_flags() { let grove_version = GroveVersion::latest(); @@ -756,7 +1005,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![1])), @@ -821,7 +1070,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), @@ -883,6 +1132,126 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_smaller_cost_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"keyref", + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => Ok(true), + _ => Ok(false), + }, + |_flags, _removed_key_bytes, removed_value_bytes| { + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, removed_value_bytes); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + Ok((NoStorageRemoval, SectionedStorageRemoval(removed_bytes))) + }, + Some(&tx), + grove_version, + ) + .cost; + + let mut removed_bytes = StorageRemovalPerEpochByIdentifier::default(); + // we are removing 1 byte from epoch 0 for an identity + let mut removed_bytes_for_identity = IntMap::new(); + removed_bytes_for_identity.insert(0, 1); + removed_bytes.insert(Identifier::default(), removed_bytes_for_identity); + + assert_eq!( + cost, + OperationCost { + seek_count: 11, // todo: verify this + storage_cost: StorageCost { + added_bytes: 0, + replaced_bytes: 315, // todo: verify this + removed_bytes: SectionedStorageRemoval(removed_bytes) + }, + storage_loaded_bytes: 556, // todo: verify this + hash_node_calls: 17, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_tree_bigger_flags_cost() { let grove_version = GroveVersion::latest(); @@ -911,7 +1280,7 @@ mod tests { .expect("expected to insert item"); // We are adding 1 byte to the flags - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_tree_with_flags(None, Some(vec![0, 1, 1])), @@ -965,4 +1334,188 @@ mod tests { } ); } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_refresh_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + ), QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + )]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 9, // todo: verify this + storage_cost: StorageCost { + added_bytes: 4, + replaced_bytes: 285, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 502, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + + #[test] + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_new_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + )]; + + let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); + cost_result.value.expect("expected to execute batch"); + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::insert_only_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + assert_eq!( + cost, + OperationCost { + seek_count: 7, // todo: verify this + storage_cost: StorageCost { + added_bytes: 160, + replaced_bytes: 168, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 266, // todo: verify this + hash_node_calls: 12, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index bf5637d0..fad794f4 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -5,7 +5,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -43,7 +43,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -101,7 +101,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_op( + let ops = vec![QualifiedGroveDbOp::delete_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), )]; @@ -146,7 +146,7 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs index 65786eb9..d58e7327 100644 --- a/grovedb/src/batch/single_sum_item_insert_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_insert_cost_tests.rs @@ -9,7 +9,7 @@ mod tests { use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; @@ -42,7 +42,7 @@ mod tests { ) .cost; tx.rollback().expect("expected to rollback"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item(150), @@ -57,7 +57,7 @@ mod tests { let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -134,7 +134,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -220,7 +220,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key1".to_vec(), Element::empty_sum_tree(), @@ -306,7 +306,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -396,7 +396,7 @@ mod tests { .unwrap() .expect("successful root tree leaf insert"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"0".to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), @@ -487,7 +487,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 42].to_vec())), @@ -562,7 +562,7 @@ mod tests { .unwrap() .expect("expected to insert sum tree"); - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"sum_tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(15, Some([0; 43].to_vec())), @@ -648,7 +648,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, None), @@ -714,7 +714,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(100000, Some(vec![1])), @@ -780,7 +780,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, None), @@ -846,7 +846,7 @@ mod tests { .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], b"key1".to_vec(), Element::new_sum_item_with_flags(5, Some(vec![1])), diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 598241f6..2d2db076 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -251,7 +251,6 @@ impl Element { ); // todo: we actually don't need to deserialize the whole element let element = Element::deserialize(value, grove_version)?; - println!("element is {}", element); let cost = match element { Element::Tree(_, flags) => { let flags_len = flags.map_or(0, |flags| { diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 3ed1abd1..a95cc7ca 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -17,7 +17,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, QualifiedGroveDbOp, KeyInfoPath}, Error, GroveDb, }; @@ -34,7 +34,7 @@ impl GroveDb { validate: bool, estimated_layer_info: IntMap, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "average_case_delete_operations_for_delete_up_tree_while_empty", grove_version @@ -145,7 +145,7 @@ impl GroveDb { except_keys_count: u16, estimated_key_element_size: EstimatedKeyAndElementSize, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "average_case_delete_operation_for_delete", grove_version @@ -188,6 +188,6 @@ impl GroveDb { estimated_key_element_size.0 + HASH_LENGTH_U32, ); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index dd331b69..7ecfce83 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -11,7 +11,7 @@ use grovedb_version::{ }; use crate::{ - batch::GroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, + batch::QualifiedGroveDbOp, operations::delete::DeleteOptions, ElementFlags, Error, GroveDb, TransactionArg, }; @@ -122,7 +122,7 @@ impl GroveDb { .delete_up_tree_while_empty_with_sectional_storage ); let mut cost = OperationCost::default(); - let mut batch_operations: Vec = Vec::new(); + let mut batch_operations: Vec = Vec::new(); let maybe_ops = cost_return_on_error!( &mut cost, @@ -170,10 +170,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - mut current_batch_operations: Vec, + mut current_batch_operations: Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -202,10 +202,10 @@ impl GroveDb { key: &[u8], options: &DeleteUpTreeOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &mut Vec, + current_batch_operations: &mut Vec, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult>, Error> { + ) -> CostResult>, Error> { check_grovedb_v0_with_cost!( "delete", grove_version diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 967b68e7..1181fc53 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -33,7 +33,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::{ - batch::{GroveDbOp, GroveOp}, + batch::{QualifiedGroveDbOp, GroveOp}, util::storage_context_with_parent_optional_tx, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -512,10 +512,10 @@ impl GroveDb { key: &[u8], options: &DeleteOptions, is_known_to_be_subtree_with_sum: Option<(bool, bool)>, - current_batch_operations: &[GroveDbOp], + current_batch_operations: &[QualifiedGroveDbOp], transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete_operation_for_delete_internal", grove_version @@ -610,7 +610,7 @@ impl GroveDb { Ok(None) } } else if is_empty { - Ok(Some(GroveDbOp::delete_tree_op( + Ok(Some(QualifiedGroveDbOp::delete_tree_op( path.to_vec(), key.to_vec(), is_subtree_with_sum, @@ -622,7 +622,7 @@ impl GroveDb { }; result.wrap_with_cost(cost) } else { - Ok(Some(GroveDbOp::delete_op(path.to_vec(), key.to_vec()))).wrap_with_cost(cost) + Ok(Some(QualifiedGroveDbOp::delete_op(path.to_vec(), key.to_vec()))).wrap_with_cost(cost) } } } diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index b2a50bb2..6b40d527 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -13,7 +13,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, GroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, QualifiedGroveDbOp, KeyInfoPath}, element::SUM_TREE_COST_SIZE, Error, GroveDb, }; @@ -29,7 +29,7 @@ impl GroveDb { intermediate_tree_info: IntMap<(bool, u32)>, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "delete", grove_version @@ -127,7 +127,7 @@ impl GroveDb { except_keys_count: u16, max_element_size: u32, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult { check_grovedb_v0_with_cost!( "worst_case_delete_operation_for_delete", grove_version @@ -165,6 +165,6 @@ impl GroveDb { // in the worst case this is a tree add_worst_case_cost_for_is_empty_tree_except(&mut cost, except_keys_count); - Ok(GroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) } } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index dfb4623c..ec3cdbfa 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -758,7 +758,7 @@ pub fn make_deep_tree_with_sum_trees(grove_version: &GroveVersion) -> TempGroveD } mod tests { - use batch::GroveDbOp; + use batch::QualifiedGroveDbOp; use grovedb_merk::proofs::query::SubqueryBranch; use super::*; @@ -4071,22 +4071,22 @@ mod tests { let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"value".to_vec(), Element::new_item(b"hello".to_vec()), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refc".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"value".to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refb".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refc".to_vec())), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"refa".to_vec(), Element::new_reference(ReferencePathType::SiblingReference(b"refb".to_vec())), diff --git a/grovedb/src/tests/query_tests.rs b/grovedb/src/tests/query_tests.rs index 5d80b994..4ca273fe 100644 --- a/grovedb/src/tests/query_tests.rs +++ b/grovedb/src/tests/query_tests.rs @@ -7,7 +7,7 @@ mod tests { use tempfile::TempDir; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, query_result_type::{ PathKeyOptionalElementTrio, QueryResultElement::PathKeyElementTrioResultItem, QueryResultElements, QueryResultType, @@ -1595,7 +1595,7 @@ mod tests { 77, 252, 86, 99, 107, 197, 226, 188, 54, 239, 64, 17, 37, ]; - let batch = vec![GroveDbOp::insert_or_replace_op( + let batch = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![], vec![1], Element::empty_tree(), @@ -1605,27 +1605,27 @@ mod tests { .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1]], tree_name_slice.to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec()], vec![1], Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![vec![1], tree_name_slice.to_vec(), vec![1]], b"person".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1635,7 +1635,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1651,7 +1651,7 @@ mod tests { .expect("should apply batch"); let batch = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1662,7 +1662,7 @@ mod tests { b"person_id_1".to_vec(), Element::new_item(vec![50]), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1673,7 +1673,7 @@ mod tests { b"cammi".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), @@ -1685,7 +1685,7 @@ mod tests { b"\0".to_vec(), Element::empty_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ vec![1], tree_name_slice.to_vec(), diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 78f73ed4..b255f653 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -9,7 +9,7 @@ use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; use crate::{ - batch::GroveDbOp, + batch::QualifiedGroveDbOp, reference_path::ReferencePathType, tests::{make_test_grovedb, TEST_LEAF}, Element, Error, GroveDb, PathQuery, @@ -779,17 +779,17 @@ fn test_sum_tree_with_batches() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec()], b"key1".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"a".to_vec(), Element::new_item(vec![214]), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"b".to_vec(), Element::new_sum_item(10), @@ -835,7 +835,7 @@ fn test_sum_tree_with_batches() { )); // Create new batch to use existing tree - let ops = vec![GroveDbOp::insert_or_replace_op( + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"c".to_vec(), Element::new_sum_item(10), @@ -871,42 +871,42 @@ fn test_sum_tree_with_batches() { // Add a new sum tree with its own sum items, should affect sum of original // tree let ops = vec![ - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"d".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"first".to_vec(), Element::new_sum_item(4), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"d".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec()], b"e".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"first".to_vec(), Element::new_sum_item(12), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"second".to_vec(), Element::new_item(vec![4]), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![TEST_LEAF.to_vec(), b"key1".to_vec(), b"e".to_vec()], b"third".to_vec(), Element::empty_sum_tree(), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), @@ -916,7 +916,7 @@ fn test_sum_tree_with_batches() { b"a".to_vec(), Element::new_sum_item(5), ), - GroveDbOp::insert_or_replace_op( + QualifiedGroveDbOp::insert_or_replace_op( vec![ TEST_LEAF.to_vec(), b"key1".to_vec(), diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 6b2710b6..fb116dac 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -158,16 +158,8 @@ impl TreeNode { self.inner.kv.feature_type.is_sum_feature() } - /// Compare current value byte cost with old cost and return - /// current value byte cost with updated `KeyValueStorageCost` - pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( - &self, - current_value_byte_cost: u32, - old_cost: u32, - ) -> Result<(u32, KeyValueStorageCost), Error> { - let key_storage_cost = StorageCost { - ..Default::default() - }; + pub fn storage_cost_for_update( current_value_byte_cost: u32, + old_cost: u32) -> StorageCost { let mut value_storage_cost = StorageCost { ..Default::default() }; @@ -190,6 +182,20 @@ impl TreeNode { value_storage_cost.added_bytes += current_value_byte_cost - old_cost; } } + value_storage_cost + } + + /// Compare current value byte cost with old cost and return + /// current value byte cost with updated `KeyValueStorageCost` + pub fn kv_with_parent_hook_size_and_storage_cost_from_old_cost( + &self, + current_value_byte_cost: u32, + old_cost: u32, + ) -> Result<(u32, KeyValueStorageCost), Error> { + let key_storage_cost = StorageCost { + ..Default::default() + }; + let value_storage_cost = Self::storage_cost_for_update(current_value_byte_cost, old_cost); let key_value_storage_cost = KeyValueStorageCost { key_storage_cost, // the key storage cost is added later From 9a093140f7ce69a867a9a44ee5ce88038f5571d6 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 05:15:55 +0700 Subject: [PATCH 06/19] more work --- grovedb/src/batch/mod.rs | 16 +++++++++------- merk/src/merk/cow_like.rs | 29 +++++++++++++++++++++++++++++ merk/src/merk/mod.rs | 1 + 3 files changed, 39 insertions(+), 7 deletions(-) create mode 100644 merk/src/merk/cow_like.rs diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index b1dc0fa7..041e51f6 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -65,6 +65,7 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; +use grovedb_merk::merk::cow_like::CowLikeMerk; use grovedb_merk::tree::kv::KV; use grovedb_merk::tree::TreeNode; use key_info::{KeyInfo, KeyInfo::KnownKey}; @@ -790,6 +791,9 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); + self + .merks.insert(reference_path.to_vec(), merk); + let referenced_element_value_hash = cost_return_on_error!( &mut cost, referenced_element_value_hash_opt @@ -899,6 +903,10 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); + let is_sum_tree = merk.is_sum_tree; + + self.merks.insert(reference_path.to_vec(), merk); + if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( &cost, @@ -907,7 +915,7 @@ where }) ); - Ok(Some((element, referenced_element, merk.is_sum_tree))).wrap_with_cost(cost) + Ok(Some((element, referenced_element, is_sum_tree))).wrap_with_cost(cost) } else { Ok(None).wrap_with_cost(cost) } @@ -1588,17 +1596,11 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { - println!( - "getting specialized cost key: {}, value {}", - key.len(), - value.len() - ); Element::specialized_costs_for_key_value(key, value, is_sum_tree, grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &mut |storage_costs, old_value, new_value| { - println!("updating to new value"); // todo: change the flags without full deserialization let old_element = Element::deserialize(old_value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; diff --git a/merk/src/merk/cow_like.rs b/merk/src/merk/cow_like.rs new file mode 100644 index 00000000..ada8c54b --- /dev/null +++ b/merk/src/merk/cow_like.rs @@ -0,0 +1,29 @@ +//! Module for [CowLikeMerk]: simple abstraction over owned and borrowed Merk. + +use std::{ + hash::{Hash, Hasher}, + ops::Deref, +}; +use grovedb_storage::StorageContext; +use crate::Merk; + +/// A smart pointer that follows the semantics of [Cow](std::borrow::Cow) except +/// provides no means for mutability and thus doesn't require [Clone]. +#[derive(Debug)] +pub enum CowLikeMerk<'db, S> { + /// Owned variant + Owned(Merk), + /// Borrowed variant + Borrowed(&'db Merk), +} + +impl<'db, S: StorageContext<'db>> Deref for CowLikeMerk<'db, S> { + type Target = Merk; + + fn deref(&self) -> &Self::Target { + match self { + Self::Owned(v) => v, + Self::Borrowed(s) => s, + } + } +} \ No newline at end of file diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index ee0deccc..73e39435 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -41,6 +41,7 @@ pub mod open; pub mod prove; pub mod restore; pub mod source; +pub mod cow_like; use std::{ cell::Cell, From dc107416b387e956599383a1f42b7af5ffe2cde2 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 05:43:16 +0700 Subject: [PATCH 07/19] more work --- grovedb/src/batch/estimated_costs/average_case_costs.rs | 4 ++-- grovedb/src/batch/estimated_costs/worst_case_costs.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index b9a5b4be..e18c9ac5 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -22,7 +22,7 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::{batch::GroveDBOp, Element}; +use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ @@ -70,7 +70,7 @@ impl GroveOp { grove_version, ) } - GroveOp::InsertOrReplace { element } => GroveDb::average_case_merk_insert_element( + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => GroveDb::average_case_merk_insert_element( key, element, in_tree_using_sums, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index a7e2edc5..cc636b5e 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -21,7 +21,7 @@ use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use crate::{batch::QualifiedGroveDbOp, Element}; +use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ @@ -67,7 +67,7 @@ impl GroveOp { grove_version, ) } - GroveOp::InsertOrReplace { element } => GroveDb::worst_case_merk_insert_element( + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => GroveDb::worst_case_merk_insert_element( key, element, is_in_parent_sum_tree, From d7a4cb601e8a11dbb1a76f54dc3deff3c8374819 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 05:47:29 +0700 Subject: [PATCH 08/19] more work --- grovedb/src/batch/estimated_costs/average_case_costs.rs | 6 +++--- grovedb/src/batch/estimated_costs/worst_case_costs.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index e18c9ac5..3b384937 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -184,8 +184,8 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, QualifiedGroveDbOp>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -247,7 +247,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op.op() + op .average_case_cost(&key, layer_element_estimates, false, grove_version) ); } diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index cc636b5e..d94b47d5 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -180,8 +180,8 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { fn execute_ops_on_path( &mut self, path: &KeyInfoPath, - ops_at_path_by_key: BTreeMap, - _ops_by_qualified_paths: &BTreeMap>, QualifiedGroveDbOp>, + ops_at_path_by_key: BTreeMap, + _ops_by_qualified_paths: &BTreeMap>, GroveOp>, _batch_apply_options: &BatchApplyOptions, _flags_update: &mut G, _split_removal_bytes: &mut SR, @@ -216,7 +216,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op.op().worst_case_cost( + op.worst_case_cost( &key, false, worst_case_layer_element_estimates, From a6b0526291128e562d34bc14f02dfd5a5a95ca75 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 07:26:10 +0700 Subject: [PATCH 09/19] temp --- grovedb/src/element/get.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1fda91dd..34e9a355 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -101,6 +101,9 @@ impl Element { ); Self::get_optional_from_storage(storage, key.as_ref(), grove_version).map(|result| { let value = result?; + if value.is_none() { + panic!("here"); + } value.ok_or_else(|| { Error::PathKeyNotFound(format!( "key not found in Merk for get from storage: {}", From 5a46913b1d016f7b828f0d7dd37e47b480284075 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 07:32:16 +0700 Subject: [PATCH 10/19] temp --- grovedb/src/element/get.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 34e9a355..1fda91dd 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -101,9 +101,6 @@ impl Element { ); Self::get_optional_from_storage(storage, key.as_ref(), grove_version).map(|result| { let value = result?; - if value.is_none() { - panic!("here"); - } value.ok_or_else(|| { Error::PathKeyNotFound(format!( "key not found in Merk for get from storage: {}", From a1e8414f44d42e65f6dbdc14685df19ef885d378 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Mon, 19 Aug 2024 16:22:02 +0700 Subject: [PATCH 11/19] temp --- grovedb/src/batch/mod.rs | 17 +++ grovedb/src/batch/multi_insert_cost_tests.rs | 103 ++++++++++++++++- grovedb/src/batch/single_insert_cost_tests.rs | 107 ++++++++++++++++++ merk/src/merk/cow_like.rs | 12 +- 4 files changed, 237 insertions(+), 2 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 041e51f6..45562ee0 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -750,6 +750,7 @@ where /// tree being updated. fn process_reference<'a, G, SR>( &'a mut self, + temp_merks: BTreeMap<&Vec>, &Merk>, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, @@ -768,6 +769,10 @@ where { let mut cost = OperationCost::default(); let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked + let held_merk = None; + if let Some(merk) = temp_merks { + + } let reference_merk_wrapped = self .merks .remove(reference_path) @@ -819,6 +824,7 @@ where path_from_reference_qualified_path_type(referenced_path.clone(), qualified_path) ); self.follow_reference_get_value_hash( + temp_merks, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -832,6 +838,7 @@ where // the referenced element as an element further down in the chain might still // change in the batch. self.process_reference_with_hop_count_greater_than_one( + temp_merks, key, reference_path, qualified_path, @@ -970,6 +977,7 @@ where /// tree being updated, which is not allowed. fn process_reference_with_hop_count_greater_than_one<'a, G, SR>( &'a mut self, + parent_tree: &Merk, key: &[u8], reference_path: &[Vec], qualified_path: &[Vec], @@ -1018,6 +1026,7 @@ where path_from_reference_qualified_path_type(path, qualified_path) ); self.follow_reference_get_value_hash( + &parent_tree, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1045,6 +1054,7 @@ where /// All these has to be taken into account. fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, + temp_merks: BTreeMap<&Vec>, &Merk>, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, @@ -1059,6 +1069,7 @@ where u32, u32, ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, + S: StorageContext<'db>, { let mut cost = OperationCost::default(); if recursions_allowed == 0 { @@ -1157,6 +1168,7 @@ where ) ); self.follow_reference_get_value_hash( + parent_tree_at_path, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1186,6 +1198,7 @@ where path_from_reference_qualified_path_type(path.clone(), qualified_path) ); self.follow_reference_get_value_hash( + parent_tree_at_path, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1211,6 +1224,7 @@ where None }; self.process_reference( + &parent_tree_at_path, qualified_path, ops_by_qualified_paths, recursions_allowed, @@ -1229,6 +1243,7 @@ where } } else { self.process_reference( + parent_tree_at_path, qualified_path, ops_by_qualified_paths, recursions_allowed, @@ -1341,6 +1356,7 @@ where let referenced_element_value_hash = cost_return_on_error!( &mut cost, self.follow_reference_get_value_hash( + &merk, path_reference.as_slice(), ops_by_qualified_paths, element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), @@ -1485,6 +1501,7 @@ where let referenced_element_value_hash = cost_return_on_error!( &mut cost, self.follow_reference_get_value_hash( + &merk, path_reference.as_slice(), ops_by_qualified_paths, max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index 8c60e9ec..c48a2fff 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -3,11 +3,13 @@ #[cfg(feature = "full")] mod tests { use std::{ops::Add, option::Option::None}; - + use integer_encoding::VarInt; use grovedb_costs::{ storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, OperationCost, }; + use grovedb_costs::storage_cost::removal::StorageRemovedBytes::BasicStorageRemoval; + use grovedb_costs::storage_cost::transition::OperationStorageTransitionType; use grovedb_version::version::GroveVersion; use crate::{ @@ -322,4 +324,103 @@ mod tests { } ); } + + #[test] + fn test_batch_insert_item_in_also_inserted_sub_tree_with_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"tree2".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ).cost_as_result().expect("expect no error"); + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 10, // todo: verify this + storage_cost: StorageCost { + added_bytes: 163, + replaced_bytes: 196, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 393, // todo: verify this + hash_node_calls: 17, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } } diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index ac66c79b..ba32bbf5 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -977,6 +977,113 @@ mod tests { ); } + #[test] + fn test_batch_root_one_update_item_bigger_cost_with_insert_reference() { + let grove_version = GroveVersion::latest(); + let db = make_empty_grovedb(); + let tx = db.start_transaction(); + db.insert( + EMPTY_PATH, + b"tree", + Element::empty_tree(), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert tree"); + + db.insert( + [b"tree".as_slice()].as_ref(), + b"key1", + Element::new_item_with_flags(b"value1".to_vec(), Some(vec![0, 0])), + None, + None, + grove_version, + ) + .unwrap() + .expect("expected to insert item"); + + // We are adding 2 bytes + let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), + )]; + + let cost = db + .apply_batch_with_element_flags_update( + ops, + None, + |cost, old_flags, new_flags| match cost.transition_type() { + OperationStorageTransitionType::OperationUpdateBiggerSize => { + if new_flags[0] == 0 { + new_flags[0] = 1; + let new_flags_epoch = new_flags[1]; + new_flags[1] = old_flags.unwrap()[1]; + new_flags.push(new_flags_epoch); + new_flags.extend(cost.added_bytes.encode_var_vec()); + assert_eq!(new_flags, &vec![1u8, 0, 1, 2]); + Ok(true) + } else { + assert_eq!(new_flags[0], 1); + Ok(false) + } + } + OperationStorageTransitionType::OperationUpdateSmallerSize => { + new_flags.extend(vec![1, 2]); + Ok(true) + } + _ => Ok(false), + }, + |_flags, removed_key_bytes, removed_value_bytes| { + Ok(( + BasicStorageRemoval(removed_key_bytes), + BasicStorageRemoval(removed_value_bytes), + )) + }, + Some(&tx), + grove_version, + ) + .cost; + + // Hash node calls + + // Seek Count + + assert_eq!( + cost, + OperationCost { + seek_count: 10, // todo: verify this + storage_cost: StorageCost { + added_bytes: 163, + replaced_bytes: 196, // todo: verify this + removed_bytes: NoStorageRemoval + }, + storage_loaded_bytes: 393, // todo: verify this + hash_node_calls: 17, // todo: verify this + } + ); + + let issues = db + .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .unwrap(); + assert_eq!( + issues.len(), + 0, + "reference issue: {}", + issues + .iter() + .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) + .collect::>() + .join(" | ") + ); + } + #[test] fn test_batch_root_one_update_item_smaller_cost_no_flags() { let grove_version = GroveVersion::latest(); diff --git a/merk/src/merk/cow_like.rs b/merk/src/merk/cow_like.rs index ada8c54b..50539305 100644 --- a/merk/src/merk/cow_like.rs +++ b/merk/src/merk/cow_like.rs @@ -4,6 +4,7 @@ use std::{ hash::{Hash, Hasher}, ops::Deref, }; +use std::ops::DerefMut; use grovedb_storage::StorageContext; use crate::Merk; @@ -14,7 +15,7 @@ pub enum CowLikeMerk<'db, S> { /// Owned variant Owned(Merk), /// Borrowed variant - Borrowed(&'db Merk), + Borrowed(&'db mut Merk), } impl<'db, S: StorageContext<'db>> Deref for CowLikeMerk<'db, S> { @@ -26,4 +27,13 @@ impl<'db, S: StorageContext<'db>> Deref for CowLikeMerk<'db, S> { Self::Borrowed(s) => s, } } +} + +impl<'db, S: StorageContext<'db>> DerefMut for CowLikeMerk<'db, S> { + fn deref_mut(&mut self) -> &mut Self::Target { + match self { + Self::Owned(v) => v, + Self::Borrowed(s) => s, + } + } } \ No newline at end of file From 62194c8eb2724c7c740f23bef3f32b1723c97675 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Mon, 19 Aug 2024 16:34:25 +0200 Subject: [PATCH 12/19] wip --- grovedb/src/batch/batch_structure.rs | 16 +- .../estimated_costs/average_case_costs.rs | 25 +- .../batch/estimated_costs/worst_case_costs.rs | 24 +- grovedb/src/batch/mod.rs | 228 ++++++++++-------- grovedb/src/batch/multi_insert_cost_tests.rs | 94 +++++--- .../src/batch/single_deletion_cost_tests.rs | 30 ++- grovedb/src/batch/single_insert_cost_tests.rs | 176 ++++++++------ .../single_sum_item_deletion_cost_tests.rs | 12 +- grovedb/src/operations/delete/average_case.rs | 8 +- grovedb/src/operations/delete/mod.rs | 8 +- grovedb/src/operations/delete/worst_case.rs | 8 +- merk/benches/merk.rs | 2 +- merk/src/merk/cow_like.rs | 39 --- merk/src/merk/mod.rs | 1 - merk/src/tree/mod.rs | 3 +- 15 files changed, 391 insertions(+), 283 deletions(-) delete mode 100644 merk/src/merk/cow_like.rs diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index 4fc3a381..9cfe03db 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -16,7 +16,7 @@ use nohash_hasher::IntMap; #[cfg(feature = "full")] use crate::{ - batch::{key_info::KeyInfo, QualifiedGroveDbOp, GroveOp, KeyInfoPath, TreeCache}, + batch::{key_info::KeyInfo, GroveOp, KeyInfoPath, QualifiedGroveDbOp, TreeCache}, Element, ElementFlags, Error, }; @@ -120,7 +120,10 @@ where ops_by_qualified_paths.insert(path.to_path_consume(), op.op.clone()); let op_cost = OperationCost::default(); let op_result = match &op.op { - GroveOp::InsertOnly { element } | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { + GroveOp::InsertOnly { element } + | GroveOp::InsertOrReplace { element } + | GroveOp::Replace { element } + | GroveOp::Patch { element, .. } => { if let Element::Tree(..) = element { cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false)); } else if let Element::SumTree(..) = element { @@ -128,9 +131,10 @@ where } Ok(()) } - GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree | GroveOp::DeleteSumTree => { - Ok(()) - } + GroveOp::RefreshReference { .. } + | GroveOp::Delete + | GroveOp::DeleteTree + | GroveOp::DeleteSumTree => Ok(()), GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", @@ -171,6 +175,6 @@ where split_removal_bytes: split_remove_bytes_function, last_level: current_last_level, }) - .wrap_with_cost(cost) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index 3b384937..ef64b0f4 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -26,8 +26,8 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, QualifiedGroveDbOp, GroveOp, KeyInfoPath, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; @@ -70,13 +70,15 @@ impl GroveOp { grove_version, ) } - GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => GroveDb::average_case_merk_insert_element( - key, - element, - in_tree_using_sums, - propagate_if_input(), - grove_version, - ), + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::average_case_merk_insert_element( + key, + element, + in_tree_using_sums, + propagate_if_input(), + grove_version, + ) + } GroveOp::RefreshReference { reference_path_type, max_reference_hop, @@ -247,8 +249,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { for (key, op) in ops_at_path_by_key.into_iter() { cost_return_on_error!( &mut cost, - op - .average_case_cost(&key, layer_element_estimates, false, grove_version) + op.average_case_cost(&key, layer_element_estimates, false, grove_version) ); } @@ -310,7 +311,7 @@ mod tests { use crate::{ batch::{ estimated_costs::EstimatedCostsType::AverageCaseCostsType, key_info::KeyInfo, - QualifiedGroveDbOp, KeyInfoPath, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index d94b47d5..9bf9a808 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -25,8 +25,8 @@ use crate::Element; #[cfg(feature = "full")] use crate::{ batch::{ - key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, QualifiedGroveDbOp, GroveOp, KeyInfoPath, - TreeCache, + key_info::KeyInfo, mode::BatchRunMode, BatchApplyOptions, GroveOp, KeyInfoPath, + QualifiedGroveDbOp, TreeCache, }, Error, GroveDb, }; @@ -67,13 +67,15 @@ impl GroveOp { grove_version, ) } - GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => GroveDb::worst_case_merk_insert_element( - key, - element, - is_in_parent_sum_tree, - propagate_if_input(), - grove_version, - ), + GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { + GroveDb::worst_case_merk_insert_element( + key, + element, + is_in_parent_sum_tree, + propagate_if_input(), + grove_version, + ) + } GroveOp::RefreshReference { reference_path_type, max_reference_hop, @@ -275,8 +277,8 @@ mod tests { use crate::{ batch::{ - estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, QualifiedGroveDbOp, - KeyInfoPath, + estimated_costs::EstimatedCostsType::WorstCaseCostsType, key_info::KeyInfo, + KeyInfoPath, QualifiedGroveDbOp, }, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, GroveDb, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 45562ee0..77a4fbc0 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,8 +48,11 @@ use grovedb_costs::{ }; use grovedb_merk::{ tree::{ - kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, + kv::{ + ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, + KV, + }, + value_hash, TreeNode, NULL_HASH, }, CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndSum, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, @@ -65,9 +68,6 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; -use grovedb_merk::merk::cow_like::CowLikeMerk; -use grovedb_merk::tree::kv::KV; -use grovedb_merk::tree::TreeNode; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -77,13 +77,12 @@ use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, - operations::get::MAX_REFERENCE_HOPS, + operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, }, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; -use crate::operations::proof::util::hex_to_ascii; /// Operations #[derive(Debug, PartialEq, Eq, Hash, Clone)] @@ -548,7 +547,9 @@ impl QualifiedGroveDbOp { } /// Verify consistency of operations - pub fn verify_consistency_of_operations(ops: &[QualifiedGroveDbOp]) -> GroveDbOpConsistencyResults { + pub fn verify_consistency_of_operations( + ops: &[QualifiedGroveDbOp], + ) -> GroveDbOpConsistencyResults { let ops_len = ops.len(); // operations should not have any duplicates let mut repeated_ops = vec![]; @@ -654,7 +655,7 @@ pub struct GroveDbOpConsistencyResults { repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, // the u16 is count same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, /* the deleted op first, - * then inserts under */ + * then inserts under */ } impl GroveDbOpConsistencyResults { @@ -750,7 +751,6 @@ where /// tree being updated. fn process_reference<'a, G, SR>( &'a mut self, - temp_merks: BTreeMap<&Vec>, &Merk>, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, @@ -769,16 +769,14 @@ where { let mut cost = OperationCost::default(); let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked - let held_merk = None; - if let Some(merk) = temp_merks { - } - let reference_merk_wrapped = self - .merks - .remove(reference_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(reference_path, false)); - let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(reference_path, false) + )), + }; // Here the element being referenced doesn't change in the same batch // and the max hop count is 1, meaning it should point directly to the base @@ -796,9 +794,6 @@ where .map_err(|e| Error::CorruptedData(e.to_string())) ); - self - .merks.insert(reference_path.to_vec(), merk); - let referenced_element_value_hash = cost_return_on_error!( &mut cost, referenced_element_value_hash_opt @@ -824,7 +819,6 @@ where path_from_reference_qualified_path_type(referenced_path.clone(), qualified_path) ); self.follow_reference_get_value_hash( - temp_merks, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -838,7 +832,6 @@ where // the referenced element as an element further down in the chain might still // change in the batch. self.process_reference_with_hop_count_greater_than_one( - temp_merks, key, reference_path, qualified_path, @@ -869,9 +862,10 @@ where /// /// # Returns /// - /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` and the serialized counterpart - /// if the retrieval and deserialization are successful, wrapped in the associated cost. - /// Also returns if the merk of the element is a sum tree as a bool. + /// * `Ok((Element, Vec, bool))` - Returns the deserialized `Element` + /// and the serialized counterpart if the retrieval and deserialization + /// are successful, wrapped in the associated cost. Also returns if the + /// merk of the element is a sum tree as a bool. /// * `Err(Error)` - Returns an error if any issue occurs during the /// retrieval or deserialization of the referenced element. /// @@ -891,13 +885,13 @@ where ) -> CostResult, bool)>, Error> { let mut cost = OperationCost::default(); - let reference_merk_wrapped = self - .merks - .remove(reference_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(reference_path, false)); - - let merk = cost_return_on_error!(&mut cost, reference_merk_wrapped); + let merk = match self.merks.entry(reference_path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(reference_path, false) + )), + }; let referenced_element = cost_return_on_error!( &mut cost, @@ -912,15 +906,13 @@ where let is_sum_tree = merk.is_sum_tree; - self.merks.insert(reference_path.to_vec(), merk); - if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( - &cost, - Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize element")) - }) - ); + &cost, + Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { + Error::CorruptedData(String::from("unable to deserialize element")) + }) + ); Ok(Some((element, referenced_element, is_sum_tree))).wrap_with_cost(cost) } else { @@ -977,7 +969,6 @@ where /// tree being updated, which is not allowed. fn process_reference_with_hop_count_greater_than_one<'a, G, SR>( &'a mut self, - parent_tree: &Merk, key: &[u8], reference_path: &[Vec], qualified_path: &[Vec], @@ -997,7 +988,7 @@ where { let mut cost = OperationCost::default(); - let Some((element, _, _)) = cost_return_on_error!( + let Some((element, ..)) = cost_return_on_error!( &mut cost, self.get_and_deserialize_referenced_element(key, reference_path, grove_version) ) else { @@ -1010,7 +1001,8 @@ where "reference to path:`{}` key:`{}` in batch is missing", reference_string, hex::encode(key) - ))).wrap_with_cost(cost); + ))) + .wrap_with_cost(cost); }; match element { @@ -1026,7 +1018,6 @@ where path_from_reference_qualified_path_type(path, qualified_path) ); self.follow_reference_get_value_hash( - &parent_tree, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1054,7 +1045,6 @@ where /// All these has to be taken into account. fn follow_reference_get_value_hash<'a, G, SR>( &'a mut self, - temp_merks: BTreeMap<&Vec>, &Merk>, qualified_path: &[Vec], ops_by_qualified_paths: &'a BTreeMap>, GroveOp>, recursions_allowed: u8, @@ -1069,7 +1059,6 @@ where u32, u32, ) -> Result<(StorageRemovedBytes, StorageRemovedBytes), Error>, - S: StorageContext<'db>, { let mut cost = OperationCost::default(); if recursions_allowed == 0 { @@ -1093,9 +1082,9 @@ where match element { Element::Item(..) | Element::SumItem(..) => { let serialized = cost_return_on_error_no_add!( - &cost, - element.serialize(grove_version) - ); + &cost, + element.serialize(grove_version) + ); if element.get_flags().is_none() { // There are no storage flags, we can just hash new element let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); @@ -1116,26 +1105,39 @@ where ) { let maybe_old_flags = old_element.get_flags_owned(); - let old_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths(key.len() as u32, old_serialized_element.len() as u32, is_in_sum_tree); - let new_storage_cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths(key.len() as u32, serialized.len() as u32, is_in_sum_tree); + let old_storage_cost = + KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + old_serialized_element.len() as u32, + is_in_sum_tree, + ); + let new_storage_cost = + KV::node_byte_cost_size_for_key_and_raw_value_lengths( + key.len() as u32, + serialized.len() as u32, + is_in_sum_tree, + ); // There are storage flags - let storage_costs = TreeNode::storage_cost_for_update(new_storage_cost, old_storage_cost); + let storage_costs = TreeNode::storage_cost_for_update( + new_storage_cost, + old_storage_cost, + ); let changed = cost_return_on_error_no_add!( - &cost, - (flags_update)(&storage_costs, maybe_old_flags, new_flags) - .map_err(|e| match e { - Error::JustInTimeElementFlagsClientError(_) => { - MerkError::ClientCorruptionError(e.to_string()) - .into() - } - _ => MerkError::ClientCorruptionError( - "non client error".to_string(), - ) - .into(), - }) - ); + &cost, + (flags_update)(&storage_costs, maybe_old_flags, new_flags) + .map_err(|e| match e { + Error::JustInTimeElementFlagsClientError(_) => { + MerkError::ClientCorruptionError(e.to_string()) + .into() + } + _ => MerkError::ClientCorruptionError( + "non client error".to_string(), + ) + .into(), + }) + ); if changed { // There are no storage flags, we can just hash new element let serialized = cost_return_on_error_no_add!( @@ -1168,7 +1170,6 @@ where ) ); self.follow_reference_get_value_hash( - parent_tree_at_path, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1198,7 +1199,6 @@ where path_from_reference_qualified_path_type(path.clone(), qualified_path) ); self.follow_reference_get_value_hash( - parent_tree_at_path, path.as_slice(), ops_by_qualified_paths, recursions_allowed - 1, @@ -1224,7 +1224,6 @@ where None }; self.process_reference( - &parent_tree_at_path, qualified_path, ops_by_qualified_paths, recursions_allowed, @@ -1243,7 +1242,6 @@ where } } else { self.process_reference( - parent_tree_at_path, qualified_path, ops_by_qualified_paths, recursions_allowed, @@ -1289,12 +1287,15 @@ where ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let base_path = vec![]; - let merk_wrapped = self - .merks - .remove(&base_path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(&[], false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); + + let merk = match self.merks.entry(base_path.clone()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(&base_path, false) + )), + }; + merk.set_base_root_key(root_key) .add_cost(cost) .map_err(|_| Error::InternalError("unable to set base root key".to_string())) @@ -1315,13 +1316,17 @@ where let p = path.to_path(); let path = &p; - let merk_wrapped = self - .merks - .remove(path) - .map(|x| Ok(x).wrap_with_cost(Default::default())) - .unwrap_or_else(|| (self.get_merk_fn)(path, false)); - let mut merk = cost_return_on_error!(&mut cost, merk_wrapped); - let is_sum_tree = merk.is_sum_tree; + // This also populates Merk trees cache + let is_sum_tree = { + let merk = match self.merks.entry(path.to_vec()) { + HashMapEntry::Occupied(o) => o.into_mut(), + HashMapEntry::Vacant(v) => v.insert(cost_return_on_error!( + &mut cost, + (self.get_merk_fn)(path, false) + )), + }; + merk.is_sum_tree + }; let mut batch_operations: Vec<(Vec, Op)> = vec![]; for (key_info, op) in ops_at_path_by_key.into_iter() { @@ -1356,7 +1361,6 @@ where let referenced_element_value_hash = cost_return_on_error!( &mut cost, self.follow_reference_get_value_hash( - &merk, path_reference.as_slice(), ops_by_qualified_paths, element_max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), @@ -1404,10 +1408,12 @@ where .wrap_with_cost(OperationCost::default()) ); if batch_apply_options.validate_insertion_does_not_override { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let inserted = cost_return_on_error!( &mut cost, element.insert_if_not_exists_into_batch_operations( - &mut merk, + merk, key_info.get_key(), &mut batch_operations, merk_feature_type, @@ -1445,6 +1451,7 @@ where let element = if trust_refresh_reference { Element::Reference(reference_path_type, max_reference_hop, flags) } else { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); let value = cost_return_on_error!( &mut cost, merk.get( @@ -1501,7 +1508,6 @@ where let referenced_element_value_hash = cost_return_on_error!( &mut cost, self.follow_reference_get_value_hash( - &merk, path_reference.as_slice(), ops_by_qualified_paths, max_reference_hop.unwrap_or(MAX_REFERENCE_HOPS as u8), @@ -1564,6 +1570,7 @@ where root_key, sum, } => { + let merk = self.merks.get_mut(path).expect("the Merk is cached"); cost_return_on_error!( &mut cost, GroveDb::update_tree_item_preserve_flag_into_batch_operations( @@ -1606,6 +1613,9 @@ where } } } + + let merk = self.merks.get_mut(path).expect("the Merk is cached"); + cost_return_on_error!( &mut cost, merk.apply_unchecked::<_, Vec, _, _, _, _>( @@ -1699,8 +1709,7 @@ where .root_hash_key_and_sum() .add_cost(cost) .map_err(Error::MerkError); - // We need to reinsert the merk - self.merks.insert(path.clone(), merk); + r } @@ -1902,8 +1911,7 @@ impl GroveDb { ops_at_level_above.insert(parent_path, ops_on_path); } } else { - let mut ops_on_path: BTreeMap = - BTreeMap::new(); + let mut ops_on_path: BTreeMap = BTreeMap::new(); ops_on_path.insert( key.clone(), GroveOp::ReplaceTreeRootKey { @@ -2179,8 +2187,14 @@ impl GroveDb { Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( |_| { Error::InvalidPath(format!( - "could not get key for parent of subtree for batch at path [{}] for key {}", - parent_path.to_vec().into_iter().map(|v| hex_to_ascii(&v)).join("/"), hex_to_ascii(parent_key) + "could not get key for parent of subtree for batch at path [{}] \ + for key {}", + parent_path + .to_vec() + .into_iter() + .map(|v| hex_to_ascii(&v)) + .join("/"), + hex_to_ascii(parent_key) )) } ) @@ -2796,7 +2810,11 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), @@ -2983,7 +3001,11 @@ mod tests { let element = Element::new_item(b"ayy".to_vec()); let element2 = Element::new_item(b"ayy2".to_vec()); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), @@ -3526,7 +3548,11 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), @@ -3564,7 +3590,11 @@ mod tests { b"key2".to_vec(), element.clone(), ), - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), @@ -3642,7 +3672,11 @@ mod tests { let db = make_test_grovedb(grove_version); let element = Element::new_item(b"ayy".to_vec()); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()], b"key4".to_vec(), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index c48a2fff..f684acfb 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -3,14 +3,17 @@ #[cfg(feature = "full")] mod tests { use std::{ops::Add, option::Option::None}; - use integer_encoding::VarInt; + use grovedb_costs::{ - storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, + storage_cost::{ + removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, + transition::OperationStorageTransitionType, + StorageCost, + }, OperationCost, }; - use grovedb_costs::storage_cost::removal::StorageRemovedBytes::BasicStorageRemoval; - use grovedb_costs::storage_cost::transition::OperationStorageTransitionType; use grovedb_version::version::GroveVersion; + use integer_encoding::VarInt; use crate::{ batch::QualifiedGroveDbOp, @@ -48,8 +51,16 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2); tx.rollback().expect("expected to rollback"); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost = db.apply_batch(ops, None, Some(&tx), grove_version).cost; assert_eq!( @@ -99,7 +110,11 @@ mod tests { let non_batch_cost = non_batch_cost_1.add(non_batch_cost_2).add(non_batch_cost_3); tx.rollback().expect("expected to rollback"); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![], b"key2".to_vec(), @@ -175,7 +190,11 @@ mod tests { .add(non_batch_cost_4); tx.rollback().expect("expected to rollback"); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), @@ -214,8 +233,16 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key2".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key2".to_vec(), + Element::empty_tree(), + ), ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); @@ -270,7 +297,11 @@ mod tests { let tx = db.start_transaction(); let ops = vec![ - QualifiedGroveDbOp::insert_or_replace_op(vec![], b"key1".to_vec(), Element::empty_tree()), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::empty_tree(), + ), QualifiedGroveDbOp::insert_or_replace_op( vec![b"key1".to_vec()], b"key2".to_vec(), @@ -338,23 +369,30 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert tree"); + .unwrap() + .expect("expected to insert tree"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"tree2".to_vec(), - Element::empty_tree(), - ), QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec(), b"tree2".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::insert_only_op( - vec![b"tree".to_vec(), b"tree2".to_vec()], - b"keyref".to_vec(), - Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"tree2".to_vec(), + Element::empty_tree(), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec(), b"tree2".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; let cost = db .apply_batch_with_element_flags_update( @@ -389,7 +427,9 @@ mod tests { }, Some(&tx), grove_version, - ).cost_as_result().expect("expect no error"); + ) + .cost_as_result() + .expect("expect no error"); // Hash node calls diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index d4c5d844..e2b3d9a9 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -72,7 +72,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -212,7 +216,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() @@ -357,7 +365,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -452,7 +464,11 @@ mod tests { )); tx.rollback().expect("expected to rollback"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch_with_element_flags_update( ops, @@ -623,7 +639,11 @@ mod tests { .cost_as_result() .expect("expected to insert successfully"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, None, grove_version) .cost_as_result() diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index ba32bbf5..b74768d7 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -21,10 +21,10 @@ mod tests { use crate::{ batch::QualifiedGroveDbOp, + reference_path::ReferencePathType::SiblingReference, tests::{common::EMPTY_PATH, make_empty_grovedb}, Element, }; - use crate::reference_path::ReferencePathType::SiblingReference; #[test] fn test_batch_one_insert_costs_match_non_batch() { @@ -872,8 +872,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert tree"); + .unwrap() + .expect("expected to insert tree"); db.insert( [b"tree".as_slice()].as_ref(), @@ -883,8 +883,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert item"); + .unwrap() + .expect("expected to insert item"); db.insert( [b"tree".as_slice()].as_ref(), @@ -894,19 +894,25 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert item"); + .unwrap() + .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::replace_op( - vec![b"tree".to_vec()], - b"keyref".to_vec(), - Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; let cost = db .apply_batch_with_element_flags_update( @@ -990,8 +996,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert tree"); + .unwrap() + .expect("expected to insert tree"); db.insert( [b"tree".as_slice()].as_ref(), @@ -1001,19 +1007,25 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert item"); + .unwrap() + .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::insert_only_op( - vec![b"tree".to_vec()], - b"keyref".to_vec(), - Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value100".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; let cost = db .apply_batch_with_element_flags_update( @@ -1252,8 +1264,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert tree"); + .unwrap() + .expect("expected to insert tree"); db.insert( [b"tree".as_slice()].as_ref(), @@ -1263,8 +1275,8 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert item"); + .unwrap() + .expect("expected to insert item"); db.insert( [b"tree".as_slice()].as_ref(), @@ -1274,20 +1286,25 @@ mod tests { None, grove_version, ) - .unwrap() - .expect("expected to insert item"); - + .unwrap() + .expect("expected to insert item"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![b"tree".to_vec()], - b"key1".to_vec(), - Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::replace_op( - vec![b"tree".to_vec()], - b"keyref".to_vec(), - Element::new_reference_with_flags(SiblingReference(b"key1".to_vec()), Some(vec![0, 1])), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![b"tree".to_vec()], + b"key1".to_vec(), + Element::new_item_with_flags(b"value".to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![b"tree".to_vec()], + b"keyref".to_vec(), + Element::new_reference_with_flags( + SiblingReference(b"key1".to_vec()), + Some(vec![0, 1]), + ), + ), + ]; let cost = db .apply_batch_with_element_flags_update( @@ -1443,33 +1460,40 @@ mod tests { } #[test] - fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_refresh_reference() { + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_refresh_reference() + { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![], - b"key1".to_vec(), - Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), - ), QualifiedGroveDbOp::insert_or_replace_op( - vec![], - b"keyref".to_vec(), - Element::new_reference(SiblingReference(b"key1".to_vec())), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 56].to_vec(), Some(vec![0, 0])), + ), + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![], - b"key1".to_vec(), - Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::replace_op( - vec![], - b"keyref".to_vec(), - Element::new_reference(SiblingReference(b"key1".to_vec())), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::replace_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; let cost = db .apply_batch_with_element_flags_update( @@ -1537,7 +1561,8 @@ mod tests { } #[test] - fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_new_reference() { + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_new_reference( + ) { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -1551,15 +1576,18 @@ mod tests { let cost_result = db.apply_batch(ops, None, Some(&tx), grove_version); cost_result.value.expect("expected to execute batch"); // We are adding 2 bytes - let ops = vec![QualifiedGroveDbOp::insert_or_replace_op( - vec![], - b"key1".to_vec(), - Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), - ), QualifiedGroveDbOp::insert_only_op( - vec![], - b"keyref".to_vec(), - Element::new_reference(SiblingReference(b"key1".to_vec())), - )]; + let ops = vec![ + QualifiedGroveDbOp::insert_or_replace_op( + vec![], + b"key1".to_vec(), + Element::new_item_with_flags([0u8; 57].to_vec(), Some(vec![0, 1])), + ), + QualifiedGroveDbOp::insert_only_op( + vec![], + b"keyref".to_vec(), + Element::new_reference(SiblingReference(b"key1".to_vec())), + ), + ]; let cost = db .apply_batch_with_element_flags_update( diff --git a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs index fad794f4..c0be9a08 100644 --- a/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_sum_item_deletion_cost_tests.rs @@ -43,7 +43,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() @@ -146,7 +150,11 @@ mod tests { ); tx.rollback().expect("expected to rollback"); - let ops = vec![QualifiedGroveDbOp::delete_tree_op(vec![], b"key1".to_vec(), false)]; + let ops = vec![QualifiedGroveDbOp::delete_tree_op( + vec![], + b"key1".to_vec(), + false, + )]; let batch_cost = db .apply_batch(ops, None, Some(&tx), grove_version) .cost_as_result() diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index a95cc7ca..986f2b90 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -17,7 +17,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, QualifiedGroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, Error, GroveDb, }; @@ -188,6 +188,10 @@ impl GroveDb { estimated_key_element_size.0 + HASH_LENGTH_U32, ); - Ok(QualifiedGroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 1181fc53..9244c60b 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -33,7 +33,7 @@ use grovedb_version::{ #[cfg(feature = "full")] use crate::{ - batch::{QualifiedGroveDbOp, GroveOp}, + batch::{GroveOp, QualifiedGroveDbOp}, util::storage_context_with_parent_optional_tx, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -622,7 +622,11 @@ impl GroveDb { }; result.wrap_with_cost(cost) } else { - Ok(Some(QualifiedGroveDbOp::delete_op(path.to_vec(), key.to_vec()))).wrap_with_cost(cost) + Ok(Some(QualifiedGroveDbOp::delete_op( + path.to_vec(), + key.to_vec(), + ))) + .wrap_with_cost(cost) } } } diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 6b40d527..8533cde5 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -13,7 +13,7 @@ use grovedb_version::{ use intmap::IntMap; use crate::{ - batch::{key_info::KeyInfo, QualifiedGroveDbOp, KeyInfoPath}, + batch::{key_info::KeyInfo, KeyInfoPath, QualifiedGroveDbOp}, element::SUM_TREE_COST_SIZE, Error, GroveDb, }; @@ -165,6 +165,10 @@ impl GroveDb { // in the worst case this is a tree add_worst_case_cost_for_is_empty_tree_except(&mut cost, except_keys_count); - Ok(QualifiedGroveDbOp::delete_estimated_op(path.clone(), key.clone())).wrap_with_cost(cost) + Ok(QualifiedGroveDbOp::delete_estimated_op( + path.clone(), + key.clone(), + )) + .wrap_with_cost(cost) } } diff --git a/merk/benches/merk.rs b/merk/benches/merk.rs index 46f83400..62bc6ebb 100644 --- a/merk/benches/merk.rs +++ b/merk/benches/merk.rs @@ -38,8 +38,8 @@ use grovedb_merk::{ }; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; -use rand::prelude::*; use grovedb_version::version::GroveVersion; +use rand::prelude::*; /// 1 million gets in 2k batches pub fn get(c: &mut Criterion) { diff --git a/merk/src/merk/cow_like.rs b/merk/src/merk/cow_like.rs deleted file mode 100644 index 50539305..00000000 --- a/merk/src/merk/cow_like.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Module for [CowLikeMerk]: simple abstraction over owned and borrowed Merk. - -use std::{ - hash::{Hash, Hasher}, - ops::Deref, -}; -use std::ops::DerefMut; -use grovedb_storage::StorageContext; -use crate::Merk; - -/// A smart pointer that follows the semantics of [Cow](std::borrow::Cow) except -/// provides no means for mutability and thus doesn't require [Clone]. -#[derive(Debug)] -pub enum CowLikeMerk<'db, S> { - /// Owned variant - Owned(Merk), - /// Borrowed variant - Borrowed(&'db mut Merk), -} - -impl<'db, S: StorageContext<'db>> Deref for CowLikeMerk<'db, S> { - type Target = Merk; - - fn deref(&self) -> &Self::Target { - match self { - Self::Owned(v) => v, - Self::Borrowed(s) => s, - } - } -} - -impl<'db, S: StorageContext<'db>> DerefMut for CowLikeMerk<'db, S> { - fn deref_mut(&mut self) -> &mut Self::Target { - match self { - Self::Owned(v) => v, - Self::Borrowed(s) => s, - } - } -} \ No newline at end of file diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 73e39435..ee0deccc 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -41,7 +41,6 @@ pub mod open; pub mod prove; pub mod restore; pub mod source; -pub mod cow_like; use std::{ cell::Cell, diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index fb116dac..dc0cbaf2 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -158,8 +158,7 @@ impl TreeNode { self.inner.kv.feature_type.is_sum_feature() } - pub fn storage_cost_for_update( current_value_byte_cost: u32, - old_cost: u32) -> StorageCost { + pub fn storage_cost_for_update(current_value_byte_cost: u32, old_cost: u32) -> StorageCost { let mut value_storage_cost = StorageCost { ..Default::default() }; From 47845a7219394f224081f0efcdb674ce6c35e20e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 11:22:43 +0700 Subject: [PATCH 13/19] fmt fixes --- grovedb/build.rs | 1 + grovedb/src/batch/just_in_time_cost_tests.rs | 12 +++----- grovedb/src/batch/mod.rs | 26 +++++++++------- grovedb/src/batch/multi_insert_cost_tests.rs | 11 ++++--- grovedb/src/batch/single_insert_cost_tests.rs | 30 +++++++++---------- 5 files changed, 41 insertions(+), 39 deletions(-) diff --git a/grovedb/build.rs b/grovedb/build.rs index 83a38900..d2185353 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -12,6 +12,7 @@ fn main() { let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); + #[rustfmt::skip] if !grovedbg_zip_path.exists() { let response = reqwest::blocking::get(format!("https://github.com/dashpay/grovedbg/releases/download/{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip")) .expect("can't download GroveDBG artifact"); diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 9c990d1b..860abacf 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -5,13 +5,9 @@ mod tests { use std::option::Option::None; - use grovedb_costs::{ - storage_cost::{ - removal::StorageRemovedBytes::{BasicStorageRemoval, NoStorageRemoval}, - transition::OperationStorageTransitionType, - StorageCost, - }, - OperationCost, + use grovedb_costs::storage_cost::{ + removal::StorageRemovedBytes::BasicStorageRemoval, + transition::OperationStorageTransitionType, }; use grovedb_version::version::GroveVersion; use integer_encoding::VarInt; @@ -381,7 +377,7 @@ mod tests { ), ]; - let cost = db + let _ = db .apply_batch_with_element_flags_update( ops, None, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 77a4fbc0..92dcbe2f 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -63,7 +63,7 @@ use grovedb_storage::{ Storage, StorageBatch, StorageContext, }; use grovedb_version::{ - check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, TryIntoVersioned, + check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; @@ -652,10 +652,13 @@ impl QualifiedGroveDbOp { /// Results of a consistency check on an operation batch #[derive(Debug)] pub struct GroveDbOpConsistencyResults { - repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, // the u16 is count + /// Repeated Ops, the second u16 element represents the count + repeated_ops: Vec<(QualifiedGroveDbOp, u16)>, + /// The same path key ops same_path_key_ops: Vec<(KeyInfoPath, KeyInfo, Vec)>, - insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, /* the deleted op first, - * then inserts under */ + /// This shows issues when we delete a tree but insert under the deleted + /// tree Deleted ops are first, with inserts under them in a tree + insert_ops_below_deleted_ops: Vec<(QualifiedGroveDbOp, Vec)>, } impl GroveDbOpConsistencyResults { @@ -1091,18 +1094,21 @@ where Ok(val_hash).wrap_with_cost(cost) } else { let mut new_element = element.clone(); - let mut new_flags = new_element.get_flags_mut().as_mut().unwrap(); + let new_flags = new_element.get_flags_mut().as_mut().unwrap(); // it can be unmerged, let's get the value on disk - let (key, reference_path) = qualified_path.split_last().unwrap(); // already checked - if let Some((old_element, old_serialized_element, is_in_sum_tree)) = cost_return_on_error!( + let (key, reference_path) = qualified_path.split_last().unwrap(); + let serialized_element_result = cost_return_on_error!( &mut cost, self.get_and_deserialize_referenced_element( key, reference_path, grove_version ) - ) { + ); + if let Some((old_element, old_serialized_element, is_in_sum_tree)) = + serialized_element_result + { let maybe_old_flags = old_element.get_flags_owned(); let old_storage_cost = @@ -1451,7 +1457,7 @@ where let element = if trust_refresh_reference { Element::Reference(reference_path_type, max_reference_hop, flags) } else { - let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let merk = self.merks.get(path).expect("the Merk is cached"); let value = cost_return_on_error!( &mut cost, merk.get( @@ -1570,7 +1576,7 @@ where root_key, sum, } => { - let merk = self.merks.get_mut(path).expect("the Merk is cached"); + let merk = self.merks.get(path).expect("the Merk is cached"); cost_return_on_error!( &mut cost, GroveDb::update_tree_item_preserve_flag_into_batch_operations( diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index f684acfb..8b0a5721 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -372,7 +372,6 @@ mod tests { .unwrap() .expect("expected to insert tree"); - // We are adding 2 bytes let ops = vec![ QualifiedGroveDbOp::insert_or_replace_op( vec![b"tree".to_vec()], @@ -438,14 +437,14 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 10, // todo: verify this + seek_count: 8, // todo: verify this storage_cost: StorageCost { - added_bytes: 163, - replaced_bytes: 196, // todo: verify this + added_bytes: 430, + replaced_bytes: 78, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 393, // todo: verify this - hash_node_calls: 17, // todo: verify this + storage_loaded_bytes: 152, // todo: verify this + hash_node_calls: 22, // todo: verify this } ); diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index b74768d7..b9fb1573 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -957,14 +957,14 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 11, // todo: verify this + seek_count: 9, // todo: verify this storage_cost: StorageCost { added_bytes: 4, replaced_bytes: 316, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 556, // todo: verify this - hash_node_calls: 17, // todo: verify this + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this } ); @@ -1070,14 +1070,14 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 10, // todo: verify this + seek_count: 8, // todo: verify this storage_cost: StorageCost { added_bytes: 163, replaced_bytes: 196, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 393, // todo: verify this - hash_node_calls: 17, // todo: verify this + storage_loaded_bytes: 236, // todo: verify this + hash_node_calls: 16, // todo: verify this } ); @@ -1350,14 +1350,14 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 11, // todo: verify this + seek_count: 9, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 315, // todo: verify this removed_bytes: SectionedStorageRemoval(removed_bytes) }, - storage_loaded_bytes: 556, // todo: verify this - hash_node_calls: 17, // todo: verify this + storage_loaded_bytes: 357, // todo: verify this + hash_node_calls: 16, // todo: verify this } ); @@ -1534,13 +1534,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 9, // todo: verify this + seek_count: 7, // todo: verify this storage_cost: StorageCost { added_bytes: 4, replaced_bytes: 285, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 502, // todo: verify this + storage_loaded_bytes: 380, // todo: verify this hash_node_calls: 12, // todo: verify this } ); @@ -1561,8 +1561,8 @@ mod tests { } #[test] - fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_new_reference( - ) { + fn test_batch_root_one_update_cost_right_above_value_required_cost_of_2_with_insert_reference() + { let grove_version = GroveVersion::latest(); let db = make_empty_grovedb(); let tx = db.start_transaction(); @@ -1628,13 +1628,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, // todo: verify this + seek_count: 5, // todo: verify this storage_cost: StorageCost { added_bytes: 160, replaced_bytes: 168, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 266, // todo: verify this + storage_loaded_bytes: 133, // todo: verify this hash_node_calls: 12, // todo: verify this } ); From 354a66ccc0a6c87d1f9994b3517e7695bbf6b162 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 12:22:51 +0700 Subject: [PATCH 14/19] removed verify on each batch --- grovedb/src/batch/mod.rs | 54 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 92dcbe2f..c4dc6bf1 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -2402,19 +2402,20 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self - .visualize_verify_grovedb(Some(tx), true, &Default::default()) - .unwrap(); - if issues.len() > 0 { - println!( - "tx_issues: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") - ); - } + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(Some(tx), true, + // &Default::default()) .unwrap(); + // if issues.len() > 0 { + // println!( + // "tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } else { cost_return_on_error!( &mut cost, @@ -2443,19 +2444,20 @@ impl GroveDb { .map_err(|e| e.into()) ); - let issues = self - .visualize_verify_grovedb(None, true, &Default::default()) - .unwrap(); - if issues.len() > 0 { - println!( - "non_tx_issues: {}", - issues - .iter() - .map(|(hash, (a, b, c))| format!("{}: {} {} {}", hash, a, b, c)) - .collect::>() - .join(" | ") - ); - } + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(None, true, &Default::default()) + // .unwrap(); + // if issues.len() > 0 { + // println!( + // "non_tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } } Ok(()).wrap_with_cost(cost) } From c4237990f61a29a4cdbfe57ac873d55516bb4e01 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 14:26:12 +0700 Subject: [PATCH 15/19] trying cache true --- grovedb/src/lib.rs | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 8c010e28..59f7cbe3 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -216,6 +216,7 @@ pub use crate::error::Error; use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; #[cfg(feature = "full")] use crate::Error::MerkError; +use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] type Hash = [u8; 32]; @@ -974,7 +975,8 @@ impl GroveDb { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; + let allow_cache = true; + let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -985,14 +987,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) ))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1027,14 +1029,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) ))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { @@ -1054,14 +1056,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for reference", hex_to_ascii(&key)) ))?; let referenced_value_hash = { @@ -1073,7 +1075,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - false, + allow_cache, None, grove_version, ) @@ -1112,7 +1114,8 @@ impl GroveDb { let mut all_query = Query::new(); all_query.insert_all(); - let _in_sum_tree = merk.is_sum_tree; + let allow_cache = true; + let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -1123,14 +1126,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) ))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1167,14 +1170,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) ))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { @@ -1194,14 +1197,14 @@ impl GroveDb { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, - false, + allow_cache, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) .unwrap() .map_err(MerkError)? .ok_or(Error::CorruptedData( - "expected merk to contain value at key".to_string(), + format!("expected merk to contain value at key {} for reference", hex_to_ascii(&key)) ))?; let referenced_value_hash = { @@ -1213,7 +1216,7 @@ impl GroveDb { let item = self .follow_reference( (full_path.as_slice()).into(), - false, + allow_cache, Some(transaction), grove_version, ) From df57853f13cba32515b6eae80bfa23752d5fc914 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 14:38:59 +0700 Subject: [PATCH 16/19] fmt --- grovedb/src/lib.rs | 48 ++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 59f7cbe3..a634d52b 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -212,11 +212,11 @@ use tokio::net::ToSocketAddrs; use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] pub use crate::error::Error; +use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; #[cfg(feature = "full")] use crate::Error::MerkError; -use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] type Hash = [u8; 32]; @@ -993,9 +993,11 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1035,9 +1037,11 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { issues.insert( @@ -1062,9 +1066,10 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for reference", hex_to_ascii(&key)) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; let referenced_value_hash = { let full_path = path_from_reference_path_type( @@ -1132,9 +1137,11 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let new_path = path.derive_owned_with_child(key); let new_path_ref = SubtreePath::from(&new_path); @@ -1176,9 +1183,11 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for {}", hex_to_ascii(&key), element.type_str()) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for {}", + hex_to_ascii(&key), + element.type_str() + )))?; let actual_value_hash = value_hash(&kv_value).unwrap(); if actual_value_hash != element_value_hash { issues.insert( @@ -1203,9 +1212,10 @@ impl GroveDb { ) .unwrap() .map_err(MerkError)? - .ok_or(Error::CorruptedData( - format!("expected merk to contain value at key {} for reference", hex_to_ascii(&key)) - ))?; + .ok_or(Error::CorruptedData(format!( + "expected merk to contain value at key {} for reference", + hex_to_ascii(&key) + )))?; let referenced_value_hash = { let full_path = path_from_reference_path_type( From d7f01a1fff3698263ee3d93717f01f863175d2b0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 14:50:31 +0700 Subject: [PATCH 17/19] fix clippy warning --- grovedb/build.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/grovedb/build.rs b/grovedb/build.rs index d2185353..45be0d23 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -12,9 +12,11 @@ fn main() { let out_dir = PathBuf::from(&env::var_os("OUT_DIR").unwrap()); let grovedbg_zip_path = out_dir.join("grovedbg.zip"); - #[rustfmt::skip] if !grovedbg_zip_path.exists() { - let response = reqwest::blocking::get(format!("https://github.com/dashpay/grovedbg/releases/download/{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip")) + let response = reqwest::blocking::get(format!( + "https://github.com/dashpay/grovedbg/releases/download/\ + {GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip" + )) .expect("can't download GroveDBG artifact"); let mut grovedbg_zip = File::create(&grovedbg_zip_path).unwrap(); From b0a360f5ac817bf1a30e36833d37668b55fc9af0 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 15:04:15 +0700 Subject: [PATCH 18/19] added option to allow cache to visualize_verify_grovedb --- grovedb/src/batch/just_in_time_cost_tests.rs | 2 +- grovedb/src/batch/multi_insert_cost_tests.rs | 2 +- grovedb/src/batch/single_insert_cost_tests.rs | 10 +++++----- grovedb/src/lib.rs | 14 +++++++++----- grovedb/src/tests/mod.rs | 15 ++++++++++----- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/grovedb/src/batch/just_in_time_cost_tests.rs b/grovedb/src/batch/just_in_time_cost_tests.rs index 860abacf..a3fa9aec 100644 --- a/grovedb/src/batch/just_in_time_cost_tests.rs +++ b/grovedb/src/batch/just_in_time_cost_tests.rs @@ -414,7 +414,7 @@ mod tests { .cost; let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), diff --git a/grovedb/src/batch/multi_insert_cost_tests.rs b/grovedb/src/batch/multi_insert_cost_tests.rs index 8b0a5721..8a1200e6 100644 --- a/grovedb/src/batch/multi_insert_cost_tests.rs +++ b/grovedb/src/batch/multi_insert_cost_tests.rs @@ -449,7 +449,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), diff --git a/grovedb/src/batch/single_insert_cost_tests.rs b/grovedb/src/batch/single_insert_cost_tests.rs index b9fb1573..2a269159 100644 --- a/grovedb/src/batch/single_insert_cost_tests.rs +++ b/grovedb/src/batch/single_insert_cost_tests.rs @@ -969,7 +969,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), @@ -1082,7 +1082,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), @@ -1362,7 +1362,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), @@ -1546,7 +1546,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), @@ -1640,7 +1640,7 @@ mod tests { ); let issues = db - .visualize_verify_grovedb(Some(&tx), true, &Default::default()) + .visualize_verify_grovedb(Some(&tx), true, false, &Default::default()) .unwrap(); assert_eq!( issues.len(), diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index a634d52b..feef0721 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -902,10 +902,11 @@ impl GroveDb { &self, transaction: TransactionArg, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result, Error> { Ok(self - .verify_grovedb(transaction, verify_references, grove_version)? + .verify_grovedb(transaction, verify_references, allow_cache, grove_version)? .iter() .map(|(path, (root_hash, expected, actual))| { ( @@ -929,6 +930,7 @@ impl GroveDb { &self, transaction: TransactionArg, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { if let Some(transaction) = transaction { @@ -946,6 +948,7 @@ impl GroveDb { None, transaction, verify_references, + allow_cache, grove_version, ) } else { @@ -957,6 +960,7 @@ impl GroveDb { &SubtreePath::empty(), None, verify_references, + allow_cache, grove_version, ) } @@ -970,13 +974,12 @@ impl GroveDb { path: &SubtreePath, batch: Option<&'db StorageBatch>, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let allow_cache = true; - let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -1024,6 +1027,7 @@ impl GroveDb { &new_path_ref, batch, verify_references, + true, grove_version, )?); } @@ -1114,13 +1118,12 @@ impl GroveDb { batch: Option<&'db StorageBatch>, transaction: &Transaction, verify_references: bool, + allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { let mut all_query = Query::new(); all_query.insert_all(); - let allow_cache = true; - let mut issues = HashMap::new(); let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); @@ -1170,6 +1173,7 @@ impl GroveDb { batch, transaction, verify_references, + true, grove_version, )?); } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index ec3cdbfa..41263669 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3990,7 +3990,12 @@ mod tests { )); // `verify_grovedb` must identify issues - assert!(db.verify_grovedb(None, true, grove_version).unwrap().len() > 0); + assert!( + db.verify_grovedb(None, true, false, grove_version) + .unwrap() + .len() + > 0 + ); } #[test] @@ -4043,7 +4048,7 @@ mod tests { .unwrap(); assert!(db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); @@ -4060,7 +4065,7 @@ mod tests { .unwrap(); assert!(!db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); } @@ -4098,7 +4103,7 @@ mod tests { .unwrap(); assert!(db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); @@ -4115,7 +4120,7 @@ mod tests { .unwrap(); assert!(!db - .verify_grovedb(None, true, grove_version) + .verify_grovedb(None, true, false, grove_version) .unwrap() .is_empty()); } From 2393272aecc5c66d70312b807b277587afe730f6 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Tue, 20 Aug 2024 17:07:52 +0700 Subject: [PATCH 19/19] fix --- grovedb/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grovedb/build.rs b/grovedb/build.rs index 45be0d23..1d5fa25b 100644 --- a/grovedb/build.rs +++ b/grovedb/build.rs @@ -15,7 +15,7 @@ fn main() { if !grovedbg_zip_path.exists() { let response = reqwest::blocking::get(format!( "https://github.com/dashpay/grovedbg/releases/download/\ - {GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip" +{GROVEDBG_VERSION}/grovedbg-{GROVEDBG_VERSION}.zip" )) .expect("can't download GroveDBG artifact");