From 87ec8c3ac77a4ff0292f95d63f2d22139d2322ab Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sun, 28 Jul 2024 15:10:16 +0700 Subject: [PATCH 01/27] start of work --- grovedb/src/batch/mod.rs | 12 +++--- grovedb/src/element/constructor.rs | 25 +++++++++++ grovedb/src/element/get.rs | 4 +- grovedb/src/element/helpers.rs | 12 ++++-- grovedb/src/element/mod.rs | 60 ++++++++++++++++++++++++++ grovedb/src/operations/get/query.rs | 14 +++--- grovedb/src/operations/proof/verify.rs | 2 + grovedb/src/visualize.rs | 19 ++++++++ 8 files changed, 130 insertions(+), 18 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7f9c119e..228bd876 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -828,13 +828,13 @@ where ); match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) | Element::SumItem(..) | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( &cost, path_from_reference_qualified_path_type(path, qualified_path) @@ -886,7 +886,7 @@ where .wrap_with_cost(cost), Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) | Element::SumItem(..) | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!( &cost, element.serialize(grove_version) @@ -894,7 +894,7 @@ where let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( &cost, path_from_reference_qualified_path_type( @@ -1027,7 +1027,7 @@ where match op { Op::Insert { element } | Op::Replace { element } | Op::Patch { element, .. } => { match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { + Element::Reference(path_reference, element_max_reference_hop, _) | Element::BidirectionalReference(path_reference, element_max_reference_hop) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1090,7 +1090,7 @@ where ) ); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) | Element::SumItem(..) | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 91143ec8..04cd5523 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -40,24 +40,49 @@ impl Element { Element::Item(item_value, None) } + #[cfg(feature = "full")] + /// Set element to an item without flags that allows for a backwards reference + pub fn new_item_allowing_bidirectional_references(item_value: Vec) -> Self { + Element::ItemWithBackwardsReferences(item_value, vec![], None) + } + #[cfg(feature = "full")] /// Set element to an item with flags pub fn new_item_with_flags(item_value: Vec, flags: Option) -> Self { Element::Item(item_value, flags) } + #[cfg(feature = "full")] + /// Set element to an item with flags and that allows for a backwards reference + pub fn new_item_allowing_bidirectional_references_with_flags(item_value: Vec, flags: Option) -> Self { + Element::ItemWithBackwardsReferences(item_value, vec![], flags) + } + #[cfg(feature = "full")] /// Set element to a sum item without flags pub fn new_sum_item(value: i64) -> Self { Element::SumItem(value, None) } + #[cfg(feature = "full")] + /// Set element to a sum item hat allows for backwards references without flags + pub fn new_sum_item_allowing_bidirectional_references(value: i64) -> Self { + Element::SumItemWithBackwardsReferences(value, vec![], None) + } + #[cfg(feature = "full")] /// Set element to a sum item with flags pub fn new_sum_item_with_flags(value: i64, flags: Option) -> Self { Element::SumItem(value, flags) } + #[cfg(feature = "full")] + /// Set element to a sum item hat allows for backwards references with flags + pub fn new_sum_item_allowing_bidirectional_references_with_flags(value: i64, flags: Option) -> Self { + Element::SumItemWithBackwardsReferences(value, vec![], flags) + } + + #[cfg(feature = "full")] /// Set element to a reference without flags pub fn new_reference(reference_path: ReferencePathType) -> Self { diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1fda91dd..3dc5d392 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -156,7 +156,7 @@ impl Element { .transpose() ); match &element { - Some(Element::Item(..)) | Some(Element::Reference(..)) => { + Some(Element::Item(..)) | Some(Element::Reference(..)) | Some(Element::ItemWithBackwardsReferences(..)) | Some(Element::BidirectionalReference(..)) => { // while the loaded item might be a sum item, it is given for free // as it would be very hard to know in advance cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( @@ -165,7 +165,7 @@ impl Element { false, ) } - Some(Element::SumItem(_, flags)) => { + Some(Element::SumItem(_, flags)) | Some(Element::SumItemWithBackwardsReferences(_, _, flags)) => { let cost_size = SUM_ITEM_COST_SIZE; let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 2d2db076..0ce311af 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -161,7 +161,9 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, _, flags) + | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, } } @@ -173,7 +175,9 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, _, flags) + | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, } } @@ -185,7 +189,9 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, _, flags) + | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, } } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index ad2540e3..ce221c25 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -62,6 +62,10 @@ pub const SUM_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 /// int 64 sum value pub type SumValue = i64; +#[cfg(any(feature = "full", feature = "verify"))] +/// if the item is deleted or updated should we cascade +pub type CascadeOnUpdate = bool; + #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// @@ -82,6 +86,12 @@ pub enum Element { /// Same as Element::Tree but underlying Merk sums value of it's summable /// nodes SumTree(Option>, SumValue, Option), + /// A reference to an object by its path + BidirectionalReference(ReferencePathType, MaxReferenceHop, Option), + /// An ordinary value that has a backwards reference + ItemWithBackwardsReferences(Vec, Vec<(ReferencePathType, CascadeOnUpdate)>, Option), + /// Signed integer value that can be totaled in a sum tree that has a backwards reference + SumItemWithBackwardsReferences(SumValue, Vec<(ReferencePathType, CascadeOnUpdate)>, Option), } impl fmt::Display for Element { @@ -108,6 +118,17 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::BidirectionalReference(path, max_hop, flags) => { + write!( + f, + "BidirectionalReference({}, max_hop: {}{})", + path, + max_hop.map_or("None".to_string(), |h| h.to_string()), + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } Element::Tree(root_key, flags) => { write!( f, @@ -139,6 +160,42 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } + Element::ItemWithBackwardsReferences(data, backwards_references, flags) => { + let backwards_references_count = backwards_references.len(); + let backwards_references_info = if backwards_references_count < 4 { + backwards_references.iter().map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|") + } else { + let first_backwards_references = backwards_references.iter().take(3).map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|"); + format!("{}...{} total backwards references", first_backwards_references, backwards_references_count) + }; + write!( + f, + "ItemWithBackwardsReferences({}, backwards_references: {}{})", + hex_to_ascii(data), + backwards_references_info, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } + Element::SumItemWithBackwardsReferences(sum_value, backwards_references, flags) => { + let backwards_references_count = backwards_references.len(); + let backwards_references_info = if backwards_references_count < 4 { + backwards_references.iter().map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|") + } else { + let first_backwards_references = backwards_references.iter().take(3).map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|"); + format!("{}...{} total backwards references", first_backwards_references, backwards_references_count) + }; + write!( + f, + "SumItemWithBackwardsReferences({}, backwards_references: {}{})", + sum_value, + backwards_references_info, + flags + .as_ref() + .map_or(String::new(), |f| format!(", flags: {:?}", f)) + ) + } } } } @@ -151,6 +208,9 @@ impl Element { Element::Tree(..) => "tree", Element::SumItem(..) => "sum item", Element::SumTree(..) => "sum tree", + Element::BidirectionalReference(..) => "bidirectional reference", + Element::ItemWithBackwardsReferences(..) => "item with backwards references", + Element::SumItemWithBackwardsReferences(..) => "sum item with backwards references", } } } diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 81046dbf..63d9587f 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -222,7 +222,7 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) => Ok(element), + Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -339,8 +339,8 @@ where { )), } } - Element::Item(item, _) => Ok(item), - Element::SumItem(item, _) => Ok(item.encode_var_vec()), + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => Ok(item), + Element::SumItem(item, _) | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item.encode_var_vec()), Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), @@ -432,8 +432,8 @@ where { )), } } - Element::Item(item, _) => Ok(QueryItemOrSumReturnType::ItemData(item)), - Element::SumItem(sum_value, _) => { + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => Ok(QueryItemOrSumReturnType::ItemData(item)), + Element::SumItem(sum_value, _) | Element::SumItemWithBackwardsReferences(sum_value, ..) => { Ok(QueryItemOrSumReturnType::SumValue(sum_value)) } Element::SumTree(_, sum_value, _) => { @@ -519,8 +519,8 @@ where { )), } } - Element::SumItem(item, _) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::Item(..) => { + Element::SumItem(item, _) | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item), + Element::Tree(..) | Element::SumTree(..) | Element::Item(..) | Element::ItemWithBackwardsReferences(..) => { Err(Error::InvalidQuery( "path_queries over sum items can only refer to sum items and \ references", diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 31cd9ae5..5c33220f 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -338,7 +338,9 @@ impl GroveDb { Element::Tree(None, _) | Element::SumTree(None, ..) | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..) | Element::Item(..) + | Element::ItemWithBackwardsReferences(..) | Element::Reference(..) => { return Err(Error::InvalidProof( "Proof has lower layer for a non Tree".into(), diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 39cf3432..c8d1507b 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -58,6 +58,16 @@ impl Visualize for Element { } } } + Element::ItemWithBackwardsReferences(value, backwards_references, flags) => { + drawer.write(format!("item_with_backwards_references {}:", backwards_references.len()).as_bytes())?; + drawer = value.visualize(drawer)?; + + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } Element::SumItem(value, flags) => { drawer.write(format!("sum_item: {value}").as_bytes())?; @@ -67,6 +77,15 @@ impl Visualize for Element { } } } + Element::SumItemWithBackwardsReferences(value, backwards_references, flags) => { + drawer.write(format!("sum_item_with_backwards_references: {value} | {}", backwards_references.len()).as_bytes())?; + + if let Some(f) = flags { + if !f.is_empty() { + drawer = f.visualize(drawer)?; + } + } + } Element::Reference(_ref, ..) => { drawer.write(b"ref")?; // drawer.write(b"ref: [path: ")?; From 088b149a2b65d27c8fca32f7b82eeef517a92566 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 22 Oct 2024 16:51:58 +0200 Subject: [PATCH 02/27] make it at least compile --- grovedb/src/batch/mod.rs | 34 +++++++--- grovedb/src/debugger.rs | 60 ++++++++++++++--- grovedb/src/element/constructor.rs | 20 ++++-- grovedb/src/element/get.rs | 8 ++- grovedb/src/element/helpers.rs | 14 ++-- grovedb/src/element/mod.rs | 91 +++++++++++++++++++++++--- grovedb/src/lib.rs | 16 +++-- grovedb/src/operations/get/query.rs | 50 +++++++++----- grovedb/src/operations/proof/verify.rs | 3 +- grovedb/src/visualize.rs | 19 +++++- 10 files changed, 255 insertions(+), 60 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 2767ab2b..7eaf87a5 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1007,13 +1007,16 @@ where }; match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( &cost, path_from_reference_qualified_path_type(path, qualified_path) @@ -1081,7 +1084,10 @@ where | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => { match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!( &cost, element.serialize(grove_version) @@ -1128,7 +1134,8 @@ where } } } - Element::Reference(path, ..) => { + Element::Reference(path, ..) + | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( &cost, path_from_reference_qualified_path_type( @@ -1154,13 +1161,16 @@ where } } GroveOp::InsertOnly { element } => match element { - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } - Element::Reference(path, ..) => { + Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( &cost, path_from_reference_qualified_path_type(path.clone(), qualified_path) @@ -1302,7 +1312,12 @@ where | GroveOp::InsertOrReplace { element } | GroveOp::Replace { element } | GroveOp::Patch { element, .. } => match &element { - Element::Reference(path_reference, element_max_reference_hop, _) => { + Element::Reference(path_reference, element_max_reference_hop, _) + | Element::BidirectionalReference( + path_reference, + element_max_reference_hop, + .., + ) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1367,7 +1382,10 @@ where ) ); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 1920ff81..64d40342 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -510,10 +510,13 @@ fn query_item_to_grovedb(item: QueryItem) -> crate::QueryItem { fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { match element { - crate::Element::Item(value, element_flags) => grovedbg_types::Element::Item { - value, - element_flags, - }, + crate::Element::Item(value, element_flags) + | crate::Element::ItemWithBackwardsReferences(value, _, element_flags) => { + grovedbg_types::Element::Item { + value, + element_flags, + } + } crate::Element::Tree(root_key, element_flags) => grovedbg_types::Element::Subtree { root_key, element_flags, @@ -522,6 +525,11 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::AbsolutePathReference(path), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::AbsolutePathReference(path), + _, + element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { path, element_flags, @@ -530,6 +538,11 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), + _, + element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightReference { n_keep: n_keep.into(), @@ -544,6 +557,14 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + n_keep, + path_append, + ), + _, + element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { n_keep: n_keep.into(), @@ -555,6 +576,11 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), + _, + element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamFromElementHeightReference { n_remove: n_remove.into(), @@ -566,6 +592,11 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::CousinReference(swap_parent), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::CousinReference(swap_parent), + _, + element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { swap_parent, element_flags, @@ -574,6 +605,11 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::RemovedCousinReference(swap_parent), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::RemovedCousinReference(swap_parent), + _, + element_flags, ) => { grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { swap_parent, @@ -584,14 +620,22 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { ReferencePathType::SiblingReference(sibling_key), _, element_flags, + ) + | crate::Element::BidirectionalReference( + ReferencePathType::SiblingReference(sibling_key), + _, + element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { sibling_key, element_flags, }), - crate::Element::SumItem(value, element_flags) => grovedbg_types::Element::SumItem { - value, - element_flags, - }, + crate::Element::SumItem(value, element_flags) + | crate::Element::SumItemWithBackwardsReferences(value, _, element_flags) => { + grovedbg_types::Element::SumItem { + value, + element_flags, + } + } crate::Element::SumTree(root_key, sum, element_flags) => grovedbg_types::Element::Sumtree { root_key, sum, diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 04cd5523..b962723c 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -41,7 +41,8 @@ impl Element { } #[cfg(feature = "full")] - /// Set element to an item without flags that allows for a backwards reference + /// Set element to an item without flags that allows for a backwards + /// reference pub fn new_item_allowing_bidirectional_references(item_value: Vec) -> Self { Element::ItemWithBackwardsReferences(item_value, vec![], None) } @@ -53,8 +54,12 @@ impl Element { } #[cfg(feature = "full")] - /// Set element to an item with flags and that allows for a backwards reference - pub fn new_item_allowing_bidirectional_references_with_flags(item_value: Vec, flags: Option) -> Self { + /// Set element to an item with flags and that allows for a backwards + /// reference + pub fn new_item_allowing_bidirectional_references_with_flags( + item_value: Vec, + flags: Option, + ) -> Self { Element::ItemWithBackwardsReferences(item_value, vec![], flags) } @@ -65,7 +70,8 @@ impl Element { } #[cfg(feature = "full")] - /// Set element to a sum item hat allows for backwards references without flags + /// Set element to a sum item hat allows for backwards references without + /// flags pub fn new_sum_item_allowing_bidirectional_references(value: i64) -> Self { Element::SumItemWithBackwardsReferences(value, vec![], None) } @@ -78,11 +84,13 @@ impl Element { #[cfg(feature = "full")] /// Set element to a sum item hat allows for backwards references with flags - pub fn new_sum_item_allowing_bidirectional_references_with_flags(value: i64, flags: Option) -> Self { + pub fn new_sum_item_allowing_bidirectional_references_with_flags( + value: i64, + flags: Option, + ) -> Self { Element::SumItemWithBackwardsReferences(value, vec![], flags) } - #[cfg(feature = "full")] /// Set element to a reference without flags pub fn new_reference(reference_path: ReferencePathType) -> Self { diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 1a0df4e3..a746fde9 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -156,7 +156,10 @@ impl Element { .transpose() ); match &element { - Some(Element::Item(..)) | Some(Element::Reference(..)) | Some(Element::ItemWithBackwardsReferences(..)) | Some(Element::BidirectionalReference(..)) => { + Some(Element::Item(..)) + | Some(Element::Reference(..)) + | Some(Element::ItemWithBackwardsReferences(..)) + | Some(Element::BidirectionalReference(..)) => { // while the loaded item might be a sum item, it is given for free // as it would be very hard to know in advance cost.storage_loaded_bytes = KV::value_byte_cost_size_for_key_and_value_lengths( @@ -165,7 +168,8 @@ impl Element { false, ) as u64 } - Some(Element::SumItem(_, flags)) | Some(Element::SumItemWithBackwardsReferences(_, _, flags)) => { + Some(Element::SumItem(_, flags)) + | Some(Element::SumItemWithBackwardsReferences(_, _, flags)) => { let cost_size = SUM_ITEM_COST_SIZE; let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index a8f60528..0142d1fe 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -164,7 +164,8 @@ impl Element { | Element::SumTree(.., flags) | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) - | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, + | Element::SumItemWithBackwardsReferences(_, _, flags) + | Element::BidirectionalReference(_, _, flags) => flags, } } @@ -178,7 +179,8 @@ impl Element { | Element::SumTree(.., flags) | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) - | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, + | Element::SumItemWithBackwardsReferences(_, _, flags) + | Element::BidirectionalReference(_, _, flags) => flags, } } @@ -192,7 +194,8 @@ impl Element { | Element::SumTree(.., flags) | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) - | Element::SumItemWithBackwardsReferences(_, _, flags) => flags, + | Element::SumItemWithBackwardsReferences(_, _, flags) + | Element::BidirectionalReference(_, _, flags) => flags, } } @@ -204,7 +207,10 @@ impl Element { | Element::Item(_, flags) | Element::Reference(_, _, flags) | Element::SumTree(.., flags) - | Element::SumItem(_, flags) => *flags = new_flags, + | Element::SumItem(_, flags) + | Element::ItemWithBackwardsReferences(_, _, flags) + | Element::SumItemWithBackwardsReferences(_, _, flags) + | Element::BidirectionalReference(_, _, flags) => *flags = new_flags, } } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 5cbbcebe..321b37af 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -92,9 +92,18 @@ pub enum Element { /// A reference to an object by its path BidirectionalReference(ReferencePathType, MaxReferenceHop, Option), /// An ordinary value that has a backwards reference - ItemWithBackwardsReferences(Vec, Vec<(ReferencePathType, CascadeOnUpdate)>, Option), - /// Signed integer value that can be totaled in a sum tree that has a backwards reference - SumItemWithBackwardsReferences(SumValue, Vec<(ReferencePathType, CascadeOnUpdate)>, Option), + ItemWithBackwardsReferences( + Vec, + Vec<(ReferencePathType, CascadeOnUpdate)>, + Option, + ), + /// Signed integer value that can be totaled in a sum tree that has a + /// backwards reference + SumItemWithBackwardsReferences( + SumValue, + Vec<(ReferencePathType, CascadeOnUpdate)>, + Option, + ), } impl fmt::Display for Element { @@ -166,10 +175,42 @@ impl fmt::Display for Element { Element::ItemWithBackwardsReferences(data, backwards_references, flags) => { let backwards_references_count = backwards_references.len(); let backwards_references_info = if backwards_references_count < 4 { - backwards_references.iter().map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|") + backwards_references + .iter() + .map(|(backwards_reference, cascade_on_update)| { + format!( + "{}:{}", + backwards_reference, + if *cascade_on_update { + "cascade" + } else { + "no cascade" + } + ) + }) + .collect::>() + .join("|") } else { - let first_backwards_references = backwards_references.iter().take(3).map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|"); - format!("{}...{} total backwards references", first_backwards_references, backwards_references_count) + let first_backwards_references = backwards_references + .iter() + .take(3) + .map(|(backwards_reference, cascade_on_update)| { + format!( + "{}:{}", + backwards_reference, + if *cascade_on_update { + "cascade" + } else { + "no cascade" + } + ) + }) + .collect::>() + .join("|"); + format!( + "{}...{} total backwards references", + first_backwards_references, backwards_references_count + ) }; write!( f, @@ -184,10 +225,42 @@ impl fmt::Display for Element { Element::SumItemWithBackwardsReferences(sum_value, backwards_references, flags) => { let backwards_references_count = backwards_references.len(); let backwards_references_info = if backwards_references_count < 4 { - backwards_references.iter().map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|") + backwards_references + .iter() + .map(|(backwards_reference, cascade_on_update)| { + format!( + "{}:{}", + backwards_reference, + if *cascade_on_update { + "cascade" + } else { + "no cascade" + } + ) + }) + .collect::>() + .join("|") } else { - let first_backwards_references = backwards_references.iter().take(3).map(|(backwards_reference, cascade_on_update)| format!("{}:{}", backwards_reference, if *cascade_on_update {"cascade"} else {"no cascade"})).collect::>().join("|"); - format!("{}...{} total backwards references", first_backwards_references, backwards_references_count) + let first_backwards_references = backwards_references + .iter() + .take(3) + .map(|(backwards_reference, cascade_on_update)| { + format!( + "{}:{}", + backwards_reference, + if *cascade_on_update { + "cascade" + } else { + "no cascade" + } + ) + }) + .collect::>() + .join("|"); + format!( + "{}...{} total backwards references", + first_backwards_references, backwards_references_count + ) }; write!( f, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 57f68d33..9467d686 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -1034,7 +1034,10 @@ impl GroveDb { grove_version, )?); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1057,7 +1060,8 @@ impl GroveDb { ); } } - Element::Reference(ref reference_path, ..) => { + Element::Reference(ref reference_path, ..) + | Element::BidirectionalReference(ref reference_path, ..) => { // Skip this whole check if we don't `verify_references` if !verify_references { continue; @@ -1180,7 +1184,10 @@ impl GroveDb { grove_version, )?); } - Element::Item(..) | Element::SumItem(..) => { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1203,7 +1210,8 @@ impl GroveDb { ); } } - Element::Reference(ref reference_path, ..) => { + Element::Reference(ref reference_path, ..) + | Element::BidirectionalReference(ref reference_path, ..) => { // Skip this whole check if we don't `verify_references` if !verify_references { continue; diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 63d9587f..ebbf04fb 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -194,7 +194,8 @@ where { .follow_element ); match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -222,7 +223,11 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => Ok(element), + Element::Item(..) + | Element::SumItem(..) + | Element::SumTree(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -309,7 +314,8 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -339,8 +345,13 @@ where { )), } } - Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => Ok(item), - Element::SumItem(item, _) | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item.encode_var_vec()), + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => { + Ok(item) + } + Element::SumItem(item, _) + | Element::SumItemWithBackwardsReferences(item, ..) => { + Ok(item.encode_var_vec()) + } Element::Tree(..) | Element::SumTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), @@ -395,7 +406,8 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -432,8 +444,11 @@ where { )), } } - Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => Ok(QueryItemOrSumReturnType::ItemData(item)), - Element::SumItem(sum_value, _) | Element::SumItemWithBackwardsReferences(sum_value, ..) => { + Element::Item(item, _) | Element::ItemWithBackwardsReferences(item, ..) => { + Ok(QueryItemOrSumReturnType::ItemData(item)) + } + Element::SumItem(sum_value, _) + | Element::SumItemWithBackwardsReferences(sum_value, ..) => { Ok(QueryItemOrSumReturnType::SumValue(sum_value)) } Element::SumTree(_, sum_value, _) => { @@ -489,7 +504,8 @@ where { .map(|result_item| match result_item { QueryResultElement::ElementResultItem(element) => { match element { - Element::Reference(reference_path, ..) => { + Element::Reference(reference_path, ..) + | Element::BidirectionalReference(reference_path, ..) => { match reference_path { ReferencePathType::AbsolutePathReference(absolute_path) => { // While `map` on iterator is lazy, we should accumulate costs @@ -519,13 +535,15 @@ where { )), } } - Element::SumItem(item, _) | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::Item(..) | Element::ItemWithBackwardsReferences(..) => { - Err(Error::InvalidQuery( - "path_queries over sum items can only refer to sum items and \ - references", - )) - } + Element::SumItem(item, _) + | Element::SumItemWithBackwardsReferences(item, ..) => Ok(item), + Element::Tree(..) + | Element::SumTree(..) + | Element::Item(..) + | Element::ItemWithBackwardsReferences(..) => Err(Error::InvalidQuery( + "path_queries over sum items can only refer to sum items and \ + references", + )), } } _ => Err(Error::CorruptedCodeExecution( diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 6e0c09b0..b0115691 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -341,7 +341,8 @@ impl GroveDb { | Element::SumItemWithBackwardsReferences(..) | Element::Item(..) | Element::ItemWithBackwardsReferences(..) - | Element::Reference(..) => { + | Element::Reference(..) + | Element::BidirectionalReference(..) => { return Err(Error::InvalidProof( "Proof has lower layer for a non Tree".into(), )); diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index c8d1507b..029d1bca 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -59,7 +59,13 @@ impl Visualize for Element { } } Element::ItemWithBackwardsReferences(value, backwards_references, flags) => { - drawer.write(format!("item_with_backwards_references {}:", backwards_references.len()).as_bytes())?; + drawer.write( + format!( + "item_with_backwards_references {}:", + backwards_references.len() + ) + .as_bytes(), + )?; drawer = value.visualize(drawer)?; if let Some(f) = flags { @@ -78,7 +84,13 @@ impl Visualize for Element { } } Element::SumItemWithBackwardsReferences(value, backwards_references, flags) => { - drawer.write(format!("sum_item_with_backwards_references: {value} | {}", backwards_references.len()).as_bytes())?; + drawer.write( + format!( + "sum_item_with_backwards_references: {value} | {}", + backwards_references.len() + ) + .as_bytes(), + )?; if let Some(f) = flags { if !f.is_empty() { @@ -120,6 +132,9 @@ impl Visualize for Element { } } } + Element::BidirectionalReference(..) => { + drawer.write(b"bidi ref")?; + } } Ok(drawer) } From b7730f125187314f9b485dca65a4e4f1d4e89bbc Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 24 Oct 2024 15:07:55 +0200 Subject: [PATCH 03/27] wip --- merk/src/merk/mod.rs | 42 ++- merk/src/merk/open.rs | 22 +- merk/src/merk/restore.rs | 14 +- merk/src/test_utils/mod.rs | 10 +- merk/src/test_utils/temp_merk.rs | 44 ++- storage/src/rocksdb_storage.rs | 2 +- storage/src/rocksdb_storage/storage.rs | 33 +- .../src/rocksdb_storage/storage_context.rs | 2 - .../storage_context/context_no_tx.rs | 286 ------------------ storage/src/rocksdb_storage/tests.rs | 32 +- storage/src/storage.rs | 13 - 11 files changed, 119 insertions(+), 381 deletions(-) delete mode 100644 storage/src/rocksdb_storage/storage_context/context_no_tx.rs diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index ee0deccc..d5f40c78 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -754,7 +754,7 @@ mod test { use grovedb_path::SubtreePath; use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbStorageContext, RocksDbStorage}, + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, RawIterator, Storage, StorageBatch, StorageContext, }; use grovedb_version::version::GroveVersion; @@ -987,9 +987,10 @@ mod test { let tmp_dir = TempDir::new().expect("cannot open tempdir"); let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1013,9 +1014,10 @@ mod test { let tmp_dir = TempDir::new().expect("cannot open tempdir"); let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1034,7 +1036,7 @@ mod test { fn reopen() { let grove_version = GroveVersion::latest(); fn collect( - mut node: RefWalker>, + mut node: RefWalker>, nodes: &mut Vec>, ) { let grove_version = GroveVersion::latest(); @@ -1069,9 +1071,10 @@ mod test { let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1085,12 +1088,12 @@ mod test { .unwrap(); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .expect("cannot commit batch"); let merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1104,14 +1107,19 @@ mod test { let mut nodes = vec![]; collect(walker, &mut nodes); + storage + .commit_transaction(tx) + .unwrap() + .expect("unable to commit transaction"); nodes }; let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); + let tx = storage.start_transaction(); let merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1129,7 +1137,7 @@ mod test { } type PrefixedStorageIter<'db, 'ctx> = - &'ctx mut as StorageContext<'db>>::RawIterator; + &'ctx mut as StorageContext<'db>>::RawIterator; #[test] fn reopen_iter() { @@ -1149,9 +1157,10 @@ mod test { let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1170,9 +1179,10 @@ mod test { .expect("cannot commit batch"); let mut nodes = vec![]; + let tx = storage.start_transaction(); let merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1185,9 +1195,10 @@ mod test { }; let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); + let tx = storage.start_transaction(); let merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1209,9 +1220,10 @@ mod test { let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -1269,13 +1281,13 @@ mod test { assert_eq!(result, Some(b"b".to_vec())); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .expect("cannot commit batch"); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index c8646afa..3c253094 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -112,9 +112,14 @@ mod test { let test_prefix = [b"ayy"]; let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(SubtreePath::from(test_prefix.as_ref()), Some(&batch)) + .get_transactional_storage_context( + SubtreePath::from(test_prefix.as_ref()), + Some(&batch), + &tx, + ) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -135,13 +140,17 @@ mod test { let root_hash = merk.root_hash(); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .expect("cannot commit batch"); let merk = Merk::open_base( storage - .get_storage_context(SubtreePath::from(test_prefix.as_ref()), None) + .get_transactional_storage_context( + SubtreePath::from(test_prefix.as_ref()), + None, + &tx, + ) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -157,10 +166,11 @@ mod test { let grove_version = GroveVersion::latest(); let storage = TempStorage::new(); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let merk_fee_context = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -184,13 +194,13 @@ mod test { .expect("apply failed"); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .expect("cannot commit batch"); let merk_fee_context = Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index 1082e80b..3fc393ed 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -549,7 +549,7 @@ mod tests { use grovedb_storage::{ rocksdb_storage::{ test_utils::TempStorage, PrefixedRocksDbImmediateStorageContext, - PrefixedRocksDbStorageContext, + PrefixedRocksDbTransactionContext, }, RawIterator, Storage, }; @@ -574,7 +574,7 @@ mod tests { Op::Push(Node::KV(vec![3], vec![3])), Op::Parent, ]; - assert!(Restorer::::verify_chunk( + assert!(Restorer::::verify_chunk( non_avl_tree_proof, &[0; 32], &None @@ -586,7 +586,7 @@ mod tests { fn test_chunk_verification_only_kv_feature_and_hash() { // should not accept kv let invalid_chunk_proof = vec![Op::Push(Node::KV(vec![1], vec![1]))]; - let verification_result = Restorer::::verify_chunk( + let verification_result = Restorer::::verify_chunk( invalid_chunk_proof, &[0; 32], &None, @@ -600,7 +600,7 @@ mod tests { // should not accept kvhash let invalid_chunk_proof = vec![Op::Push(Node::KVHash([0; 32]))]; - let verification_result = Restorer::::verify_chunk( + let verification_result = Restorer::::verify_chunk( invalid_chunk_proof, &[0; 32], &None, @@ -614,7 +614,7 @@ mod tests { // should not accept kvdigest let invalid_chunk_proof = vec![Op::Push(Node::KVDigest(vec![0], [0; 32]))]; - let verification_result = Restorer::::verify_chunk( + let verification_result = Restorer::::verify_chunk( invalid_chunk_proof, &[0; 32], &None, @@ -628,7 +628,7 @@ mod tests { // should not accept kvvaluehash let invalid_chunk_proof = vec![Op::Push(Node::KVValueHash(vec![0], vec![0], [0; 32]))]; - let verification_result = Restorer::::verify_chunk( + let verification_result = Restorer::::verify_chunk( invalid_chunk_proof, &[0; 32], &None, @@ -642,7 +642,7 @@ mod tests { // should not accept kvrefvaluehash let invalid_chunk_proof = vec![Op::Push(Node::KVRefValueHash(vec![0], vec![0], [0; 32]))]; - let verification_result = Restorer::::verify_chunk( + let verification_result = Restorer::::verify_chunk( invalid_chunk_proof, &[0; 32], &None, diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 45beda4f..e21736d1 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -311,14 +311,15 @@ pub fn make_tree_seq_with_start_key( pub fn empty_path_merk<'db, S>( storage: &'db S, batch: &'db StorageBatch, + tx: &'db >::Transaction, grove_version: &GroveVersion, -) -> Merk<>::BatchStorageContext> +) -> Merk<>::BatchTransactionalStorageContext> where S: Storage<'db>, { Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), Some(batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(batch), tx) .unwrap(), false, None:: Option>, @@ -331,14 +332,15 @@ where /// Shortcut to open a Merk for read only pub fn empty_path_merk_read_only<'db, S>( storage: &'db S, + tx: &'db >::Transaction, grove_version: &GroveVersion, -) -> Merk<>::BatchStorageContext> +) -> Merk<>::BatchTransactionalStorageContext> where S: Storage<'db>, { Merk::open_base( storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, tx) .unwrap(), false, None:: Option>, diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 69e5b555..60f9986f 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -32,11 +32,11 @@ use std::ops::{Deref, DerefMut}; use grovedb_path::SubtreePath; -use grovedb_storage::StorageBatch; #[cfg(feature = "full")] +use grovedb_storage::{rocksdb_storage::test_utils::TempStorage, Storage}; use grovedb_storage::{ - rocksdb_storage::{test_utils::TempStorage, PrefixedRocksDbStorageContext}, - Storage, + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, + StorageBatch, }; use grovedb_version::version::GroveVersion; @@ -49,7 +49,8 @@ use crate::Merk; pub struct TempMerk { storage: &'static TempStorage, batch: &'static StorageBatch, - merk: Merk>, + merk: Merk>, + tx: &'static >::Transaction, } #[cfg(feature = "full")] @@ -59,9 +60,10 @@ impl TempMerk { pub fn new(grove_version: &GroveVersion) -> Self { let storage = Box::leak(Box::new(TempStorage::new())); let batch = Box::leak(Box::new(StorageBatch::new())); + let tx = Box::leak(Box::new(storage.start_transaction())); let context = storage - .get_storage_context(SubtreePath::empty(), Some(batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(batch), tx) .unwrap(); let merk = Merk::open_base( @@ -76,20 +78,32 @@ impl TempMerk { storage, merk, batch, + tx, } } /// Commits pending batch operations. pub fn commit(&mut self, grove_version: &GroveVersion) { - let batch = unsafe { Box::from_raw(self.batch as *const _ as *mut StorageBatch) }; + let batch: Box = + unsafe { Box::from_raw(self.batch as *const _ as *mut StorageBatch) }; + let tx: Box<>::Transaction> = unsafe { + Box::from_raw( + self.tx as *const _ as *mut >::Transaction, + ) + }; self.storage - .commit_multi_context_batch(*batch, None) + .commit_multi_context_batch(*batch, Some(self.tx)) .unwrap() .expect("unable to commit batch"); + self.storage + .commit_transaction(*tx) + .unwrap() + .expect("unable to commit transaction"); self.batch = Box::leak(Box::new(StorageBatch::new())); + self.tx = Box::leak(Box::new(self.storage.start_transaction())); let context = self .storage - .get_storage_context(SubtreePath::empty(), Some(self.batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(self.batch), self.tx) .unwrap(); self.merk = Merk::open_base( context, @@ -107,7 +121,13 @@ impl Drop for TempMerk { fn drop(&mut self) { unsafe { let batch = Box::from_raw(self.batch as *const _ as *mut StorageBatch); - let _ = self.storage.commit_multi_context_batch(*batch, None); + + let tx: Box<>::Transaction> = Box::from_raw( + self.tx as *const _ as *mut >::Transaction, + ); + + let _ = self.storage.commit_multi_context_batch(*batch, Some(&tx)); + let _ = self.storage.commit_transaction(*tx).unwrap(); drop(Box::from_raw(self.storage as *const _ as *mut TempStorage)); } } @@ -122,16 +142,16 @@ impl Default for TempMerk { #[cfg(feature = "full")] impl Deref for TempMerk { - type Target = Merk>; + type Target = Merk>; - fn deref(&self) -> &Merk> { + fn deref(&self) -> &Merk> { &self.merk } } #[cfg(feature = "full")] impl DerefMut for TempMerk { - fn deref_mut(&mut self) -> &mut Merk> { + fn deref_mut(&mut self) -> &mut Merk> { &mut self.merk } } diff --git a/storage/src/rocksdb_storage.rs b/storage/src/rocksdb_storage.rs index 14c4df5a..2905adce 100644 --- a/storage/src/rocksdb_storage.rs +++ b/storage/src/rocksdb_storage.rs @@ -36,7 +36,7 @@ mod tests; pub use rocksdb::{Error, WriteBatchWithTransaction}; pub use storage_context::{ PrefixedRocksDbBatch, PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbRawIterator, - PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext, + PrefixedRocksDbTransactionContext, }; pub use self::storage::RocksDbStorage; diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 8a91d4f4..8913e9ea 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -44,10 +44,7 @@ use rocksdb::{ Transaction, WriteBatchWithTransaction, DEFAULT_COLUMN_FAMILY_NAME, }; -use super::{ - PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbStorageContext, - PrefixedRocksDbTransactionContext, -}; +use super::{PrefixedRocksDbImmediateStorageContext, PrefixedRocksDbTransactionContext}; use crate::{ error, error::Error::{CostError, RocksDBError}, @@ -435,7 +432,6 @@ impl RocksDbStorage { } impl<'db> Storage<'db> for RocksDbStorage { - type BatchStorageContext = PrefixedRocksDbStorageContext<'db>; type BatchTransactionalStorageContext = PrefixedRocksDbTransactionContext<'db>; type ImmediateStorageContext = PrefixedRocksDbImmediateStorageContext<'db>; type Transaction = Tx<'db>; @@ -460,18 +456,6 @@ impl<'db> Storage<'db> for RocksDbStorage { self.db.flush().map_err(RocksDBError) } - fn get_storage_context<'b, B>( - &'db self, - path: SubtreePath<'b, B>, - batch: Option<&'db StorageBatch>, - ) -> CostContext - where - B: AsRef<[u8]> + 'b, - { - Self::build_prefix(path) - .map(|prefix| PrefixedRocksDbStorageContext::new(&self.db, prefix, batch)) - } - fn get_transactional_storage_context<'b, B>( &'db self, path: SubtreePath<'b, B>, @@ -594,11 +578,12 @@ mod tests { }; let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let left = storage - .get_storage_context(left_path.clone(), Some(&batch)) + .get_transactional_storage_context(left_path.clone(), Some(&batch), &tx) .unwrap(); let right = storage - .get_storage_context(right_path.clone(), Some(&batch)) + .get_transactional_storage_context(right_path.clone(), Some(&batch), &tx) .unwrap(); left.put(b"a", b"a", None, None).unwrap().unwrap(); @@ -615,11 +600,12 @@ mod tests { .expect("cannot commit batch"); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let left = storage - .get_storage_context(left_path.clone(), Some(&batch)) + .get_transactional_storage_context(left_path.clone(), Some(&batch), &tx) .unwrap(); let right = storage - .get_storage_context(right_path.clone(), Some(&batch)) + .get_transactional_storage_context(right_path.clone(), Some(&batch), &tx) .unwrap(); // Iterate over left subtree while right subtree contains 1 byte keys: @@ -660,7 +646,10 @@ mod tests { .unwrap() .expect("cannot commit batch"); - let left = storage.get_storage_context(left_path, None).unwrap(); + let tx = storage.start_transaction(); + let left = storage + .get_transactional_storage_context(left_path, None, &tx) + .unwrap(); // Iterate over left subtree once again let mut iteration_cost_after = OperationCost::default(); let mut iter = left.raw_iter(); diff --git a/storage/src/rocksdb_storage/storage_context.rs b/storage/src/rocksdb_storage/storage_context.rs index 0611d51c..758ba16f 100644 --- a/storage/src/rocksdb_storage/storage_context.rs +++ b/storage/src/rocksdb_storage/storage_context.rs @@ -30,13 +30,11 @@ mod batch; pub mod context_immediate; -mod context_no_tx; mod context_tx; mod raw_iterator; pub use batch::PrefixedRocksDbBatch; pub use context_immediate::PrefixedRocksDbImmediateStorageContext; -pub use context_no_tx::PrefixedRocksDbStorageContext; pub use context_tx::PrefixedRocksDbTransactionContext; pub use raw_iterator::PrefixedRocksDbRawIterator; diff --git a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs b/storage/src/rocksdb_storage/storage_context/context_no_tx.rs deleted file mode 100644 index 80ad0149..00000000 --- a/storage/src/rocksdb_storage/storage_context/context_no_tx.rs +++ /dev/null @@ -1,286 +0,0 @@ -// MIT LICENSE -// -// Copyright (c) 2021 Dash Core Group -// -// Permission is hereby granted, free of charge, to any -// person obtaining a copy of this software and associated -// documentation files (the "Software"), to deal in the -// Software without restriction, including without -// limitation the rights to use, copy, modify, merge, -// publish, distribute, sublicense, and/or sell copies of -// the Software, and to permit persons to whom the Software -// is furnished to do so, subject to the following -// conditions: -// -// The above copyright notice and this permission notice -// shall be included in all copies or substantial portions -// of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -//! Storage context batch implementation without a transaction - -use error::Error; -use grovedb_costs::{ - storage_cost::key_value_cost::KeyValueStorageCost, ChildrenSizesWithIsSumTree, CostResult, - CostsExt, OperationCost, -}; -use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode}; - -use super::{batch::PrefixedMultiContextBatchPart, make_prefixed_key, PrefixedRocksDbRawIterator}; -use crate::{ - error, - error::Error::RocksDBError, - rocksdb_storage::storage::{Db, SubtreePrefix, AUX_CF_NAME, META_CF_NAME, ROOTS_CF_NAME}, - StorageBatch, StorageContext, -}; - -/// Storage context with a prefix applied to be used in a subtree to be used -/// outside of transaction. -pub struct PrefixedRocksDbStorageContext<'db> { - storage: &'db Db, - prefix: SubtreePrefix, - batch: Option<&'db StorageBatch>, -} - -impl<'db> PrefixedRocksDbStorageContext<'db> { - /// Create a new prefixed storage_cost context instance - pub fn new(storage: &'db Db, prefix: SubtreePrefix, batch: Option<&'db StorageBatch>) -> Self { - PrefixedRocksDbStorageContext { - storage, - prefix, - batch, - } - } -} - -impl<'db> PrefixedRocksDbStorageContext<'db> { - /// Get auxiliary data column family - fn cf_aux(&self) -> &'db ColumnFamily { - self.storage - .cf_handle(AUX_CF_NAME) - .expect("aux column family must exist") - } - - /// Get trees roots data column family - fn cf_roots(&self) -> &'db ColumnFamily { - self.storage - .cf_handle(ROOTS_CF_NAME) - .expect("roots column family must exist") - } - - /// Get metadata column family - fn cf_meta(&self) -> &'db ColumnFamily { - self.storage - .cf_handle(META_CF_NAME) - .expect("meta column family must exist") - } -} - -impl<'db> StorageContext<'db> for PrefixedRocksDbStorageContext<'db> { - type Batch = PrefixedMultiContextBatchPart; - type RawIterator = PrefixedRocksDbRawIterator>; - - fn put>( - &self, - key: K, - value: &[u8], - children_sizes: ChildrenSizesWithIsSumTree, - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.put( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - children_sizes, - cost_info, - ); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn put_aux>( - &self, - key: K, - value: &[u8], - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.put_aux( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - cost_info, - ); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn put_root>( - &self, - key: K, - value: &[u8], - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.put_root( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - cost_info, - ); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn put_meta>( - &self, - key: K, - value: &[u8], - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.put_meta( - make_prefixed_key(&self.prefix, key), - value.to_vec(), - cost_info, - ); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn delete>( - &self, - key: K, - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.delete(make_prefixed_key(&self.prefix, key), cost_info); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn delete_aux>( - &self, - key: K, - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.delete_aux(make_prefixed_key(&self.prefix, key), cost_info); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn delete_root>( - &self, - key: K, - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.delete_root(make_prefixed_key(&self.prefix, key), cost_info); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn delete_meta>( - &self, - key: K, - cost_info: Option, - ) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.delete_meta(make_prefixed_key(&self.prefix, key), cost_info); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn get>(&self, key: K) -> CostResult>, Error> { - self.storage - .get(make_prefixed_key(&self.prefix, key)) - .map_err(RocksDBError) - .wrap_fn_cost(|value| OperationCost { - seek_count: 1, - storage_loaded_bytes: value - .as_ref() - .ok() - .and_then(Option::as_ref) - .map(|x| x.len() as u64) - .unwrap_or(0), - ..Default::default() - }) - } - - fn get_aux>(&self, key: K) -> CostResult>, Error> { - self.storage - .get_cf(self.cf_aux(), make_prefixed_key(&self.prefix, key)) - .map_err(RocksDBError) - .wrap_fn_cost(|value| OperationCost { - seek_count: 1, - storage_loaded_bytes: value - .as_ref() - .ok() - .and_then(Option::as_ref) - .map(|x| x.len() as u64) - .unwrap_or(0), - ..Default::default() - }) - } - - fn get_root>(&self, key: K) -> CostResult>, Error> { - self.storage - .get_cf(self.cf_roots(), make_prefixed_key(&self.prefix, key)) - .map_err(RocksDBError) - .wrap_fn_cost(|value| OperationCost { - seek_count: 1, - storage_loaded_bytes: value - .as_ref() - .ok() - .and_then(Option::as_ref) - .map(|x| x.len() as u64) - .unwrap_or(0), - ..Default::default() - }) - } - - fn get_meta>(&self, key: K) -> CostResult>, Error> { - self.storage - .get_cf(self.cf_meta(), make_prefixed_key(&self.prefix, key)) - .map_err(RocksDBError) - .wrap_fn_cost(|value| OperationCost { - seek_count: 1, - storage_loaded_bytes: value - .as_ref() - .ok() - .and_then(Option::as_ref) - .map(|x| x.len() as u64) - .unwrap_or(0), - ..Default::default() - }) - } - - fn new_batch(&self) -> Self::Batch { - PrefixedMultiContextBatchPart { - prefix: self.prefix, - batch: StorageBatch::new(), - } - } - - fn commit_batch(&self, batch: Self::Batch) -> CostResult<(), Error> { - if let Some(existing_batch) = self.batch { - existing_batch.merge(batch.batch); - } - Ok(()).wrap_with_cost(OperationCost::default()) - } - - fn raw_iter(&self) -> Self::RawIterator { - PrefixedRocksDbRawIterator { - prefix: self.prefix, - raw_iterator: self.storage.raw_iterator(), - } - } -} diff --git a/storage/src/rocksdb_storage/tests.rs b/storage/src/rocksdb_storage/tests.rs index c75568cd..4000be36 100644 --- a/storage/src/rocksdb_storage/tests.rs +++ b/storage/src/rocksdb_storage/tests.rs @@ -555,11 +555,12 @@ mod batch_no_transaction { fn test_various_cf_methods() { let storage = TempStorage::new(); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayya"].as_ref().into(), Some(&batch), &tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), Some(&batch), &tx) .unwrap(); context_ayya @@ -611,10 +612,10 @@ mod batch_no_transaction { .expect("cannot commit batch"); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), None) + .get_transactional_storage_context([b"ayya"].as_ref().into(), None, &tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), None) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), None, &tx) .unwrap(); assert_eq!( @@ -696,11 +697,12 @@ mod batch_no_transaction { fn test_with_db_batches() { let storage = TempStorage::new(); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayya"].as_ref().into(), Some(&batch), &tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), Some(&batch), &tx) .unwrap(); context_ayya @@ -761,7 +763,7 @@ mod batch_no_transaction { .expect("cannot commit multi context batch"); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), None) + .get_transactional_storage_context([b"ayya"].as_ref().into(), None, &tx) .unwrap(); assert_eq!( context_ayya @@ -786,11 +788,13 @@ mod batch_transaction { let batch = StorageBatch::new(); let batch_tx = StorageBatch::new(); + + let other_tx = storage.start_transaction(); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayya"].as_ref().into(), Some(&batch), &other_tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), Some(&batch)) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), Some(&batch), &other_tx) .unwrap(); let context_ayya_tx = storage .get_transactional_storage_context( @@ -1033,11 +1037,12 @@ mod batch_transaction { ); // And still no data in the database until transaction is commited + let other_tx = storage.start_transaction(); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), None) + .get_transactional_storage_context([b"ayya"].as_ref().into(), None, &other_tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), None) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), None, &other_tx) .unwrap(); let mut iter = context_ayya.raw_iter(); @@ -1053,11 +1058,12 @@ mod batch_transaction { .unwrap() .expect("cannot commit transaction"); + let other_tx = storage.start_transaction(); let context_ayya = storage - .get_storage_context([b"ayya"].as_ref().into(), None) + .get_transactional_storage_context([b"ayya"].as_ref().into(), None, &other_tx) .unwrap(); let context_ayyb = storage - .get_storage_context([b"ayyb"].as_ref().into(), None) + .get_transactional_storage_context([b"ayyb"].as_ref().into(), None, &other_tx) .unwrap(); assert_eq!( diff --git a/storage/src/storage.rs b/storage/src/storage.rs index 5ef26e06..13f8d083 100644 --- a/storage/src/storage.rs +++ b/storage/src/storage.rs @@ -50,9 +50,6 @@ pub trait Storage<'db> { /// Storage transaction type type Transaction; - /// Storage context type for mutli-tree batch operations - type BatchStorageContext: StorageContext<'db>; - /// Storage context type for multi-tree batch operations inside transaction type BatchTransactionalStorageContext: StorageContext<'db>; @@ -79,16 +76,6 @@ pub trait Storage<'db> { /// Forces data to be written fn flush(&self) -> Result<(), Error>; - /// Make storage context for a subtree with path, keeping all write - /// operations inside a `batch` if provided. - fn get_storage_context<'b, B>( - &'db self, - path: SubtreePath<'b, B>, - batch: Option<&'db StorageBatch>, - ) -> CostContext - where - B: AsRef<[u8]> + 'b; - /// Make context for a subtree on transactional data, keeping all write /// operations inside a `batch` if provided. fn get_transactional_storage_context<'b, B>( From 49411e64144269113cfb2929b93255f806e6aea1 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 25 Oct 2024 16:14:38 +0200 Subject: [PATCH 04/27] wip --- grovedb/src/batch/mod.rs | 432 ++++--------- grovedb/src/debugger.rs | 21 +- grovedb/src/element/get.rs | 34 +- grovedb/src/element/insert.rs | 12 +- grovedb/src/element/query.rs | 293 +++++---- .../src/estimated_costs/average_case_costs.rs | 9 +- .../src/estimated_costs/worst_case_costs.rs | 8 +- grovedb/src/lib.rs | 427 ++----------- grovedb/src/operations/auxiliary.rs | 112 ++-- grovedb/src/operations/delete/mod.rs | 412 +++---------- grovedb/src/operations/get/mod.rs | 147 ++--- grovedb/src/operations/insert/mod.rs | 231 +------ grovedb/src/operations/is_empty_tree.rs | 34 +- grovedb/src/operations/proof/generate.rs | 6 +- grovedb/src/replication.rs | 157 ++--- grovedb/src/tests/mod.rs | 27 +- grovedb/src/tests/sum_tree_tests.rs | 69 ++- grovedb/src/tests/tree_hashes_tests.rs | 22 +- grovedb/src/util.rs | 575 +++--------------- grovedb/src/visualize.rs | 64 +- 20 files changed, 816 insertions(+), 2276 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 7eaf87a5..0217943c 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -57,8 +57,7 @@ use grovedb_merk::{ }; use grovedb_path::SubtreePath; use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, - Storage, StorageBatch, StorageContext, + rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch, StorageContext, }; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, @@ -79,6 +78,7 @@ use crate::{ reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, }, + util::TxRef, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; @@ -2235,74 +2235,6 @@ impl GroveDb { } } - /// Opens merk at path with given storage batch context. Returns CostResult. - pub fn open_batch_merk_at_path<'a, B: AsRef<[u8]>>( - &'a self, - storage_batch: &'a StorageBatch, - path: SubtreePath, - new_merk: bool, - grove_version: &GroveVersion, - ) -> CostResult, Error> { - check_grovedb_v0_with_cost!( - "open_batch_merk_at_path", - grove_version - .grovedb_versions - .apply_batch - .open_batch_merk_at_path - ); - let mut local_cost = OperationCost::default(); - let storage = self - .db - .get_storage_context(path.clone(), Some(storage_batch)) - .unwrap_add_cost(&mut local_cost); - - if new_merk { - let merk_type = if path.is_root() { - MerkType::BaseMerk - } else { - MerkType::LayeredMerk - }; - Ok(Merk::open_empty(storage, merk_type, false)).wrap_with_cost(local_cost) - } else if let Some((base_path, last)) = path.derive_parent() { - let parent_storage = self - .db - .get_storage_context(base_path, Some(storage_batch)) - .unwrap_add_cost(&mut local_cost); - let element = cost_return_on_error!( - &mut local_cost, - Element::get_from_storage(&parent_storage, last, grove_version) - ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .add_cost(local_cost) - } else { - Err(Error::CorruptedData( - "cannot open a subtree as parent exists but is not a tree".to_owned(), - )) - .wrap_with_cost(local_cost) - } - } else { - Merk::open_base( - storage, - false, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|_| Error::CorruptedData("cannot open a subtree".to_owned())) - .add_cost(local_cost) - } - } - /// Applies batch of operations on GroveDB pub fn apply_batch_with_element_flags_update( &self, @@ -2337,6 +2269,8 @@ impl GroveDb { return Ok(()).wrap_with_cost(cost); } + let tx = TxRef::new(&self.db, transaction); + // Determines whether to check batch operation consistency // return false if the disable option is set to true, returns true for any other // case @@ -2369,92 +2303,48 @@ impl GroveDb { // 5. Remove operation from the tree, repeat until there are operations to do; // 6. Add root leaves save operation to the batch // 7. Apply storage_cost batch - if let Some(tx) = transaction { - cost_return_on_error!( - &mut cost, - self.apply_body( - ops, - batch_apply_options, - update_element_flags_function, - split_removal_bytes_function, - |path, new_merk| { - self.open_batch_transactional_merk_at_path( - &storage_batch, - path.into(), - tx, - new_merk, - grove_version, - ) - }, - grove_version - ) - ); - - // TODO: compute batch costs - cost_return_on_error!( - &mut cost, - self.db - .commit_multi_context_batch(storage_batch, Some(tx)) - .map_err(|e| e.into()) - ); - - // Keep this commented for easy debugging in the future. - // let issues = self - // .visualize_verify_grovedb(Some(tx), true, - // &Default::default()) .unwrap(); - // if issues.len() > 0 { - // println!( - // "tx_issues: {}", - // issues - // .iter() - // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", - // hash, a, b, c)) .collect::>() - // .join(" | ") - // ); - // } - } else { - cost_return_on_error!( - &mut cost, - self.apply_body( - ops, - batch_apply_options, - update_element_flags_function, - split_removal_bytes_function, - |path, new_merk| { - self.open_batch_merk_at_path( - &storage_batch, - path.into(), - new_merk, - grove_version, - ) - }, - grove_version - ) - ); + cost_return_on_error!( + &mut cost, + self.apply_body( + ops, + batch_apply_options, + update_element_flags_function, + split_removal_bytes_function, + |path, new_merk| { + self.open_batch_transactional_merk_at_path( + &storage_batch, + path.into(), + tx.as_ref(), + new_merk, + grove_version, + ) + }, + grove_version + ) + ); - // TODO: compute batch costs - cost_return_on_error!( - &mut cost, - self.db - .commit_multi_context_batch(storage_batch, None) - .map_err(|e| e.into()) - ); + // TODO: compute batch costs + cost_return_on_error!( + &mut cost, + self.db + .commit_multi_context_batch(storage_batch, Some(tx.as_ref())) + .map_err(|e| e.into()) + ); - // Keep this commented for easy debugging in the future. - // let issues = self - // .visualize_verify_grovedb(None, true, &Default::default()) - // .unwrap(); - // if issues.len() > 0 { - // println!( - // "non_tx_issues: {}", - // issues - // .iter() - // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", - // hash, a, b, c)) .collect::>() - // .join(" | ") - // ); - // } - } + // Keep this commented for easy debugging in the future. + // let issues = self + // .visualize_verify_grovedb(Some(tx), true, + // &Default::default()) .unwrap(); + // if issues.len() > 0 { + // println!( + // "tx_issues: {}", + // issues + // .iter() + // .map(|(hash, (a, b, c))| format!("{}: {} {} {}", + // hash, a, b, c)) .collect::>() + // .join(" | ") + // ); + // } Ok(()).wrap_with_cost(cost) } @@ -2499,6 +2389,8 @@ impl GroveDb { return Ok(()).wrap_with_cost(cost); } + let tx = TxRef::new(&self.db, transaction); + let mut batch_apply_options = batch_apply_options.unwrap_or_default(); if batch_apply_options.batch_pause_height.is_none() { // we default to pausing at the root tree, which is the most common case @@ -2535,176 +2427,92 @@ impl GroveDb { // 5. Remove operation from the tree, repeat until there are operations to do; // 6. Add root leaves save operation to the batch // 7. Apply storage_cost batch - if let Some(tx) = transaction { - let left_over_operations = cost_return_on_error!( - &mut cost, - self.apply_body( - ops, - Some(batch_apply_options.clone()), - &mut update_element_flags_function, - &mut split_removal_bytes_function, - |path, new_merk| { - self.open_batch_transactional_merk_at_path( - &storage_batch, - path.into(), - tx, - new_merk, - grove_version, - ) - }, - grove_version - ) - ); - // if we paused at the root height, the left over operations would be to replace - // a lot of leaf nodes in the root tree - - // let's build the write batch - let (mut write_batch, mut pending_costs) = cost_return_on_error!( - &mut cost, - self.db - .build_write_batch(storage_batch) - .map_err(|e| e.into()) - ); - - let total_current_costs = cost.clone().add(pending_costs.clone()); - - // todo: estimate root costs - - // at this point we need to send the pending costs back - // we will get GroveDB a new set of GroveDBOps - - let new_operations = cost_return_on_error_no_add!( - &cost, - add_on_operations(&total_current_costs, &left_over_operations) - ); - - // we are trying to finalize - batch_apply_options.batch_pause_height = None; - - let continue_storage_batch = StorageBatch::new(); - - cost_return_on_error!( - &mut cost, - self.continue_partial_apply_body( - left_over_operations, - new_operations, - Some(batch_apply_options), - update_element_flags_function, - split_removal_bytes_function, - |path, new_merk| { - self.open_batch_transactional_merk_at_path( - &continue_storage_batch, - path.into(), - tx, - new_merk, - grove_version, - ) - }, - grove_version - ) - ); - - // let's build the write batch - let continued_pending_costs = cost_return_on_error!( - &mut cost, - self.db - .continue_write_batch(&mut write_batch, continue_storage_batch) - .map_err(|e| e.into()) - ); - - pending_costs.add_assign(continued_pending_costs); + let left_over_operations = cost_return_on_error!( + &mut cost, + self.apply_body( + ops, + Some(batch_apply_options.clone()), + &mut update_element_flags_function, + &mut split_removal_bytes_function, + |path, new_merk| { + self.open_batch_transactional_merk_at_path( + &storage_batch, + path.into(), + tx.as_ref(), + new_merk, + grove_version, + ) + }, + grove_version + ) + ); + // if we paused at the root height, the left over operations would be to replace + // a lot of leaf nodes in the root tree - // TODO: compute batch costs - cost_return_on_error!( - &mut cost, - self.db - .commit_db_write_batch(write_batch, pending_costs, Some(tx)) - .map_err(|e| e.into()) - ); - } else { - let left_over_operations = cost_return_on_error!( - &mut cost, - self.apply_body( - ops, - Some(batch_apply_options.clone()), - &mut update_element_flags_function, - &mut split_removal_bytes_function, - |path, new_merk| { - self.open_batch_merk_at_path( - &storage_batch, - path.into(), - new_merk, - grove_version, - ) - }, - grove_version - ) - ); + // let's build the write batch + let (mut write_batch, mut pending_costs) = cost_return_on_error!( + &mut cost, + self.db + .build_write_batch(storage_batch) + .map_err(|e| e.into()) + ); - // if we paused at the root height, the left over operations would be to replace - // a lot of leaf nodes in the root tree + let total_current_costs = cost.clone().add(pending_costs.clone()); - // let's build the write batch - let (mut write_batch, mut pending_costs) = cost_return_on_error!( - &mut cost, - self.db - .build_write_batch(storage_batch) - .map_err(|e| e.into()) - ); + // todo: estimate root costs - let total_current_costs = cost.clone().add(pending_costs.clone()); + // at this point we need to send the pending costs back + // we will get GroveDB a new set of GroveDBOps - // at this point we need to send the pending costs back - // we will get GroveDB a new set of GroveDBOps + let new_operations = cost_return_on_error_no_add!( + &cost, + add_on_operations(&total_current_costs, &left_over_operations) + ); - let new_operations = cost_return_on_error_no_add!( - &cost, - add_on_operations(&total_current_costs, &left_over_operations) - ); + // we are trying to finalize + batch_apply_options.batch_pause_height = None; - // we are trying to finalize - batch_apply_options.batch_pause_height = None; + let continue_storage_batch = StorageBatch::new(); - let continue_storage_batch = StorageBatch::new(); + cost_return_on_error!( + &mut cost, + self.continue_partial_apply_body( + left_over_operations, + new_operations, + Some(batch_apply_options), + update_element_flags_function, + split_removal_bytes_function, + |path, new_merk| { + self.open_batch_transactional_merk_at_path( + &continue_storage_batch, + path.into(), + tx.as_ref(), + new_merk, + grove_version, + ) + }, + grove_version + ) + ); - cost_return_on_error!( - &mut cost, - self.continue_partial_apply_body( - left_over_operations, - new_operations, - Some(batch_apply_options), - update_element_flags_function, - split_removal_bytes_function, - |path, new_merk| { - self.open_batch_merk_at_path( - &continue_storage_batch, - path.into(), - new_merk, - grove_version, - ) - }, - grove_version - ) - ); + // let's build the write batch + let continued_pending_costs = cost_return_on_error!( + &mut cost, + self.db + .continue_write_batch(&mut write_batch, continue_storage_batch) + .map_err(|e| e.into()) + ); - // let's build the write batch - let continued_pending_costs = cost_return_on_error!( - &mut cost, - self.db - .continue_write_batch(&mut write_batch, continue_storage_batch) - .map_err(|e| e.into()) - ); + pending_costs.add_assign(continued_pending_costs); - pending_costs.add_assign(continued_pending_costs); + // TODO: compute batch costs + cost_return_on_error!( + &mut cost, + self.db + .commit_db_write_batch(write_batch, pending_costs, Some(tx.as_ref())) + .map_err(|e| e.into()) + ); - // TODO: compute batch costs - cost_return_on_error!( - &mut cost, - self.db - .commit_db_write_batch(write_batch, pending_costs, None) - .map_err(|e| e.into()) - ); - } Ok(()).wrap_with_cost(cost) } diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 64d40342..32beec15 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -35,7 +35,7 @@ use crate::{ operations::proof::{GroveDBProof, LayerProof, ProveOptions}, query_result_type::{QueryResultElement, QueryResultElements, QueryResultType}, reference_path::ReferencePathType, - GroveDb, + GroveDb, Transaction, }; const GROVEDBG_ZIP: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/grovedbg.zip")); @@ -227,9 +227,10 @@ async fn fetch_node( }): Json>, ) -> Result>, AppError> { let db = state.get_snapshot(session_id).await?; + let tx = db.start_transaction(); let merk = db - .open_non_transactional_merk_at_path(path.as_slice().into(), None, GroveVersion::latest()) + .open_transactional_merk_at_path(path.as_slice().into(), &tx, None, GroveVersion::latest()) .unwrap()?; let node = merk.get_node_dbg(&key)?; @@ -249,9 +250,10 @@ async fn fetch_root_node( }): Json>, ) -> Result>, AppError> { let db = state.get_snapshot(session_id).await?; + let tx = db.start_transaction(); let merk = db - .open_non_transactional_merk_at_path(SubtreePath::empty(), None, GroveVersion::latest()) + .open_transactional_merk_at_path(SubtreePath::empty(), &tx, None, GroveVersion::latest()) .unwrap()?; let node = merk.get_root_node_dbg()?; @@ -289,6 +291,7 @@ async fn fetch_with_path_query( }): Json>, ) -> Result>, AppError> { let db = state.get_snapshot(session_id).await?; + let tx = db.start_transaction(); let path_query = path_query_to_grovedb(json_path_query); @@ -299,16 +302,21 @@ async fn fetch_with_path_query( true, false, QueryResultType::QueryPathKeyElementTrioResultType, - None, + Some(&tx), GroveVersion::latest(), ) .unwrap()? .0; - Ok(Json(query_result_to_grovedbg(&db, grovedb_query_result)?)) + Ok(Json(query_result_to_grovedbg( + &db, + &tx, + grovedb_query_result, + )?)) } fn query_result_to_grovedbg( db: &GroveDb, + tx: &Transaction, query_result: QueryResultElements, ) -> Result, crate::Error> { let mut result = Vec::new(); @@ -322,8 +330,9 @@ fn query_result_to_grovedbg( _ => { last_merk = Some(( path.clone(), - db.open_non_transactional_merk_at_path( + db.open_transactional_merk_at_path( path.as_slice().into(), + &tx, None, GroveVersion::latest(), ) diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index a746fde9..10618130 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -1,28 +1,26 @@ //! Get //! Implements functions in Element for getting -#[cfg(feature = "full")] use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::kv::KV; -#[cfg(feature = "full")] -use grovedb_merk::Merk; -#[cfg(feature = "full")] -use grovedb_merk::{ed::Decode, tree::TreeNodeInner}; -#[cfg(feature = "full")] +use grovedb_merk::{ + ed::Decode, + tree::{kv::KV, TreeNodeInner}, + Merk, +}; use grovedb_storage::StorageContext; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use crate::element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; -#[cfg(feature = "full")] -use crate::{Element, Error, Hash}; +use crate::{ + element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + Element, Error, Hash, +}; impl Element { - #[cfg(feature = "full")] /// Get an element from Merk under a key; path should be resolved and proper /// Merk should be loaded by this moment pub fn get<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( @@ -47,7 +45,6 @@ impl Element { }) } - #[cfg(feature = "full")] /// Get an element from Merk under a key; path should be resolved and proper /// Merk should be loaded by this moment pub fn get_optional<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( @@ -86,7 +83,6 @@ impl Element { Ok(element).wrap_with_cost(cost) } - #[cfg(feature = "full")] /// Get an element directly from storage under a key /// Merk does not need to be loaded /// Errors if element doesn't exist @@ -110,7 +106,6 @@ impl Element { }) } - #[cfg(feature = "full")] /// Get an element directly from storage under a key /// Merk does not need to be loaded pub fn get_optional_from_storage<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( @@ -202,7 +197,6 @@ impl Element { Ok(element).wrap_with_cost(cost) } - #[cfg(feature = "full")] /// Get an element from Merk under a key; path should be resolved and proper /// Merk should be loaded by this moment pub fn get_with_absolute_refs<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( @@ -234,7 +228,6 @@ impl Element { Ok(absolute_element).wrap_with_cost(cost) } - #[cfg(feature = "full")] /// Get an element's value hash from Merk under a key pub fn get_value_hash<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( merk: &Merk, @@ -263,7 +256,6 @@ impl Element { } } -#[cfg(feature = "full")] #[cfg(test)] mod tests { use grovedb_path::SubtreePath; @@ -276,8 +268,10 @@ mod tests { let grove_version = GroveVersion::latest(); let storage = TempStorage::new(); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); + let ctx = storage - .get_storage_context(SubtreePath::empty(), Some(&batch)) + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), &tx) .unwrap(); let mut merk = Merk::open_base( ctx, @@ -297,12 +291,12 @@ mod tests { .expect("expected successful insertion 2"); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .unwrap(); let ctx = storage - .get_storage_context(SubtreePath::empty(), None) + .get_transactional_storage_context(SubtreePath::empty(), None, &tx) .unwrap(); let mut merk = Merk::open_base( ctx, diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index ce0144a2..75a07d6d 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -532,7 +532,9 @@ mod tests { let grove_version = GroveVersion::latest(); let storage = TempStorage::new(); let batch = StorageBatch::new(); - let mut merk = empty_path_merk(&*storage, &batch, grove_version); + let tx = storage.start_transaction(); + + let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); Element::empty_tree() .insert(&mut merk, b"mykey", None, grove_version) @@ -544,12 +546,12 @@ mod tests { .expect("expected successful insertion 2"); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .unwrap(); let batch = StorageBatch::new(); - let mut merk = empty_path_merk(&*storage, &batch, grove_version); + let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); let (inserted, previous) = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() @@ -559,10 +561,10 @@ mod tests { assert_eq!(previous, Some(Element::new_item(b"value".to_vec())),); storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .unwrap(); - let merk = empty_path_merk_read_only(&*storage, grove_version); + let merk = empty_path_merk_read_only(&*storage, &tx, grove_version); assert_eq!( Element::get(&merk, b"another-key", true, grove_version) diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 7226db55..da45e061 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -37,7 +37,6 @@ use crate::{ QueryPathKeyElementTrioResultType, }, }, - util::{merk_optional_tx, merk_optional_tx_internal_error, storage_context_optional_tx}, Error, PathQuery, TransactionArg, }; #[cfg(feature = "full")] @@ -449,6 +448,8 @@ impl Element { args: PathQueryPushArgs, grove_version: &GroveVersion, ) -> CostResult<(), Error> { + use crate::util::{self, TxRef}; + check_grovedb_v0_with_cost!( "path_query_push", grove_version.grovedb_versions.element.path_query_push @@ -479,6 +480,9 @@ impl Element { decrease_limit_on_range_with_no_sub_elements, .. } = query_options; + + let tx = TxRef::new(storage, transaction); + if element.is_any_tree() { let mut path_vec = path.to_vec(); let key = cost_return_on_error_no_add!( @@ -531,88 +535,62 @@ impl Element { let subtree_path: SubtreePath<_> = path_vec.as_slice().into(); + let subtree = cost_return_on_error!( + &mut cost, + util::open_transactional_merk_at_path( + storage, + subtree_path, + tx.as_ref(), + None, + grove_version + ) + ); + match result_type { QueryElementResultType => { - merk_optional_tx!( - &mut cost, - storage, - subtree_path, - None, - transaction, - subtree, - grove_version, - { - results.push(QueryResultElement::ElementResultItem( - cost_return_on_error!( - &mut cost, - Element::get_with_absolute_refs( - &subtree, - path_vec.as_slice(), - subquery_path_last_key.as_slice(), - allow_cache, - grove_version, - ) - ), - )); - } - ); + results.push(QueryResultElement::ElementResultItem( + cost_return_on_error!( + &mut cost, + Element::get_with_absolute_refs( + &subtree, + path_vec.as_slice(), + subquery_path_last_key.as_slice(), + allow_cache, + grove_version, + ) + ), + )); } QueryKeyElementPairResultType => { - merk_optional_tx!( - &mut cost, - storage, - subtree_path, - None, - transaction, - subtree, - grove_version, - { - results.push(QueryResultElement::KeyElementPairResultItem( - ( - subquery_path_last_key.to_vec(), - cost_return_on_error!( - &mut cost, - Element::get_with_absolute_refs( - &subtree, - path_vec.as_slice(), - subquery_path_last_key.as_slice(), - allow_cache, - grove_version, - ) - ), - ), - )); - } - ); + results.push(QueryResultElement::KeyElementPairResultItem(( + subquery_path_last_key.to_vec(), + cost_return_on_error!( + &mut cost, + Element::get_with_absolute_refs( + &subtree, + path_vec.as_slice(), + subquery_path_last_key.as_slice(), + allow_cache, + grove_version, + ) + ), + ))); } QueryPathKeyElementTrioResultType => { - merk_optional_tx!( - &mut cost, - storage, - subtree_path, - None, - transaction, - subtree, - grove_version, - { - results.push( - QueryResultElement::PathKeyElementTrioResultItem(( - path_vec.iter().map(|p| p.to_vec()).collect(), - subquery_path_last_key.to_vec(), - cost_return_on_error!( - &mut cost, - Element::get_with_absolute_refs( - &subtree, - path_vec.as_slice(), - subquery_path_last_key.as_slice(), - allow_cache, - grove_version, - ) - ), - )), - ); - } - ); + results.push(QueryResultElement::PathKeyElementTrioResultItem(( + path_vec.iter().map(|p| p.to_vec()).collect(), + subquery_path_last_key.to_vec(), + cost_return_on_error!( + &mut cost, + Element::get_with_absolute_refs( + &subtree, + path_vec.as_slice(), + subquery_path_last_key.as_slice(), + allow_cache, + grove_version, + ) + ), + ))); } } } else { @@ -741,11 +719,17 @@ impl Element { add_element_function: fn(PathQueryPushArgs, &GroveVersion) -> CostResult<(), Error>, grove_version: &GroveVersion, ) -> CostResult<(), Error> { + use grovedb_storage::Storage; + + use crate::util::{self, TxRef}; + check_grovedb_v0_with_cost!( "query_item", grove_version.grovedb_versions.element.query_item ); + let tx = TxRef::new(storage, transaction); + let mut cost = OperationCost::default(); let subtree_path: SubtreePath<_> = path.into(); @@ -753,19 +737,19 @@ impl Element { if !item.is_range() { // this is a query on a key if let QueryItem::Key(key) = item { - let element_res = merk_optional_tx_internal_error!( + let subtree = cost_return_on_error!( &mut cost, - storage, - subtree_path, - None, - transaction, - subtree, - grove_version, - { - Element::get(&subtree, key, query_options.allow_cache, grove_version) - .unwrap_add_cost(&mut cost) - } + util::open_transactional_merk_at_path( + storage, + subtree_path, + tx.as_ref(), + None, + grove_version + ) ); + let element_res = + Element::get(&subtree, key, query_options.allow_cache, grove_version) + .unwrap_add_cost(&mut cost); match element_res { Ok(element) => { let (subquery_path, subquery) = @@ -822,74 +806,74 @@ impl Element { } } else { // this is a query on a range - storage_context_optional_tx!(storage, subtree_path, None, transaction, ctx, { - let ctx = ctx.unwrap_add_cost(&mut cost); - let mut iter = ctx.raw_iter(); + let ctx = storage + .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); - item.seek_for_iter(&mut iter, sized_query.query.left_to_right) - .unwrap_add_cost(&mut cost); + let mut iter = ctx.raw_iter(); - while item - .iter_is_valid_for_type(&iter, *limit, sized_query.query.left_to_right) + item.seek_for_iter(&mut iter, sized_query.query.left_to_right) + .unwrap_add_cost(&mut cost); + + while item + .iter_is_valid_for_type(&iter, *limit, sized_query.query.left_to_right) + .unwrap_add_cost(&mut cost) + { + let element = cost_return_on_error_no_add!( + &cost, + raw_decode( + iter.value() + .unwrap_add_cost(&mut cost) + .expect("if key exists then value should too"), + grove_version + ) + ); + let key = iter + .key() .unwrap_add_cost(&mut cost) - { - let element = cost_return_on_error_no_add!( - &cost, - raw_decode( - iter.value() - .unwrap_add_cost(&mut cost) - .expect("if key exists then value should too"), - grove_version - ) - ); - let key = iter - .key() - .unwrap_add_cost(&mut cost) - .expect("key should exist"); - let (subquery_path, subquery) = - Self::subquery_paths_and_value_for_sized_query(sized_query, key); - let result_with_cost = add_element_function( - PathQueryPushArgs { - storage, - transaction, - key: Some(key), - element, - path, - subquery_path, - subquery, - left_to_right: sized_query.query.left_to_right, - query_options, - result_type, - results, - limit, - offset, - }, - grove_version, - ); - let result = result_with_cost.unwrap_add_cost(&mut cost); - match result { - Ok(x) => x, - Err(e) => { - if !query_options.error_if_intermediate_path_tree_not_present { - match e { - Error::PathKeyNotFound(_) - | Error::PathParentLayerNotFound(_) => (), - _ => return Err(e).wrap_with_cost(cost), - } - } else { - return Err(e).wrap_with_cost(cost); + .expect("key should exist"); + let (subquery_path, subquery) = + Self::subquery_paths_and_value_for_sized_query(sized_query, key); + let result_with_cost = add_element_function( + PathQueryPushArgs { + storage, + transaction, + key: Some(key), + element, + path, + subquery_path, + subquery, + left_to_right: sized_query.query.left_to_right, + query_options, + result_type, + results, + limit, + offset, + }, + grove_version, + ); + let result = result_with_cost.unwrap_add_cost(&mut cost); + match result { + Ok(x) => x, + Err(e) => { + if !query_options.error_if_intermediate_path_tree_not_present { + match e { + Error::PathKeyNotFound(_) | Error::PathParentLayerNotFound(_) => (), + _ => return Err(e).wrap_with_cost(cost), } + } else { + return Err(e).wrap_with_cost(cost); } } - if sized_query.query.left_to_right { - iter.next().unwrap_add_cost(&mut cost); - } else { - iter.prev().unwrap_add_cost(&mut cost); - } - cost.seek_count += 1; } - Ok(()) - }) + if sized_query.query.left_to_right { + iter.next().unwrap_add_cost(&mut cost); + } else { + iter.prev().unwrap_add_cost(&mut cost); + } + cost.seek_count += 1; + } + Ok(()) } .wrap_with_cost(cost) } @@ -1197,9 +1181,12 @@ mod tests { let batch = StorageBatch::new(); let storage = &db.db; + let tx = db.start_transaction(); + let mut merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -1307,11 +1294,13 @@ mod tests { let db = make_test_grovedb(grove_version); let batch = StorageBatch::new(); - + let tx = db.start_transaction(); let storage = &db.db; + let mut merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 74cfe807..cc877ba6 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -601,10 +601,11 @@ mod test { let storage = RocksDbStorage::default_rocksdb_with_path(tmp_dir.path()) .expect("cannot open rocksdb storage"); let batch = StorageBatch::new(); + let tx = storage.start_transaction(); let mut merk = Merk::open_base( storage - .get_storage_context(EMPTY_PATH, Some(&batch)) + .get_transactional_storage_context(EMPTY_PATH, Some(&batch), &tx) .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, @@ -619,13 +620,15 @@ mod test { // this consumes the batch so storage contexts and merks will be dropped storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .unwrap(); // Reopen merk: this time, only root node is loaded to memory let merk = Merk::open_base( - storage.get_storage_context(EMPTY_PATH, None).unwrap(), + storage + .get_transactional_storage_context(EMPTY_PATH, None, &tx) + .unwrap(), false, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index 4c409b99..f9c350d2 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -536,7 +536,9 @@ mod test { // Open a merk and insert 10 elements. let storage = TempStorage::new(); let batch = StorageBatch::new(); - let mut merk = empty_path_merk(&*storage, &batch, grove_version); + let tx = storage.start_transaction(); + + let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); let merk_batch = make_batch_seq(1..10); merk.apply::<_, Vec<_>>(merk_batch.as_slice(), &[], None, grove_version) @@ -545,12 +547,12 @@ mod test { // this consumes the batch so storage contexts and merks will be dropped storage - .commit_multi_context_batch(batch, None) + .commit_multi_context_batch(batch, Some(&tx)) .unwrap() .unwrap(); // Reopen merk: this time, only root node is loaded to memory - let merk = empty_path_merk_read_only(&*storage, grove_version); + let merk = empty_path_merk_read_only(&*storage, &tx, grove_version); // To simulate worst case, we need to pick a node that: // 1. Is not in memory diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 9467d686..1c2638ec 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -193,10 +193,7 @@ use grovedb_storage::rocksdb_storage::PrefixedRocksDbImmediateStorageContext; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; #[cfg(feature = "full")] -use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, - StorageBatch, -}; +use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; #[cfg(feature = "full")] use grovedb_storage::{Storage, StorageContext}; #[cfg(feature = "full")] @@ -209,6 +206,7 @@ pub use query::{PathQuery, SizedQuery}; use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; +use util::TxRef; #[cfg(feature = "full")] use crate::element::helpers::raw_decode; @@ -217,8 +215,6 @@ pub use crate::error::Error; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; #[cfg(feature = "full")] -use crate::util::{root_merk_optional_tx, storage_context_optional_tx}; -#[cfg(feature = "full")] use crate::Error::MerkError; #[cfg(feature = "full")] @@ -275,59 +271,7 @@ impl GroveDb { where B: AsRef<[u8]> + 'b, { - let mut cost = OperationCost::default(); - - let storage = self - .db - .get_transactional_storage_context(path.clone(), batch, tx) - .unwrap_add_cost(&mut cost); - if let Some((parent_path, parent_key)) = path.derive_parent() { - let parent_storage = self - .db - .get_transactional_storage_context(parent_path.clone(), batch, tx) - .unwrap_add_cost(&mut cost); - let element = cost_return_on_error!( - &mut cost, - Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( - |e| { - Error::InvalidParentLayerPath(format!( - "could not get key {} for parent {:?} of subtree: {}", - hex::encode(parent_key), - DebugByteVectors(parent_path.to_vec()), - e - )) - } - ) - ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) - }) - .add_cost(cost) - } else { - Err(Error::CorruptedPath( - "cannot open a subtree as parent exists but is not a tree".to_string(), - )) - .wrap_with_cost(cost) - } - } else { - Merk::open_base( - storage, - false, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) - } + util::open_transactional_merk_at_path(&self.db, path, tx, batch, grove_version) } /// Opens a Merk at given path for with direct write access. Intended for @@ -392,75 +336,31 @@ impl GroveDb { } } - /// Opens the non-transactional Merk at the given path. Returns CostResult. - fn open_non_transactional_merk_at_path<'db, 'b, B>( + /// Creates a checkpoint + pub fn create_checkpoint>(&self, path: P) -> Result<(), Error> { + self.db.create_checkpoint(path).map_err(|e| e.into()) + } + + fn open_root_merk<'tx, 'db>( &'db self, - path: SubtreePath<'b, B>, - batch: Option<&'db StorageBatch>, + tx: &'tx Transaction<'db>, grove_version: &GroveVersion, - ) -> CostResult, Error> - where - B: AsRef<[u8]> + 'b, - { - let mut cost = OperationCost::default(); - - let storage = self - .db - .get_storage_context(path.clone(), batch) - .unwrap_add_cost(&mut cost); - - if let Some((parent_path, parent_key)) = path.derive_parent() { - let parent_storage = self - .db - .get_storage_context(parent_path.clone(), batch) - .unwrap_add_cost(&mut cost); - let element = cost_return_on_error!( - &mut cost, - Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err( - |e| { - Error::InvalidParentLayerPath(format!( - "could not get key {} for parent {:?} of subtree: {}", - hex::encode(parent_key), - DebugByteVectors(parent_path.to_vec()), - e - )) - } - ) - ); - let is_sum_tree = element.is_sum_tree(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { - Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), + ) -> CostResult>, Error> { + self.db + .get_transactional_storage_context(SubtreePath::empty(), None, tx) + .flat_map(|storage_ctx| { + grovedb_merk::Merk::open_base( + storage_ctx, + false, + Some(Element::value_defined_cost_for_serialized_value), grove_version, ) - .map_err(|_| { - Error::CorruptedData("cannot open a subtree with given root key".to_owned()) + .map(|merk_res| { + merk_res.map_err(|_| { + crate::Error::CorruptedData("cannot open a subtree".to_owned()) + }) }) - .add_cost(cost) - } else { - Err(Error::CorruptedPath( - "cannot open a subtree as parent exists but is not a tree".to_string(), - )) - .wrap_with_cost(cost) - } - } else { - Merk::open_base( - storage, - false, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version, - ) - .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) - .add_cost(cost) - } - } - - /// Creates a checkpoint - pub fn create_checkpoint>(&self, path: P) -> Result<(), Error> { - self.db.create_checkpoint(path).map_err(|e| e.into()) + }) } /// Returns root key of GroveDb. @@ -469,23 +369,11 @@ impl GroveDb { &self, transaction: TransactionArg, grove_version: &GroveVersion, - ) -> CostResult, Error> { - let mut cost = OperationCost { - ..Default::default() - }; + ) -> CostResult>, Error> { + let tx = TxRef::new(&self.db, transaction); - root_merk_optional_tx!( - &mut cost, - self.db, - None, - transaction, - subtree, - grove_version, - { - let root_key = subtree.root_key().unwrap(); - Ok(root_key).wrap_with_cost(cost) - } - ) + self.open_root_merk(tx.as_ref(), grove_version) + .map_ok(|merk| merk.root_key()) } /// Returns root hash of GroveDb. @@ -495,22 +383,12 @@ impl GroveDb { transaction: TransactionArg, grove_version: &GroveVersion, ) -> CostResult { - let mut cost = OperationCost { - ..Default::default() - }; + let tx = TxRef::new(&self.db, transaction); + let mut cost = Default::default(); - root_merk_optional_tx!( - &mut cost, - self.db, - None, - transaction, - subtree, - grove_version, - { - let root_hash = subtree.root_hash().unwrap_add_cost(&mut cost); - Ok(root_hash).wrap_with_cost(cost) - } - ) + let subtree = + cost_return_on_error!(&mut cost, self.open_root_merk(tx.as_ref(), grove_version)); + subtree.root_hash().map(Ok).add_cost(cost) } /// Method to propagate updated subtree key changes one level up inside a @@ -620,57 +498,6 @@ impl GroveDb { Ok(()).wrap_with_cost(cost) } - /// Method to propagate updated subtree key changes one level up - fn propagate_changes_without_transaction<'b, B: AsRef<[u8]>>( - &self, - mut merk_cache: HashMap, Merk>, - path: SubtreePath<'b, B>, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult<(), Error> { - let mut cost = OperationCost::default(); - - let mut child_tree = cost_return_on_error_no_add!( - &cost, - merk_cache - .remove(&path) - .ok_or(Error::CorruptedCodeExecution( - "Merk Cache should always contain the last path", - )) - ); - - let mut current_path: SubtreePath = path; - - while let Some((parent_path, parent_key)) = current_path.derive_parent() { - let mut parent_tree: Merk = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path( - parent_path.clone(), - Some(batch), - grove_version - ) - ); - let (root_hash, root_key, sum) = cost_return_on_error!( - &mut cost, - child_tree.root_hash_key_and_sum().map_err(Error::MerkError) - ); - cost_return_on_error!( - &mut cost, - Self::update_tree_item_preserve_flag( - &mut parent_tree, - parent_key, - root_key, - root_hash, - sum, - grove_version, - ) - ); - child_tree = parent_tree; - current_path = parent_path; - } - Ok(()).wrap_with_cost(cost) - } - /// Updates a tree item and preserves flags. Returns CostResult. pub(crate) fn update_tree_item_preserve_flag<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( parent_tree: &mut Merk, @@ -936,186 +763,18 @@ impl GroveDb { allow_cache: bool, grove_version: &GroveVersion, ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { - if let Some(transaction) = transaction { - let root_merk = self - .open_transactional_merk_at_path( - SubtreePath::empty(), - transaction, - None, - grove_version, - ) - .unwrap()?; - self.verify_merk_and_submerks_in_transaction( - root_merk, - &SubtreePath::empty(), - None, - transaction, - verify_references, - allow_cache, - grove_version, - ) - } else { - let root_merk = self - .open_non_transactional_merk_at_path(SubtreePath::empty(), None, grove_version) - .unwrap()?; - self.verify_merk_and_submerks( - root_merk, - &SubtreePath::empty(), - None, - verify_references, - allow_cache, - grove_version, - ) - } - } - - /// Verifies that the root hash of the given merk and all submerks match - /// those of the merk and submerks at the given path. Returns any issues. - fn verify_merk_and_submerks<'db, B: AsRef<[u8]>, S: StorageContext<'db>>( - &'db self, - merk: Merk, - path: &SubtreePath, - batch: Option<&'db StorageBatch>, - verify_references: bool, - allow_cache: bool, - grove_version: &GroveVersion, - ) -> Result>, (CryptoHash, CryptoHash, CryptoHash)>, Error> { - let mut all_query = Query::new(); - all_query.insert_all(); + let tx = TxRef::new(&self.db, transaction); + let root_merk = self.open_root_merk(tx.as_ref(), grove_version).unwrap()?; - let mut issues = HashMap::new(); - let mut element_iterator = KVIterator::new(merk.storage.raw_iter(), &all_query).unwrap(); - - while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { - let element = raw_decode(&element_value, grove_version)?; - match element { - Element::SumTree(..) | Element::Tree(..) => { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, - allow_cache, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData(format!( - "expected merk to contain value at key {} for {}", - hex_to_ascii(&key), - element.type_str() - )))?; - let new_path = path.derive_owned_with_child(key); - let new_path_ref = SubtreePath::from(&new_path); - - let inner_merk = self - .open_non_transactional_merk_at_path( - new_path_ref.clone(), - batch, - grove_version, - ) - .unwrap()?; - let root_hash = inner_merk.root_hash().unwrap(); - - let actual_value_hash = value_hash(&kv_value).unwrap(); - let combined_value_hash = combine_hash(&actual_value_hash, &root_hash).unwrap(); - - if combined_value_hash != element_value_hash { - issues.insert( - new_path.to_vec(), - (root_hash, combined_value_hash, element_value_hash), - ); - } - issues.extend(self.verify_merk_and_submerks( - inner_merk, - &new_path_ref, - batch, - verify_references, - true, - grove_version, - )?); - } - Element::Item(..) - | Element::SumItem(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItemWithBackwardsReferences(..) => { - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, - allow_cache, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData(format!( - "expected merk to contain value at key {} for {}", - hex_to_ascii(&key), - element.type_str() - )))?; - let actual_value_hash = value_hash(&kv_value).unwrap(); - if actual_value_hash != element_value_hash { - issues.insert( - path.derive_owned_with_child(key).to_vec(), - (actual_value_hash, element_value_hash, actual_value_hash), - ); - } - } - Element::Reference(ref reference_path, ..) - | Element::BidirectionalReference(ref reference_path, ..) => { - // Skip this whole check if we don't `verify_references` - if !verify_references { - continue; - } - - // Merk we're checking: - let (kv_value, element_value_hash) = merk - .get_value_and_value_hash( - &key, - allow_cache, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .unwrap() - .map_err(MerkError)? - .ok_or(Error::CorruptedData(format!( - "expected merk to contain value at key {} for reference", - hex_to_ascii(&key) - )))?; - - let referenced_value_hash = { - let full_path = path_from_reference_path_type( - reference_path.clone(), - &path.to_vec(), - Some(&key), - )?; - let item = self - .follow_reference( - (full_path.as_slice()).into(), - allow_cache, - None, - grove_version, - ) - .unwrap()?; - item.value_hash(grove_version).unwrap()? - }; - - // Take the current item (reference) hash and combine it with referenced value's - // hash - - let self_actual_value_hash = value_hash(&kv_value).unwrap(); - let combined_value_hash = - combine_hash(&self_actual_value_hash, &referenced_value_hash).unwrap(); - - if combined_value_hash != element_value_hash { - issues.insert( - path.derive_owned_with_child(key).to_vec(), - (combined_value_hash, element_value_hash, combined_value_hash), - ); - } - } - } - } - Ok(issues) + self.verify_merk_and_submerks_in_transaction( + root_merk, + &SubtreePath::empty(), + None, + tx.as_ref(), + verify_references, + allow_cache, + grove_version, + ) } fn verify_merk_and_submerks_in_transaction<'db, B: AsRef<[u8]>, S: StorageContext<'db>>( diff --git a/grovedb/src/operations/auxiliary.rs b/grovedb/src/operations/auxiliary.rs index 516796ed..84c7fee5 100644 --- a/grovedb/src/operations/auxiliary.rs +++ b/grovedb/src/operations/auxiliary.rs @@ -28,22 +28,16 @@ //! Auxiliary operations -#[cfg(feature = "full")] use grovedb_costs::{ - cost_return_on_error, cost_return_on_error_no_add, - storage_cost::key_value_cost::KeyValueStorageCost, CostResult, CostsExt, OperationCost, + cost_return_on_error, storage_cost::key_value_cost::KeyValueStorageCost, CostResult, CostsExt, + OperationCost, }; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] -use grovedb_storage::StorageContext; -use grovedb_storage::{Storage, StorageBatch}; +use grovedb_storage::{Storage, StorageBatch, StorageContext}; use grovedb_version::version::GroveVersion; -use crate::util::storage_context_optional_tx; -#[cfg(feature = "full")] -use crate::{util::meta_storage_context_optional_tx, Element, Error, GroveDb, TransactionArg}; +use crate::{util::TxRef, Element, Error, GroveDb, TransactionArg}; -#[cfg(feature = "full")] impl GroveDb { /// Put op for aux storage pub fn put_aux>( @@ -55,20 +49,22 @@ impl GroveDb { ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let batch = StorageBatch::new(); + let tx = TxRef::new(&self.db, transaction); - meta_storage_context_optional_tx!(self.db, Some(&batch), transaction, aux_storage, { - cost_return_on_error_no_add!( - &cost, - aux_storage - .unwrap_add_cost(&mut cost) - .put_aux(key.as_ref(), value, cost_info) - .unwrap_add_cost(&mut cost) - .map_err(|e| e.into()) - ); - }); + let storage = self + .db + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), tx.as_ref()) + .unwrap_add_cost(&mut cost); + + cost_return_on_error!( + &mut cost, + storage + .put_aux(key.as_ref(), value, cost_info) + .map_err(|e| e.into()) + ); self.db - .commit_multi_context_batch(batch, transaction) + .commit_multi_context_batch(batch, Some(tx.as_ref())) .add_cost(cost) .map_err(Into::into) } @@ -82,20 +78,22 @@ impl GroveDb { ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let batch = StorageBatch::new(); + let tx = TxRef::new(&self.db, transaction); + + let storage = self + .db + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), tx.as_ref()) + .unwrap_add_cost(&mut cost); - meta_storage_context_optional_tx!(self.db, Some(&batch), transaction, aux_storage, { - cost_return_on_error_no_add!( - &cost, - aux_storage - .unwrap_add_cost(&mut cost) - .delete_aux(key.as_ref(), cost_info) - .unwrap_add_cost(&mut cost) - .map_err(|e| e.into()) - ); - }); + cost_return_on_error!( + &mut cost, + storage + .delete_aux(key.as_ref(), cost_info) + .map_err(|e| e.into()) + ); self.db - .commit_multi_context_batch(batch, transaction) + .commit_multi_context_batch(batch, Some(tx.as_ref())) .add_cost(cost) .map_err(Into::into) } @@ -107,19 +105,15 @@ impl GroveDb { transaction: TransactionArg, ) -> CostResult>, Error> { let mut cost = OperationCost::default(); + let batch = StorageBatch::new(); + let tx = TxRef::new(&self.db, transaction); - meta_storage_context_optional_tx!(self.db, None, transaction, aux_storage, { - let value = cost_return_on_error_no_add!( - &cost, - aux_storage - .unwrap_add_cost(&mut cost) - .get_aux(key) - .unwrap_add_cost(&mut cost) - .map_err(|e| e.into()) - ); - - Ok(value).wrap_with_cost(cost) - }) + let storage = self + .db + .get_transactional_storage_context(SubtreePath::empty(), Some(&batch), tx.as_ref()) + .unwrap_add_cost(&mut cost); + + storage.get_aux(key).map_err(|e| e.into()).add_cost(cost) } // TODO: dumb traversal should not be tolerated @@ -134,6 +128,8 @@ impl GroveDb { ) -> CostResult>>, Error> { let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + // TODO: remove conversion to vec; // However, it's not easy for a reason: // new keys to enqueue are taken from raw iterator which returns Vec; @@ -151,20 +147,22 @@ impl GroveDb { while let Some(q) = queue.pop() { let subtree_path: SubtreePath> = q.as_slice().into(); // Get the correct subtree with q_ref as path - storage_context_optional_tx!(self.db, subtree_path, None, transaction, storage, { - let storage = storage.unwrap_add_cost(&mut cost); - let mut raw_iter = Element::iterator(storage.raw_iter()).unwrap_add_cost(&mut cost); - while let Some((key, value)) = - cost_return_on_error!(&mut cost, raw_iter.next_element(grove_version)) - { - if value.is_any_tree() { - let mut sub_path = q.clone(); - sub_path.push(key.to_vec()); - queue.push(sub_path.clone()); - result.push(sub_path); - } + let storage = self + .db + .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .unwrap_add_cost(&mut cost); + + let mut raw_iter = Element::iterator(storage.raw_iter()).unwrap_add_cost(&mut cost); + while let Some((key, value)) = + cost_return_on_error!(&mut cost, raw_iter.next_element(grove_version)) + { + if value.is_any_tree() { + let mut sub_path = q.clone(); + sub_path.push(key.to_vec()); + queue.push(sub_path.clone()); + result.push(sub_path); } - }) + } } Ok(result).wrap_with_cost(cost) } diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 9244c60b..a3571499 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -2,44 +2,34 @@ #[cfg(feature = "estimated_costs")] mod average_case; -#[cfg(feature = "full")] mod delete_up_tree; #[cfg(feature = "estimated_costs")] mod worst_case; -#[cfg(feature = "full")] use std::collections::{BTreeSet, HashMap}; -#[cfg(feature = "full")] pub use delete_up_tree::DeleteUpTreeOptions; -#[cfg(feature = "full")] use grovedb_costs::{ cost_return_on_error, storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{proofs::Query, KVIterator}; -#[cfg(feature = "full")] -use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; +use grovedb_merk::{proofs::Query, Error as MerkError, KVIterator, Merk, MerkOptions}; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext}, - Storage, StorageBatch, StorageContext, + rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch, StorageContext, }; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -#[cfg(feature = "full")] use crate::{ batch::{GroveOp, QualifiedGroveDbOp}, - util::storage_context_with_parent_optional_tx, + raw_decode, + util::TxRef, Element, ElementFlags, Error, GroveDb, Transaction, TransactionArg, }; -use crate::{raw_decode, util::merk_optional_tx_path_not_empty}; -#[cfg(feature = "full")] #[derive(Clone)] /// Clear options pub struct ClearOptions { @@ -52,7 +42,6 @@ pub struct ClearOptions { pub trying_to_clear_with_subtrees_returns_error: bool, } -#[cfg(feature = "full")] impl Default for ClearOptions { fn default() -> Self { ClearOptions { @@ -63,7 +52,6 @@ impl Default for ClearOptions { } } -#[cfg(feature = "full")] #[derive(Clone)] /// Delete options pub struct DeleteOptions { @@ -77,7 +65,6 @@ pub struct DeleteOptions { pub validate_tree_at_path_exists: bool, } -#[cfg(feature = "full")] impl Default for DeleteOptions { fn default() -> Self { DeleteOptions { @@ -89,7 +76,6 @@ impl Default for DeleteOptions { } } -#[cfg(feature = "full")] impl DeleteOptions { fn as_merk_options(&self) -> MerkOptions { MerkOptions { @@ -98,7 +84,6 @@ impl DeleteOptions { } } -#[cfg(feature = "full")] impl GroveDb { /// Delete an element at a specified subtree path and key. pub fn delete<'b, B, P>( @@ -121,12 +106,14 @@ impl GroveDb { let options = options.unwrap_or_default(); let batch = StorageBatch::new(); + let tx = TxRef::new(&self.db, transaction); + let collect_costs = self .delete_internal( path.into(), key, &options, - transaction, + tx.as_ref(), &mut |_, removed_key_bytes, removed_value_bytes| { Ok(( BasicStorageRemoval(removed_key_bytes), @@ -158,7 +145,8 @@ impl GroveDb { B: AsRef<[u8]> + 'b, P: Into>, { - self.clear_subtree_with_costs(path, options, transaction, grove_version) + let tx = TxRef::new(&self.db, transaction); + self.clear_subtree_with_costs(path, options, tx.as_ref(), grove_version) .unwrap() } @@ -170,7 +158,7 @@ impl GroveDb { &self, path: P, options: Option, - transaction: TransactionArg, + transaction: &Transaction, grove_version: &GroveVersion, ) -> CostResult where @@ -192,114 +180,44 @@ impl GroveDb { let options = options.unwrap_or_default(); - if let Some(transaction) = transaction { - let mut merk_to_clear = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - subtree_path.clone(), - transaction, - Some(&batch), - grove_version, - ) - ); - - if options.check_for_subtrees { - let mut all_query = Query::new(); - all_query.insert_all(); - - let mut element_iterator = - KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); - - // delete all nested subtrees - while let Some((key, element_value)) = - element_iterator.next_kv().unwrap_add_cost(&mut cost) - { - let element = raw_decode(&element_value, grove_version).unwrap(); - if element.is_any_tree() { - if options.allow_deleting_subtrees { - cost_return_on_error!( - &mut cost, - self.delete( - subtree_path.clone(), - key.as_slice(), - Some(DeleteOptions { - allow_deleting_non_empty_trees: true, - deleting_non_empty_trees_returns_error: false, - ..Default::default() - }), - Some(transaction), - grove_version, - ) - ); - } else if options.trying_to_clear_with_subtrees_returns_error { - return Err(Error::ClearingTreeWithSubtreesNotAllowed( - "options do not allow to clear this merk tree as it contains \ - subtrees", - )) - .wrap_with_cost(cost); - } else { - return Ok(false).wrap_with_cost(cost); - } - } - } - } - - // delete non subtree values - cost_return_on_error!(&mut cost, merk_to_clear.clear().map_err(Error::MerkError)); - - // propagate changes - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - merk_cache.insert(subtree_path.clone(), merk_to_clear); - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_transaction( - merk_cache, - subtree_path.clone(), - transaction, - &batch, - grove_version, - ) - ); - } else { - let mut merk_to_clear = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path( - subtree_path.clone(), - Some(&batch), - grove_version - ) - ); + let mut merk_to_clear = cost_return_on_error!( + &mut cost, + self.open_transactional_merk_at_path( + subtree_path.clone(), + transaction, + Some(&batch), + grove_version, + ) + ); - if options.check_for_subtrees { - let mut all_query = Query::new(); - all_query.insert_all(); + if options.check_for_subtrees { + let mut all_query = Query::new(); + all_query.insert_all(); - let mut element_iterator = - KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); + let mut element_iterator = + KVIterator::new(merk_to_clear.storage.raw_iter(), &all_query).unwrap(); - // delete all nested subtrees - while let Some((key, element_value)) = - element_iterator.next_kv().unwrap_add_cost(&mut cost) - { - let element = raw_decode(&element_value, grove_version).unwrap(); + // delete all nested subtrees + while let Some((key, element_value)) = + element_iterator.next_kv().unwrap_add_cost(&mut cost) + { + let element = raw_decode(&element_value, grove_version).unwrap(); + if element.is_any_tree() { if options.allow_deleting_subtrees { - if element.is_any_tree() { - cost_return_on_error!( - &mut cost, - self.delete( - subtree_path.clone(), - key.as_slice(), - Some(DeleteOptions { - allow_deleting_non_empty_trees: true, - deleting_non_empty_trees_returns_error: false, - ..Default::default() - }), - None, - grove_version, - ) - ); - } + cost_return_on_error!( + &mut cost, + self.delete( + subtree_path.clone(), + key.as_slice(), + Some(DeleteOptions { + allow_deleting_non_empty_trees: true, + deleting_non_empty_trees_returns_error: false, + ..Default::default() + }), + Some(transaction), + grove_version, + ) + ); } else if options.trying_to_clear_with_subtrees_returns_error { return Err(Error::ClearingTreeWithSubtreesNotAllowed( "options do not allow to clear this merk tree as it contains subtrees", @@ -310,29 +228,30 @@ impl GroveDb { } } } + } - // delete non subtree values - cost_return_on_error!(&mut cost, merk_to_clear.clear().map_err(Error::MerkError)); + // delete non subtree values + cost_return_on_error!(&mut cost, merk_to_clear.clear().map_err(Error::MerkError)); - // propagate changes - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - merk_cache.insert(subtree_path.clone(), merk_to_clear); - cost_return_on_error!( - &mut cost, - self.propagate_changes_without_transaction( - merk_cache, - subtree_path.clone(), - &batch, - grove_version, - ) - ); - } + // propagate changes + let mut merk_cache: HashMap, Merk> = + HashMap::default(); + merk_cache.insert(subtree_path.clone(), merk_to_clear); + cost_return_on_error!( + &mut cost, + self.propagate_changes_with_transaction( + merk_cache, + subtree_path.clone(), + transaction, + &batch, + grove_version, + ) + ); cost_return_on_error!( &mut cost, self.db - .commit_multi_context_batch(batch, transaction) + .commit_multi_context_batch(batch, Some(transaction)) .map_err(Into::into) ); @@ -368,12 +287,14 @@ impl GroveDb { let options = options.unwrap_or_default(); let batch = StorageBatch::new(); + let tx = TxRef::new(&self.db, transaction); + let collect_costs = self .delete_internal( path, key, &options, - transaction, + tx.as_ref(), &mut |value, removed_key_bytes, removed_value_bytes| { let mut element = Element::deserialize(value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; @@ -480,11 +401,13 @@ impl GroveDb { ..Default::default() }; + let tx = TxRef::new(&self.db, transaction); + self.delete_internal( path, key, &options, - transaction, + tx.as_ref(), &mut |value, removed_key_bytes, removed_value_bytes| { let mut element = Element::deserialize(value.as_slice(), grove_version) .map_err(|e| MerkError::ClientCorruptionError(e.to_string()))?; @@ -527,6 +450,8 @@ impl GroveDb { let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); + if path.is_root() { // Attempt to delete a root tree leaf Err(Error::InvalidPath( @@ -577,21 +502,20 @@ impl GroveDb { _ => None, }) .collect::>(); - let mut is_empty = merk_optional_tx_path_not_empty!( + let subtree = cost_return_on_error!( &mut cost, - self.db, - SubtreePath::from(&subtree_merk_path), - None, - transaction, - subtree, - grove_version, - { - subtree - .is_empty_tree_except(batch_deleted_keys) - .unwrap_add_cost(&mut cost) - } + self.open_transactional_merk_at_path( + SubtreePath::from(&subtree_merk_path), + tx.as_ref(), + None, + grove_version, + ) ); + let mut is_empty = subtree + .is_empty_tree_except(batch_deleted_keys) + .unwrap_add_cost(&mut cost); + // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { @@ -632,45 +556,6 @@ impl GroveDb { } fn delete_internal>( - &self, - path: SubtreePath, - key: &[u8], - options: &DeleteOptions, - transaction: TransactionArg, - sectioned_removal: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - MerkError, - >, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult { - if let Some(transaction) = transaction { - self.delete_internal_on_transaction( - path, - key, - options, - transaction, - sectioned_removal, - batch, - grove_version, - ) - } else { - self.delete_internal_without_transaction( - path, - key, - options, - sectioned_removal, - batch, - grove_version, - ) - } - } - - fn delete_internal_on_transaction>( &self, path: SubtreePath, key: &[u8], @@ -867,134 +752,8 @@ impl GroveDb { Ok(true).wrap_with_cost(cost) } - - fn delete_internal_without_transaction>( - &self, - path: SubtreePath, - key: &[u8], - options: &DeleteOptions, - sectioned_removal: &mut impl FnMut( - &Vec, - u32, - u32, - ) -> Result< - (StorageRemovedBytes, StorageRemovedBytes), - MerkError, - >, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult { - check_grovedb_v0_with_cost!( - "delete_internal_without_transaction", - grove_version - .grovedb_versions - .operations - .delete - .delete_internal_without_transaction - ); - - let mut cost = OperationCost::default(); - - let element = cost_return_on_error!( - &mut cost, - self.get_raw(path.clone(), key.as_ref(), None, grove_version) - ); - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - let mut subtree_to_delete_from = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path(path.clone(), Some(batch), grove_version) - ); - let uses_sum_tree = subtree_to_delete_from.is_sum_tree; - if element.is_any_tree() { - let subtree_merk_path = path.derive_owned_with_child(key); - let subtree_of_tree_we_are_deleting = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path( - SubtreePath::from(&subtree_merk_path), - Some(batch), - grove_version, - ) - ); - let is_empty = subtree_of_tree_we_are_deleting - .is_empty_tree() - .unwrap_add_cost(&mut cost); - - if !options.allow_deleting_non_empty_trees && !is_empty { - return if options.deleting_non_empty_trees_returns_error { - Err(Error::DeletingNonEmptyTree( - "trying to do a delete operation for a non empty tree, but options not \ - allowing this", - )) - .wrap_with_cost(cost) - } else { - Ok(false).wrap_with_cost(cost) - }; - } else { - if !is_empty { - let subtrees_paths = cost_return_on_error!( - &mut cost, - self.find_subtrees( - &SubtreePath::from(&subtree_merk_path), - None, - grove_version - ) - ); - // TODO: dumb traversal should not be tolerated - for subtree_path in subtrees_paths.into_iter().rev() { - let p: SubtreePath<_> = subtree_path.as_slice().into(); - let mut inner_subtree_to_delete_from = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path(p, Some(batch), grove_version) - ); - cost_return_on_error!( - &mut cost, - inner_subtree_to_delete_from.clear().map_err(|e| { - Error::CorruptedData(format!( - "unable to cleanup tree from storage: {e}", - )) - }) - ); - } - } - cost_return_on_error!( - &mut cost, - Element::delete_with_sectioned_removal_bytes( - &mut subtree_to_delete_from, - key, - Some(options.as_merk_options()), - true, - uses_sum_tree, - sectioned_removal, - grove_version, - ) - ); - } - } else { - cost_return_on_error!( - &mut cost, - Element::delete_with_sectioned_removal_bytes( - &mut subtree_to_delete_from, - key, - Some(options.as_merk_options()), - false, - uses_sum_tree, - sectioned_removal, - grove_version, - ) - ); - } - merk_cache.insert(path.clone(), subtree_to_delete_from); - cost_return_on_error!( - &mut cost, - self.propagate_changes_without_transaction(merk_cache, path, batch, grove_version) - ); - - Ok(true).wrap_with_cost(cost) - } } -#[cfg(feature = "full")] #[cfg(test)] mod tests { use grovedb_costs::{ @@ -1885,9 +1644,13 @@ mod tests { .unwrap() .unwrap(); assert!(!matches!(key1_tree, Element::Tree(None, _))); + + let tx = db.start_transaction(); + let key1_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, None, grove_version, ) @@ -1950,8 +1713,9 @@ mod tests { assert!(matches!(key1_tree, Element::Tree(None, _))); let key1_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, None, grove_version, ) diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index b6289699..fa58f94b 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -2,38 +2,31 @@ #[cfg(feature = "estimated_costs")] mod average_case; -#[cfg(feature = "full")] mod query; -#[cfg(feature = "full")] pub use query::QueryItemOrSumReturnType; #[cfg(feature = "estimated_costs")] mod worst_case; -#[cfg(feature = "full")] use std::collections::HashSet; -use grovedb_costs::cost_return_on_error_no_add; -#[cfg(feature = "full")] -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] -use grovedb_storage::StorageContext; +use grovedb_storage::{Storage, StorageContext}; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -#[cfg(feature = "full")] use crate::{ reference_path::{path_from_reference_path_type, path_from_reference_qualified_path_type}, - util::storage_context_optional_tx, + util::TxRef, Element, Error, GroveDb, Transaction, TransactionArg, }; -#[cfg(feature = "full")] /// Limit of possible indirections pub const MAX_REFERENCE_HOPS: usize = 10; -#[cfg(feature = "full")] impl GroveDb { /// Get an element from the backing store /// Merk Caching is on by default @@ -211,17 +204,15 @@ impl GroveDb { .get_raw_caching_optional ); - if let Some(transaction) = transaction { - self.get_raw_on_transaction_caching_optional( - path, - key, - allow_cache, - transaction, - grove_version, - ) - } else { - self.get_raw_without_transaction_caching_optional(path, key, allow_cache, grove_version) - } + let tx = TxRef::new(&self.db, transaction); + + self.get_raw_on_transaction_caching_optional( + path, + key, + allow_cache, + tx.as_ref(), + grove_version, + ) } /// Get Element at specified path and key @@ -264,22 +255,15 @@ impl GroveDb { .get_raw_optional_caching_optional ); - if let Some(transaction) = transaction { - self.get_raw_optional_on_transaction_caching_optional( - path, - key, - allow_cache, - transaction, - grove_version, - ) - } else { - self.get_raw_optional_without_transaction_caching_optional( - path, - key, - allow_cache, - grove_version, - ) - } + let tx = TxRef::new(&self.db, transaction); + + self.get_raw_optional_on_transaction_caching_optional( + path, + key, + allow_cache, + tx.as_ref(), + grove_version, + ) } /// Get tree item without following references @@ -341,64 +325,6 @@ impl GroveDb { } } - /// Get tree item without following references - pub(crate) fn get_raw_without_transaction_caching_optional>( - &self, - path: SubtreePath, - key: &[u8], - allow_cache: bool, - grove_version: &GroveVersion, - ) -> CostResult { - let mut cost = OperationCost::default(); - - let merk_to_get_from = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path(path, None, grove_version) - .map_err(|e| match e { - Error::InvalidParentLayerPath(s) => { - Error::PathParentLayerNotFound(s) - } - _ => e, - }) - ); - - Element::get(&merk_to_get_from, key, allow_cache, grove_version).add_cost(cost) - } - - /// Get tree item without following references - pub(crate) fn get_raw_optional_without_transaction_caching_optional>( - &self, - path: SubtreePath, - key: &[u8], - allow_cache: bool, - grove_version: &GroveVersion, - ) -> CostResult, Error> { - let mut cost = OperationCost::default(); - - let merk_result = self - .open_non_transactional_merk_at_path(path, None, grove_version) - .map_err(|e| match e { - Error::InvalidParentLayerPath(s) => Error::PathParentLayerNotFound(s), - _ => e, - }) - .unwrap_add_cost(&mut cost); - let merk = cost_return_on_error_no_add!( - &cost, - match merk_result { - Ok(result) => Ok(Some(result)), - Err(Error::PathParentLayerNotFound(_)) | Err(Error::InvalidParentLayerPath(_)) => - Ok(None), - Err(e) => Err(e), - } - ); - - if let Some(merk_to_get_from) = merk { - Element::get_optional(&merk_to_get_from, key, allow_cache, grove_version).add_cost(cost) - } else { - Ok(None).wrap_with_cost(cost) - } - } - /// Does tree element exist without following references /// There is no cache for has_raw pub fn has_raw<'b, B, P>( @@ -417,23 +343,25 @@ impl GroveDb { grove_version.grovedb_versions.operations.get.has_raw ); + let tx = TxRef::new(&self.db, transaction); + // Merk's items should be written into data storage and checked accordingly - storage_context_optional_tx!(self.db, path.into(), None, transaction, storage, { - storage.flat_map(|s| s.get(key).map_err(|e| e.into()).map_ok(|x| x.is_some())) - }) + self.db + .get_transactional_storage_context(path.into(), None, tx.as_ref()) + .flat_map(|s| s.get(key).map_err(|e| e.into()).map_ok(|x| x.is_some())) } fn check_subtree_exists>( &self, path: SubtreePath, - transaction: TransactionArg, + transaction: &Transaction, error_fn: impl FnOnce() -> Error, grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); if let Some((parent_path, parent_key)) = path.derive_parent() { - let element = if let Some(transaction) = transaction { + let element = { let merk_to_get_from = cost_return_on_error!( &mut cost, self.open_transactional_merk_at_path( @@ -444,13 +372,6 @@ impl GroveDb { ) ); - Element::get(&merk_to_get_from, parent_key, true, grove_version) - } else { - let merk_to_get_from = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path(parent_path, None, grove_version) - ); - Element::get(&merk_to_get_from, parent_key, true, grove_version) } .unwrap_add_cost(&mut cost); @@ -474,9 +395,11 @@ impl GroveDb { where B: AsRef<[u8]> + 'b, { + let tx = TxRef::new(&self.db, transaction); + self.check_subtree_exists( path.clone(), - transaction, + tx.as_ref(), || { Error::PathNotFound(format!( "subtree doesn't exist at path {:?}", @@ -506,9 +429,11 @@ impl GroveDb { .check_subtree_exists_invalid_path ); + let tx = TxRef::new(&self.db, transaction); + self.check_subtree_exists( path, - transaction, + tx.as_ref(), || Error::InvalidPath("subtree doesn't exist".to_owned()), grove_version, ) diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 5926fedd..cbbb1d54 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -1,31 +1,21 @@ //! Insert operations -#[cfg(feature = "full")] use std::{collections::HashMap, option::Option::None}; -#[cfg(feature = "full")] use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -#[cfg(feature = "full")] use grovedb_merk::{tree::NULL_HASH, Merk, MerkOptions}; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] -use grovedb_storage::rocksdb_storage::{ - PrefixedRocksDbStorageContext, PrefixedRocksDbTransactionContext, -}; -use grovedb_storage::{Storage, StorageBatch}; +use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -#[cfg(feature = "full")] use crate::{ - reference_path::path_from_reference_path_type, Element, Error, GroveDb, Transaction, - TransactionArg, + reference_path::path_from_reference_path_type, util::TxRef, Element, Error, GroveDb, + Transaction, TransactionArg, }; - -#[cfg(feature = "full")] #[derive(Clone)] /// Insert options pub struct InsertOptions { @@ -37,7 +27,6 @@ pub struct InsertOptions { pub base_root_storage_is_free: bool, } -#[cfg(feature = "full")] impl Default for InsertOptions { fn default() -> Self { InsertOptions { @@ -48,7 +37,6 @@ impl Default for InsertOptions { } } -#[cfg(feature = "full")] impl InsertOptions { fn checks_for_override(&self) -> bool { self.validate_insertion_does_not_override_tree || self.validate_insertion_does_not_override @@ -61,7 +49,6 @@ impl InsertOptions { } } -#[cfg(feature = "full")] impl GroveDb { /// Insert a GroveDB element given a path to the subtree and the key to /// insert at @@ -86,26 +73,17 @@ impl GroveDb { let subtree_path: SubtreePath = path.into(); let batch = StorageBatch::new(); - let collect_costs = if let Some(transaction) = transaction { - self.insert_on_transaction( - subtree_path, - key, - element, - options.unwrap_or_default(), - transaction, - &batch, - grove_version, - ) - } else { - self.insert_without_transaction( - subtree_path, - key, - element, - options.unwrap_or_default(), - &batch, - grove_version, - ) - }; + let tx = TxRef::new(&self.db, transaction); + + let collect_costs = self.insert_on_transaction( + subtree_path, + key, + element, + options.unwrap_or_default(), + tx.as_ref(), + &batch, + grove_version, + ); collect_costs.flat_map_ok(|_| { self.db @@ -165,50 +143,6 @@ impl GroveDb { Ok(()).wrap_with_cost(cost) } - fn insert_without_transaction<'b, B: AsRef<[u8]>>( - &self, - path: SubtreePath<'b, B>, - key: &[u8], - element: Element, - options: InsertOptions, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult<(), Error> { - check_grovedb_v0_with_cost!( - "insert_without_transaction", - grove_version - .grovedb_versions - .operations - .insert - .insert_without_transaction - ); - - let mut cost = OperationCost::default(); - - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - - let merk = cost_return_on_error!( - &mut cost, - self.add_element_without_transaction( - &path.to_vec(), - key, - element, - options, - batch, - grove_version - ) - ); - merk_cache.insert(path.clone(), merk); - - cost_return_on_error!( - &mut cost, - self.propagate_changes_without_transaction(merk_cache, path, batch, grove_version) - ); - - Ok(()).wrap_with_cost(cost) - } - /// Add subtree to another subtree. /// We want to add a new empty merk to another merk at a key /// first make sure other merk exist @@ -352,142 +286,6 @@ impl GroveDb { Ok(subtree_to_insert_into).wrap_with_cost(cost) } - /// Add an empty tree or item to a parent tree. - /// We want to add a new empty merk to another merk at a key - /// first make sure other merk exist - /// if it exists, then create merk to be inserted, and get root hash - /// we only care about root hash of merk to be inserted - fn add_element_without_transaction<'db, B: AsRef<[u8]>>( - &'db self, - path: &[B], - key: &[u8], - element: Element, - options: InsertOptions, - batch: &'db StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult, Error> { - check_grovedb_v0_with_cost!( - "add_element_without_transaction", - grove_version - .grovedb_versions - .operations - .insert - .add_element_without_transaction - ); - - let mut cost = OperationCost::default(); - let mut subtree_to_insert_into = cost_return_on_error!( - &mut cost, - self.open_non_transactional_merk_at_path(path.into(), Some(batch), grove_version) - ); - - if options.checks_for_override() { - let maybe_element_bytes = cost_return_on_error!( - &mut cost, - subtree_to_insert_into - .get( - key, - true, - Some(&Element::value_defined_cost_for_serialized_value), - grove_version - ) - .map_err(|e| Error::CorruptedData(e.to_string())) - ); - if let Some(element_bytes) = maybe_element_bytes { - if options.validate_insertion_does_not_override { - return Err(Error::OverrideNotAllowed( - "insertion not allowed to override", - )) - .wrap_with_cost(cost); - } - if options.validate_insertion_does_not_override_tree { - let element = cost_return_on_error_no_add!( - &cost, - Element::deserialize(element_bytes.as_slice(), grove_version).map_err( - |_| { - Error::CorruptedData(String::from("unable to deserialize element")) - } - ) - ); - if element.is_any_tree() { - return Err(Error::OverrideNotAllowed( - "insertion not allowed to override tree", - )) - .wrap_with_cost(cost); - } - } - } - } - - match element { - Element::Reference(ref reference_path, ..) => { - let reference_path = cost_return_on_error!( - &mut cost, - path_from_reference_path_type(reference_path.clone(), path, Some(key)) - .wrap_with_cost(OperationCost::default()) - ); - let referenced_item = cost_return_on_error!( - &mut cost, - self.follow_reference( - reference_path.as_slice().into(), - false, - None, - grove_version - ) - ); - - let referenced_element_value_hash = - cost_return_on_error!(&mut cost, referenced_item.value_hash(grove_version)); - - cost_return_on_error!( - &mut cost, - element.insert_reference( - &mut subtree_to_insert_into, - key, - referenced_element_value_hash, - Some(options.as_merk_options()), - grove_version - ) - ); - } - Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { - if value.is_some() { - return Err(Error::InvalidCodeExecution( - "a tree should be empty at the moment of insertion when not using batches", - )) - .wrap_with_cost(cost); - } else { - cost_return_on_error!( - &mut cost, - element.insert_subtree( - &mut subtree_to_insert_into, - key, - NULL_HASH, - Some(options.as_merk_options()), - grove_version - ) - ); - } - } - _ => { - cost_return_on_error!( - &mut cost, - element.insert( - &mut subtree_to_insert_into, - key, - Some(options.as_merk_options()), - grove_version - ) - ); - } - } - - Ok(subtree_to_insert_into).wrap_with_cost(cost) - } - - /// Insert if not exists - /// Insert if not exists - /// /// Inserts an element at the specified path and key if it does not already /// exist. /// @@ -640,7 +438,6 @@ impl GroveDb { } } -#[cfg(feature = "full")] #[cfg(test)] mod tests { use grovedb_costs::{ diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 07c34999..ae666d56 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -1,16 +1,13 @@ //! Check if empty tree operations -#[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; use grovedb_path::SubtreePath; -#[cfg(feature = "full")] -use grovedb_version::error::GroveVersionError; -use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; +use grovedb_version::{ + check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, +}; -#[cfg(feature = "full")] -use crate::{util::merk_optional_tx, Element, Error, GroveDb, TransactionArg}; +use crate::{util::TxRef, Error, GroveDb, TransactionArg}; -#[cfg(feature = "full")] impl GroveDb { /// Check if it's an empty tree pub fn is_empty_tree<'b, B, P>( @@ -30,19 +27,22 @@ impl GroveDb { let mut cost = OperationCost::default(); let path: SubtreePath = path.into(); + let tx = TxRef::new(&self.db, transaction); + cost_return_on_error!( &mut cost, - self.check_subtree_exists_path_not_found(path.clone(), transaction, grove_version) + self.check_subtree_exists_path_not_found( + path.clone(), + Some(tx.as_ref()), + grove_version + ) ); - merk_optional_tx!( + + let subtree = cost_return_on_error!( &mut cost, - self.db, - path, - None, - transaction, - subtree, - grove_version, - { Ok(subtree.is_empty_tree().unwrap_add_cost(&mut cost)).wrap_with_cost(cost) } - ) + self.open_transactional_merk_at_path(path, tx.as_ref(), None, grove_version) + ); + + subtree.is_empty_tree().add_cost(cost).map(Ok) } } diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 6e814f67..4895402c 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -11,7 +11,7 @@ use grovedb_merk::{ tree::value_hash, Merk, ProofWithoutEncodingResult, }; -use grovedb_storage::StorageContext; +use grovedb_storage::{Storage, StorageContext}; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; @@ -209,9 +209,11 @@ impl GroveDb { }) ); + let tx = self.db.start_transaction(); + let subtree = cost_return_on_error!( &mut cost, - self.open_non_transactional_merk_at_path(path.as_slice().into(), None, grove_version) + self.open_transactional_merk_at_path(path.as_slice().into(), &tx, None, grove_version) ); let limit = if path.len() < path_query.path.len() { diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 876fe62c..8e33df50 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -16,7 +16,7 @@ use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_storage::rocksdb_storage::storage_context::context_immediate::PrefixedRocksDbImmediateStorageContext; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; -use crate::{replication, Error, GroveDb, Transaction, TransactionArg}; +use crate::{replication, util::TxRef, Error, GroveDb, Transaction, TransactionArg}; pub(crate) type SubtreePrefix = [u8; blake3::OUT_LEN]; @@ -169,7 +169,7 @@ impl GroveDb { // of root (as it is now) pub fn get_subtrees_metadata( &self, - tx: TransactionArg, + transaction: TransactionArg, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -181,8 +181,10 @@ impl GroveDb { ); let mut subtrees_metadata = SubtreesMetadata::new(); + let tx = TxRef::new(&self.db, transaction); + let subtrees_root = self - .find_subtrees(&SubtreePath::empty(), tx, grove_version) + .find_subtrees(&SubtreePath::empty(), Some(tx.as_ref()), grove_version) .value?; for subtree in subtrees_root.into_iter() { let subtree_path: Vec<&[u8]> = subtree.iter().map(|vec| vec.as_slice()).collect(); @@ -192,48 +194,31 @@ impl GroveDb { let current_path = SubtreePath::from(path); match (current_path.derive_parent(), subtree.last()) { - (Some((parent_path, _)), Some(parent_key)) => match tx { - None => { - let parent_merk = self - .open_non_transactional_merk_at_path(parent_path, None, grove_version) - .value?; - if let Ok(Some((elem_value, elem_value_hash))) = parent_merk - .get_value_and_value_hash( - parent_key, - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .value - { - let actual_value_hash = value_hash(&elem_value).unwrap(); - subtrees_metadata.data.insert( - prefix, - (current_path.to_vec(), actual_value_hash, elem_value_hash), - ); - } - } - Some(t) => { - let parent_merk = self - .open_transactional_merk_at_path(parent_path, t, None, grove_version) - .value?; - if let Ok(Some((elem_value, elem_value_hash))) = parent_merk - .get_value_and_value_hash( - parent_key, - true, - None::<&fn(&[u8], &GroveVersion) -> Option>, - grove_version, - ) - .value - { - let actual_value_hash = value_hash(&elem_value).unwrap(); - subtrees_metadata.data.insert( - prefix, - (current_path.to_vec(), actual_value_hash, elem_value_hash), - ); - } + (Some((parent_path, _)), Some(parent_key)) => { + let parent_merk = self + .open_transactional_merk_at_path( + parent_path, + tx.as_ref(), + None, + grove_version, + ) + .value?; + if let Ok(Some((elem_value, elem_value_hash))) = parent_merk + .get_value_and_value_hash( + parent_key, + true, + None::<&fn(&[u8], &GroveVersion) -> Option>, + grove_version, + ) + .value + { + let actual_value_hash = value_hash(&elem_value).unwrap(); + subtrees_metadata.data.insert( + prefix, + (current_path.to_vec(), actual_value_hash, elem_value_hash), + ); } - }, + } _ => { subtrees_metadata.data.insert( prefix, @@ -262,7 +247,7 @@ impl GroveDb { pub fn fetch_chunk( &self, global_chunk_id: &[u8], - tx: TransactionArg, + transaction: TransactionArg, version: u16, grove_version: &GroveVersion, ) -> Result, Error> { @@ -277,11 +262,13 @@ impl GroveDb { )); } - let root_app_hash = self.root_hash(tx, grove_version).value?; + let tx = TxRef::new(&self.db, transaction); + + let root_app_hash = self.root_hash(Some(tx.as_ref()), grove_version).value?; let (chunk_prefix, chunk_id) = replication::util_split_global_chunk_id(global_chunk_id, &root_app_hash)?; - let subtrees_metadata = self.get_subtrees_metadata(tx, grove_version)?; + let subtrees_metadata = self.get_subtrees_metadata(Some(tx.as_ref()), grove_version)?; match subtrees_metadata.data.get(&chunk_prefix) { Some(path_data) => { @@ -289,67 +276,33 @@ impl GroveDb { let subtree_path: Vec<&[u8]> = subtree.iter().map(|vec| vec.as_slice()).collect(); let path: &[&[u8]] = &subtree_path; - match tx { - None => { - let merk = self - .open_non_transactional_merk_at_path(path.into(), None, grove_version) - .value?; + let merk = self + .open_transactional_merk_at_path(path.into(), tx.as_ref(), None, grove_version) + .value?; - if merk.is_empty_tree().unwrap() { - return Ok(vec![]); - } - - let chunk_producer_res = ChunkProducer::new(&merk); - match chunk_producer_res { - Ok(mut chunk_producer) => { - let chunk_res = chunk_producer.chunk(&chunk_id, grove_version); - match chunk_res { - Ok((chunk, _)) => match util_encode_vec_ops(chunk) { - Ok(op_bytes) => Ok(op_bytes), - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - }, - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - } - } - Err(_) => Err(Error::CorruptedData( - "Unable to create Chunk producer".to_string(), - )), - } - } - Some(t) => { - let merk = self - .open_transactional_merk_at_path(path.into(), t, None, grove_version) - .value?; - - if merk.is_empty_tree().unwrap() { - return Ok(vec![]); - } + if merk.is_empty_tree().unwrap() { + return Ok(vec![]); + } - let chunk_producer_res = ChunkProducer::new(&merk); - match chunk_producer_res { - Ok(mut chunk_producer) => { - let chunk_res = chunk_producer.chunk(&chunk_id, grove_version); - match chunk_res { - Ok((chunk, _)) => match util_encode_vec_ops(chunk) { - Ok(op_bytes) => Ok(op_bytes), - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - }, - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - } - } + let chunk_producer_res = ChunkProducer::new(&merk); + match chunk_producer_res { + Ok(mut chunk_producer) => { + let chunk_res = chunk_producer.chunk(&chunk_id, grove_version); + match chunk_res { + Ok((chunk, _)) => match util_encode_vec_ops(chunk) { + Ok(op_bytes) => Ok(op_bytes), + Err(_) => Err(Error::CorruptedData( + "Unable to create to load chunk".to_string(), + )), + }, Err(_) => Err(Error::CorruptedData( - "Unable to create Chunk producer".to_string(), + "Unable to create to load chunk".to_string(), )), } } + Err(_) => Err(Error::CorruptedData( + "Unable to create Chunk producer".to_string(), + )), } } None => Err(Error::CorruptedData("Prefix not found".to_string())), diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 41263669..b0540e72 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3118,10 +3118,13 @@ mod tests { // let mut iter = db // .elements_iterator([TEST_LEAF, b"subtree1"].as_ref(), None) // .expect("cannot create iterator"); + + let tx = db.start_transaction(); + let storage_context = db .grove_db .db - .get_storage_context([TEST_LEAF, b"subtree1"].as_ref().into(), None) + .get_transactional_storage_context([TEST_LEAF, b"subtree1"].as_ref().into(), None, &tx) .unwrap(); let mut iter = Element::iterator(storage_context.raw_iter()).unwrap(); assert_eq!( @@ -3210,7 +3213,12 @@ mod tests { fn test_root_subtree_has_root_key() { let grove_version = GroveVersion::latest(); let db = make_test_grovedb(grove_version); - let storage = db.db.get_storage_context(EMPTY_PATH, None).unwrap(); + let tx = db.start_transaction(); + + let storage = db + .db + .get_transactional_storage_context(EMPTY_PATH, None, &tx) + .unwrap(); let root_merk = Merk::open_base( storage, false, @@ -3310,10 +3318,16 @@ mod tests { // Retrieve subtree instance // Check if it returns the same instance that was inserted { + let tx = db.start_transaction(); + let subtree_storage = db .grove_db .db - .get_storage_context([TEST_LEAF, b"key1", b"key2"].as_ref().into(), None) + .get_transactional_storage_context( + [TEST_LEAF, b"key1", b"key2"].as_ref().into(), + None, + &tx, + ) .unwrap(); let subtree = Merk::open_layered_with_root_key( subtree_storage, @@ -3379,10 +3393,15 @@ mod tests { assert_eq!(result_element, Element::new_item(b"ayy".to_vec())); // Should be able to retrieve instances created before transaction + let tx = db.start_transaction(); let subtree_storage = db .grove_db .db - .get_storage_context([TEST_LEAF, b"key1", b"key2"].as_ref().into(), None) + .get_transactional_storage_context( + [TEST_LEAF, b"key1", b"key2"].as_ref().into(), + None, + &tx, + ) .unwrap(); let subtree = Merk::open_layered_with_root_key( subtree_storage, diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index b255f653..7f52b68f 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -264,11 +264,13 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("should insert item"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); // Open merk and check all elements in it let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -354,8 +356,9 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("should insert item"); let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -402,12 +405,14 @@ fn test_sum_tree_feature() { .expect("should insert tree"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); // Sum should be non for non sum tree // TODO: change interface to retrieve element directly let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -421,7 +426,7 @@ fn test_sum_tree_feature() { b"key2", Element::empty_sum_tree(), None, - None, + Some(&tx), grove_version, ) .unwrap() @@ -438,15 +443,16 @@ fn test_sum_tree_feature() { b"item1", Element::new_sum_item(30), None, - None, + Some(&tx), grove_version, ) .unwrap() .expect("should insert item"); // TODO: change interface to retrieve element directly let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -460,7 +466,7 @@ fn test_sum_tree_feature() { b"item2", Element::new_sum_item(-10), None, - None, + Some(&tx), grove_version, ) .unwrap() @@ -470,14 +476,15 @@ fn test_sum_tree_feature() { b"item3", Element::new_sum_item(50), None, - None, + Some(&tx), grove_version, ) .unwrap() .expect("should insert item"); let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -491,14 +498,15 @@ fn test_sum_tree_feature() { b"item4", Element::new_item(vec![29]), None, - None, + Some(&tx), grove_version, ) .unwrap() .expect("should insert item"); let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -512,7 +520,7 @@ fn test_sum_tree_feature() { b"item2", Element::new_sum_item(10), None, - None, + Some(&tx), grove_version, ) .unwrap() @@ -522,14 +530,15 @@ fn test_sum_tree_feature() { b"item3", Element::new_sum_item(-100), None, - None, + Some(&tx), grove_version, ) .unwrap() .expect("should insert item"); let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -542,7 +551,7 @@ fn test_sum_tree_feature() { [TEST_LEAF, b"key2"].as_ref(), b"item4", None, - None, + Some(&tx), grove_version, ) .unwrap() @@ -553,14 +562,15 @@ fn test_sum_tree_feature() { b"item4", Element::new_sum_item(10000000), None, - None, + Some(&tx), grove_version, ) .unwrap() .expect("should insert item"); let merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -668,11 +678,13 @@ fn test_sum_tree_propagation() { assert_eq!(sum_tree.sum_value_or_default(), 35); let batch = StorageBatch::new(); + let tx = db.start_transaction(); // Assert node feature types let test_leaf_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -692,8 +704,9 @@ fn test_sum_tree_propagation() { )); let parent_sum_tree = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -715,8 +728,9 @@ fn test_sum_tree_propagation() { )); let child_sum_tree = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key", b"tree2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -800,9 +814,12 @@ fn test_sum_tree_with_batches() { .expect("should apply batch"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); + let sum_tree = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -840,14 +857,15 @@ fn test_sum_tree_with_batches() { b"c".to_vec(), Element::new_sum_item(10), )]; - db.apply_batch(ops, None, None, grove_version) + db.apply_batch(ops, None, Some(&tx), grove_version) .unwrap() .expect("should apply batch"); let batch = StorageBatch::new(); let sum_tree = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -932,9 +950,12 @@ fn test_sum_tree_with_batches() { .expect("should apply batch"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); + let sum_tree = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, Some(&batch), grove_version, ) diff --git a/grovedb/src/tests/tree_hashes_tests.rs b/grovedb/src/tests/tree_hashes_tests.rs index e86b8fd0..2ca66b49 100644 --- a/grovedb/src/tests/tree_hashes_tests.rs +++ b/grovedb/src/tests/tree_hashes_tests.rs @@ -56,10 +56,12 @@ fn test_node_hashes_when_inserting_item() { .expect("successful subtree insert"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); let test_leaf_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -130,9 +132,12 @@ fn test_tree_hashes_when_inserting_empty_tree() { let batch = StorageBatch::new(); + let tx = db.start_transaction(); + let test_leaf_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -173,8 +178,9 @@ fn test_tree_hashes_when_inserting_empty_tree() { .expect("value hash should be some"); let underlying_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -225,10 +231,12 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { .expect("successful subtree insert"); let batch = StorageBatch::new(); + let tx = db.start_transaction(); let under_top_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -236,8 +244,9 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { .expect("should open merk"); let middle_merk_key1 = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), + &tx, Some(&batch), grove_version, ) @@ -258,8 +267,9 @@ fn test_tree_hashes_when_inserting_empty_trees_twice_under_each_other() { .expect("value hash should be some"); let bottom_merk = db - .open_non_transactional_merk_at_path( + .open_transactional_merk_at_path( [TEST_LEAF, b"key1", b"key2"].as_ref().into(), + &tx, Some(&batch), grove_version, ) diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index b9b624a4..3d5283cd 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -1,515 +1,96 @@ -/// Macro to execute same piece of code on different storage contexts -/// (transactional or not) using path argument. -macro_rules! storage_context_optional_tx { - ($db:expr, $path:expr, $batch:expr, $transaction:ident, $storage:ident, { $($body:tt)* }) => { - { - use ::grovedb_storage::Storage; - if let Some(tx) = $transaction { - let $storage = $db - .get_transactional_storage_context($path, $batch, tx); - $($body)* - } else { - let $storage = $db - .get_storage_context($path, $batch); - $($body)* - } - } - }; -} +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_merk::Merk; +use grovedb_path::SubtreePath; +use grovedb_storage::{ + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, + Storage, StorageBatch, +}; +use grovedb_version::version::GroveVersion; +use grovedb_visualize::DebugByteVectors; -/// Macro to execute same piece of code on different storage contexts -/// (transactional or not) using path argument. -macro_rules! storage_context_with_parent_optional_tx { - ( - &mut $cost:ident, - $db:expr, - $path:expr, - $batch:expr, - $transaction:ident, - $storage:ident, - $root_key:ident, - $is_sum_tree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - { - use ::grovedb_storage::Storage; - if let Some(tx) = $transaction { - let $storage = $db - .get_transactional_storage_context($path.clone(), $batch, tx) - .unwrap_add_cost(&mut $cost); - if let Some((parent_path, parent_key)) = $path.derive_parent() { - let parent_storage = $db - .get_transactional_storage_context(parent_path, $batch, tx) - .unwrap_add_cost(&mut $cost); - let element = cost_return_on_error!( - &mut $cost, - Element::get_from_storage(&parent_storage, parent_key, $grove_version) - .map_err(|e| { - Error::PathParentLayerNotFound( - format!( - "could not get key for parent of subtree optional on tx: {}", - e - ) - ) - }) - ); - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } - } else { - return Err(Error::CorruptedData( - "path is empty".to_owned(), - )).wrap_with_cost($cost); - } - } else { - let $storage = $db - .get_storage_context($path.clone(), $batch).unwrap_add_cost(&mut $cost); - if let Some((parent_path, parent_key)) = $path.derive_parent() { - let parent_storage = $db.get_storage_context( - parent_path, $batch - ).unwrap_add_cost(&mut $cost); - let element = cost_return_on_error!( - &mut $cost, - Element::get_from_storage(&parent_storage, parent_key, $grove_version).map_err(|e| { - Error::PathParentLayerNotFound( - format!( - "could not get key for parent of subtree optional no tx: {}", - e - ) - ) - }) - ); - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } - } else { - return Err(Error::CorruptedData( - "path is empty".to_owned(), - )).wrap_with_cost($cost); - } - } - } - }; +use crate::{Element, Error, Transaction, TransactionArg}; + +pub(crate) enum TxRef<'a, 'db: 'a> { + Owned(Transaction<'db>), + Borrowed(&'a Transaction<'db>), } -/// Macro to execute same piece of code on different storage contexts -/// (transactional or not) using path argument. -macro_rules! storage_context_with_parent_optional_tx_internal_error { - ( - &mut $cost:ident, - $db:expr, - $path:expr, - $batch:expr, - $transaction:ident, - $storage:ident, - $root_key:ident, - $is_sum_tree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - { - use ::grovedb_storage::Storage; - if let Some(tx) = $transaction { - let $storage = $db - .get_transactional_storage_context($path.clone(), $batch, tx) - .unwrap_add_cost(&mut $cost); - if let Some((parent_path, parent_key)) = $path.derive_parent() { - let parent_storage = $db - .get_transactional_storage_context(parent_path, $batch, tx) - .unwrap_add_cost(&mut $cost); - let result = Element::get_from_storage( - &parent_storage, - parent_key, - $grove_version - ).map_err(|e| { - Error::PathParentLayerNotFound( - format!( - "could not get key for parent of subtree optional on tx: {}", - e - ) - ) - }).unwrap_add_cost(&mut $cost); - match result { - Ok(element) => { - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } - }, - Err(e) => Err(e), - } - } else { - return Err(Error::CorruptedData( - "path is empty".to_owned(), - )).wrap_with_cost($cost); - } - } else { - let $storage = $db - .get_storage_context($path.clone(), $batch).unwrap_add_cost(&mut $cost); - if let Some((parent_path, parent_key)) = $path.derive_parent() { - let parent_storage = $db.get_storage_context( - parent_path, - $batch - ).unwrap_add_cost(&mut $cost); - let result = Element::get_from_storage( - &parent_storage, - parent_key, - $grove_version - ).map_err(|e| { - Error::PathParentLayerNotFound( - format!( - "could not get key for parent of subtree optional no tx: {}", - e - ) - ) - }).unwrap_add_cost(&mut $cost); - match result { - Ok(element) => { - match element { - Element::Tree(root_key, _) => { - let $root_key = root_key; - let $is_sum_tree = false; - $($body)* - } - Element::SumTree(root_key, ..) => { - let $root_key = root_key; - let $is_sum_tree = true; - $($body)* - } - _ => { - return Err(Error::CorruptedData( - "parent is not a tree" - .to_owned(), - )).wrap_with_cost($cost); - } - } - }, - Err(e) => Err(e), - } - } else { - return Err(Error::CorruptedData( - "path is empty".to_owned(), - )).wrap_with_cost($cost); - } - } +impl<'a, 'db> TxRef<'a, 'db> { + pub(crate) fn new(db: &'db RocksDbStorage, transaction_arg: TransactionArg<'db, 'a>) -> Self { + if let Some(tx) = transaction_arg { + Self::Borrowed(tx) + } else { + Self::Owned(db.start_transaction()) } - }; + } } -/// Macro to execute same piece of code on different storage contexts with -/// empty prefix. -macro_rules! meta_storage_context_optional_tx { - ($db:expr, $batch:expr, $transaction:ident, $storage:ident, { $($body:tt)* }) => { - { - use ::grovedb_storage::Storage; - if let Some(tx) = $transaction { - let $storage = $db - .get_transactional_storage_context( - ::grovedb_path::SubtreePath::empty(), - $batch, - tx - ); - $($body)* - } else { - let $storage = $db - .get_storage_context( - ::grovedb_path::SubtreePath::empty(), - $batch - ); - $($body)* - } +impl<'a, 'db> AsRef> for TxRef<'a, 'db> { + fn as_ref(&self) -> &Transaction<'db> { + match self { + TxRef::Owned(tx) => tx, + TxRef::Borrowed(tx) => tx, } - }; + } } -/// Macro to execute same piece of code on Merk with varying storage -/// contexts. -macro_rules! merk_optional_tx { - ( - &mut $cost:ident, - $db:expr, - $path:expr, - $batch:expr, - $transaction:ident, - $subtree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - if $path.is_root() { - use crate::util::storage_context_optional_tx; - storage_context_optional_tx!( - $db, - ::grovedb_path::SubtreePath::empty(), - $batch, - $transaction, - storage, - { - let $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_base( - storage.unwrap_add_cost(&mut $cost), - false, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version, - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* - }) - } else { - use crate::util::storage_context_with_parent_optional_tx; - storage_context_with_parent_optional_tx!( - &mut $cost, - $db, - $path, - $batch, - $transaction, - storage, - root_key, - is_sum_tree, - $grove_version, - { - #[allow(unused_mut)] - let mut $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version, - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* - } - ) - } - }; -} +pub(crate) fn open_transactional_merk_at_path<'db, 'b, B>( + db: &'db RocksDbStorage, + path: SubtreePath<'b, B>, + tx: &'db Transaction, + batch: Option<&'db StorageBatch>, + grove_version: &GroveVersion, +) -> CostResult>, Error> +where + B: AsRef<[u8]> + 'b, +{ + let mut cost = OperationCost::default(); -/// Macro to execute same piece of code on Merk with varying storage -/// contexts. -macro_rules! merk_optional_tx_internal_error { - ( - &mut $cost:ident, - $db:expr, - $path:expr, - $batch:expr, - $transaction:ident, - $subtree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - if $path.is_root() { - use crate::util::storage_context_optional_tx; - storage_context_optional_tx!( - $db, - ::grovedb_path::SubtreePath::empty(), - $batch, - $transaction, - storage, - { - let $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_base( - storage.unwrap_add_cost(&mut $cost), - false, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* + let storage = db + .get_transactional_storage_context(path.clone(), batch, tx) + .unwrap_add_cost(&mut cost); + if let Some((parent_path, parent_key)) = path.derive_parent() { + let parent_storage = db + .get_transactional_storage_context(parent_path.clone(), batch, tx) + .unwrap_add_cost(&mut cost); + let element = cost_return_on_error!( + &mut cost, + Element::get_from_storage(&parent_storage, parent_key, grove_version).map_err(|e| { + Error::InvalidParentLayerPath(format!( + "could not get key {} for parent {:?} of subtree: {}", + hex::encode(parent_key), + DebugByteVectors(parent_path.to_vec()), + e + )) }) - } else { - use crate::util::storage_context_with_parent_optional_tx_internal_error; - storage_context_with_parent_optional_tx_internal_error!( - &mut $cost, - $db, - $path, - $batch, - $transaction, - storage, - root_key, - is_sum_tree, - $grove_version, - { - #[allow(unused_mut)] - let mut $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version, - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* - } - ) - } - }; -} - -/// Macro to execute same piece of code on Merk with varying storage -/// contexts. -macro_rules! merk_optional_tx_path_not_empty { - ( - &mut $cost:ident, - $db:expr, - $path:expr, - $batch:expr, - $transaction:ident, - $subtree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - { - use crate::util::storage_context_with_parent_optional_tx; - storage_context_with_parent_optional_tx!( - &mut $cost, - $db, - $path, - $batch, - $transaction, + ); + let is_sum_tree = element.is_sum_tree(); + if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) = element { + Merk::open_layered_with_root_key( storage, root_key, is_sum_tree, - $grove_version, - { - #[allow(unused_mut)] - let mut $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_layered_with_root_key( - storage, - root_key, - is_sum_tree, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version, - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* - } + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, ) - } - }; -} - -/// Macro to execute same piece of code on Merk with varying storage -/// contexts. -macro_rules! root_merk_optional_tx { - ( - &mut $cost:ident, - $db:expr, - $batch:expr, - $transaction:ident, - $subtree:ident, - $grove_version:ident, - { $($body:tt)* } - ) => { - { - use crate::util::storage_context_optional_tx; - storage_context_optional_tx!( - $db, - ::grovedb_path::SubtreePath::empty(), - $batch, - $transaction, - storage, - { - let $subtree = cost_return_on_error!( - &mut $cost, - ::grovedb_merk::Merk::open_base( - storage.unwrap_add_cost(&mut $cost), - false, - Some(&Element::value_defined_cost_for_serialized_value), - $grove_version, - ).map(|merk_res| - merk_res - .map_err(|_| crate::Error::CorruptedData( - "cannot open a subtree".to_owned() - )) - ) - ); - $($body)* + .map_err(|_| { + Error::CorruptedData("cannot open a subtree with given root key".to_owned()) }) + .add_cost(cost) + } else { + Err(Error::CorruptedPath( + "cannot open a subtree as parent exists but is not a tree".to_string(), + )) + .wrap_with_cost(cost) } - }; + } else { + Merk::open_base( + storage, + false, + Some(&Element::value_defined_cost_for_serialized_value), + grove_version, + ) + .map_err(|_| Error::CorruptedData("cannot open a the root subtree".to_owned())) + .add_cost(cost) + } } - -pub(crate) use merk_optional_tx; -pub(crate) use merk_optional_tx_internal_error; -pub(crate) use merk_optional_tx_path_not_empty; -pub(crate) use meta_storage_context_optional_tx; -pub(crate) use root_merk_optional_tx; -pub(crate) use storage_context_optional_tx; -pub(crate) use storage_context_with_parent_optional_tx; -pub(crate) use storage_context_with_parent_optional_tx_internal_error; diff --git a/grovedb/src/visualize.rs b/grovedb/src/visualize.rs index 029d1bca..9d609ebc 100644 --- a/grovedb/src/visualize.rs +++ b/grovedb/src/visualize.rs @@ -36,13 +36,12 @@ use bincode::{ }; use grovedb_merk::{Merk, VisualizeableMerk}; use grovedb_path::SubtreePathBuilder; -use grovedb_storage::StorageContext; +use grovedb_storage::{Storage, StorageContext}; use grovedb_version::version::GroveVersion; use grovedb_visualize::{visualize_stdout, Drawer, Visualize}; use crate::{ - element::Element, reference_path::ReferencePathType, util::storage_context_optional_tx, - GroveDb, TransactionArg, + element::Element, reference_path::ReferencePathType, util::TxRef, GroveDb, TransactionArg, }; impl Visualize for Element { @@ -226,35 +225,40 @@ impl GroveDb { ) -> Result> { drawer.down(); - storage_context_optional_tx!(self.db, (&path).into(), None, transaction, storage, { - let mut iter = Element::iterator(storage.unwrap().raw_iter()).unwrap(); - while let Some((key, element)) = iter - .next_element(grove_version) - .unwrap() - .expect("cannot get next element") - { - drawer.write(b"\n[key: ")?; - drawer = key.visualize(drawer)?; - drawer.write(b" ")?; - match element { - Element::Tree(..) => { - drawer.write(b"Merk root is: ")?; - drawer = element.visualize(drawer)?; - drawer.down(); - drawer = self.draw_subtree( - drawer, - path.derive_owned_with_child(key), - transaction, - grove_version, - )?; - drawer.up(); - } - other => { - drawer = other.visualize(drawer)?; - } + let tx = TxRef::new(&self.db, transaction); + + let storage = self + .db + .get_transactional_storage_context((&path).into(), None, tx.as_ref()) + .unwrap(); + + let mut iter = Element::iterator(storage.raw_iter()).unwrap(); + while let Some((key, element)) = iter + .next_element(grove_version) + .unwrap() + .expect("cannot get next element") + { + drawer.write(b"\n[key: ")?; + drawer = key.visualize(drawer)?; + drawer.write(b" ")?; + match element { + Element::Tree(..) => { + drawer.write(b"Merk root is: ")?; + drawer = element.visualize(drawer)?; + drawer.down(); + drawer = self.draw_subtree( + drawer, + path.derive_owned_with_child(key), + transaction, + grove_version, + )?; + drawer.up(); + } + other => { + drawer = other.visualize(drawer)?; } } - }); + } drawer.up(); Ok(drawer) From bdef7194ddbc242659151f6b0113eae087a8d9a8 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 25 Oct 2024 21:27:01 +0200 Subject: [PATCH 05/27] wip --- costs/src/context.rs | 2 +- .../estimated_costs/average_case_costs.rs | 8 ++-- .../batch/estimated_costs/worst_case_costs.rs | 6 +-- .../batch/just_in_time_reference_update.rs | 14 +++--- grovedb/src/batch/mod.rs | 28 ++++++------ grovedb/src/element/get.rs | 8 ++-- grovedb/src/element/insert.rs | 4 +- grovedb/src/element/query.rs | 34 +++++++------- .../src/estimated_costs/average_case_costs.rs | 15 +++---- .../src/estimated_costs/worst_case_costs.rs | 7 ++- grovedb/src/lib.rs | 4 +- grovedb/src/operations/auxiliary.rs | 4 ++ grovedb/src/operations/delete/average_case.rs | 10 ++--- .../src/operations/delete/delete_up_tree.rs | 2 +- grovedb/src/operations/delete/mod.rs | 45 ++++++++++++++----- grovedb/src/operations/delete/worst_case.rs | 6 +-- grovedb/src/operations/get/mod.rs | 2 +- grovedb/src/operations/get/query.rs | 14 +++--- grovedb/src/operations/insert/mod.rs | 6 ++- grovedb/src/operations/proof/generate.rs | 19 ++++---- grovedb/src/tests/sum_tree_tests.rs | 15 ++++++- grovedb/src/util.rs | 10 +++++ merk/src/merk/mod.rs | 8 ++-- merk/src/proofs/tree.rs | 22 ++++----- merk/src/tree/encoding.rs | 2 +- merk/src/tree/mod.rs | 10 ++--- merk/src/tree/ops.rs | 6 +-- merk/src/tree/walk/mod.rs | 8 ++-- storage/src/rocksdb_storage/storage.rs | 16 +++---- 29 files changed, 194 insertions(+), 141 deletions(-) diff --git a/costs/src/context.rs b/costs/src/context.rs index d69cb054..8c0faffd 100644 --- a/costs/src/context.rs +++ b/costs/src/context.rs @@ -193,7 +193,7 @@ macro_rules! cost_return_on_error { /// so no costs will be added except previously accumulated. #[macro_export] macro_rules! cost_return_on_error_no_add { - ( &$cost:ident, $($body:tt)+ ) => { + ( $cost:ident, $($body:tt)+ ) => { { use $crate::CostsExt; let result = { $($body)+ }; diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index ef64b0f4..c7bbbc05 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -196,7 +196,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { let mut cost = OperationCost::default(); let layer_element_estimates = cost_return_on_error_no_add!( - &cost, + cost, self.paths.get(path).ok_or_else(|| { let paths = self .paths @@ -218,7 +218,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // Then we have to get the tree if self.cached_merks.get(path).is_none() { let layer_info = cost_return_on_error_no_add!( - &cost, + cost, self.paths.get(path).ok_or_else(|| { let paths = self .paths @@ -233,7 +233,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { }) ); cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_average_case_get_merk_at_path::( &mut cost, path, @@ -272,7 +272,7 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { // Then we have to get the tree if !self.cached_merks.contains_key(&base_path) { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_average_case_get_merk_at_path::( &mut cost, &base_path, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 9bf9a808..37cc74b1 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -192,7 +192,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { let mut cost = OperationCost::default(); let worst_case_layer_element_estimates = cost_return_on_error_no_add!( - &cost, + cost, self.paths .get(path) .ok_or_else(|| Error::PathNotFoundInCacheForEstimatedCosts(format!( @@ -204,7 +204,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { // Then we have to get the tree if !self.cached_merks.contains(path) { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, @@ -247,7 +247,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { // Then we have to get the tree if !self.cached_merks.contains(&base_path) { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_worst_case_get_merk_at_path::( &mut cost, &base_path, diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index f4385b89..c52b3cc4 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -53,7 +53,7 @@ where updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); // There are no storage flags, we can just hash new element let new_serialized_bytes = cost_return_on_error_no_add!( - &cost, + cost, updated_new_element_with_old_flags.serialize(grove_version) ); let val_hash = value_hash(&new_serialized_bytes).unwrap_add_cost(&mut cost); @@ -93,7 +93,7 @@ where updated_new_element_with_old_flags.set_flags(maybe_old_flags.clone()); let serialized_with_old_flags = cost_return_on_error_no_add!( - &cost, + cost, updated_new_element_with_old_flags.serialize(grove_version) ); KV::node_value_byte_cost_size( @@ -115,7 +115,7 @@ where if let Some(old_element_flags) = maybe_old_flags.as_mut() { if let BasicStorageRemoval(removed_bytes) = storage_costs.removed_bytes { let (_, value_removed_bytes) = cost_return_on_error_no_add!( - &cost, + cost, split_removal_bytes(old_element_flags, 0, removed_bytes) ); storage_costs.removed_bytes = value_removed_bytes; @@ -125,7 +125,7 @@ where let mut new_element_cloned = original_new_element.clone(); let changed = cost_return_on_error_no_add!( - &cost, + cost, (flags_update)( &storage_costs, maybe_old_flags.clone(), @@ -145,10 +145,8 @@ where return Ok(val_hash).wrap_with_cost(cost); } else { // There are no storage flags, we can just hash new element - let new_serialized_bytes = cost_return_on_error_no_add!( - &cost, - new_element_cloned.serialize(grove_version) - ); + let new_serialized_bytes = + cost_return_on_error_no_add!(cost, new_element_cloned.serialize(grove_version)); new_storage_cost = KV::node_value_byte_cost_size( key.len() as u32, diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 0217943c..82b272fc 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -816,7 +816,7 @@ where Ok(referenced_element_value_hash).wrap_with_cost(cost) } else if let Some(referenced_path) = intermediate_reference_info { let path = cost_return_on_error_no_add!( - &cost, + cost, path_from_reference_qualified_path_type(referenced_path.clone(), qualified_path) ); self.follow_reference_get_value_hash( @@ -909,7 +909,7 @@ where if let Some(referenced_element) = referenced_element { let element = cost_return_on_error_no_add!( - &cost, + cost, Element::deserialize(referenced_element.as_slice(), grove_version).map_err(|_| { Error::CorruptedData(String::from("unable to deserialize element")) }) @@ -1012,13 +1012,13 @@ where | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let serialized = - cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + cost_return_on_error_no_add!(cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( - &cost, + cost, path_from_reference_qualified_path_type(path, qualified_path) ); self.follow_reference_get_value_hash( @@ -1089,7 +1089,7 @@ where | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let serialized = cost_return_on_error_no_add!( - &cost, + cost, element.serialize(grove_version) ); if element.get_flags().is_none() { @@ -1137,7 +1137,7 @@ where Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( - &cost, + cost, path_from_reference_qualified_path_type( path.clone(), qualified_path @@ -1166,13 +1166,13 @@ where | Element::ItemWithBackwardsReferences(..) | Element::SumItemWithBackwardsReferences(..) => { let serialized = - cost_return_on_error_no_add!(&cost, element.serialize(grove_version)); + cost_return_on_error_no_add!(cost, element.serialize(grove_version)); let val_hash = value_hash(&serialized).unwrap_add_cost(&mut cost); Ok(val_hash).wrap_with_cost(cost) } Element::Reference(path, ..) | Element::BidirectionalReference(path, ..) => { let path = cost_return_on_error_no_add!( - &cost, + cost, path_from_reference_qualified_path_type(path.clone(), qualified_path) ); self.follow_reference_get_value_hash( @@ -1454,7 +1454,7 @@ where ) ); cost_return_on_error_no_add!( - &cost, + cost, Element::deserialize(value.as_slice(), grove_version).map_err(|_| { Error::CorruptedData(String::from("unable to deserialize element")) }) @@ -1582,7 +1582,7 @@ where ), }; let merk_feature_type = - cost_return_on_error_no_add!(&cost, element.get_feature_type(is_sum_tree)); + cost_return_on_error_no_add!(cost, element.get_feature_type(is_sum_tree)); cost_return_on_error!( &mut cost, @@ -1779,7 +1779,7 @@ impl GroveDb { if batch_apply_options.base_root_storage_is_free { // the base root is free let mut update_root_cost = cost_return_on_error_no_add!( - &cost, + cost, merk_tree_cache .update_base_merk_root_key(calculated_root_key, grove_version) .cost_as_result() @@ -2345,7 +2345,7 @@ impl GroveDb { // .join(" | ") // ); // } - Ok(()).wrap_with_cost(cost) + tx.commit_local().wrap_with_cost(cost) } /// Applies a partial batch of operations on GroveDB @@ -2465,7 +2465,7 @@ impl GroveDb { // we will get GroveDB a new set of GroveDBOps let new_operations = cost_return_on_error_no_add!( - &cost, + cost, add_on_operations(&total_current_costs, &left_over_operations) ); @@ -2513,7 +2513,7 @@ impl GroveDb { .map_err(|e| e.into()) ); - Ok(()).wrap_with_cost(cost) + tx.commit_local().wrap_with_cost(cost) } #[cfg(feature = "estimated_costs")] diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 10618130..d9a453fb 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -70,7 +70,7 @@ impl Element { .map_err(|e| Error::CorruptedData(e.to_string())) ); let element = cost_return_on_error_no_add!( - &cost, + cost, value_opt .map(|value| { Self::deserialize(value.as_slice(), grove_version).map_err(|_| { @@ -129,7 +129,7 @@ impl Element { .map_err(|e| Error::CorruptedData(e.to_string())) ); let maybe_tree_inner: Option = cost_return_on_error_no_add!( - &cost, + cost, node_value_opt .map(|node_value| { Decode::decode(node_value.as_slice()) @@ -140,7 +140,7 @@ impl Element { let value = maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned()); let element = cost_return_on_error_no_add!( - &cost, + cost, value .as_ref() .map(|value| { @@ -221,7 +221,7 @@ impl Element { ); let absolute_element = cost_return_on_error_no_add!( - &cost, + cost, element.convert_if_reference_to_absolute_reference(path, Some(key.as_ref())) ); diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 75a07d6d..c8e5e189 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -387,10 +387,10 @@ impl Element { let cost = OperationCost::default(); let merk_feature_type = - cost_return_on_error_no_add!(&cost, self.get_feature_type(merk.is_sum_tree)); + cost_return_on_error_no_add!(cost, self.get_feature_type(merk.is_sum_tree)); let tree_cost = - cost_return_on_error_no_add!(&cost, self.get_specialized_cost(grove_version)); + cost_return_on_error_no_add!(cost, self.get_specialized_cost(grove_version)); let cost = tree_cost + self.get_flags().as_ref().map_or(0, |flags| { diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index da45e061..98502575 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -28,6 +28,8 @@ use crate::operations::proof::util::hex_to_ascii; #[cfg(any(feature = "full", feature = "verify"))] use crate::Element; #[cfg(feature = "full")] +use crate::Transaction; +#[cfg(feature = "full")] use crate::{ element::helpers::raw_decode, query_result_type::{ @@ -309,6 +311,8 @@ impl Element { add_element_function: fn(PathQueryPushArgs, &GroveVersion) -> CostResult<(), Error>, grove_version: &GroveVersion, ) -> CostResult<(QueryResultElements, u16), Error> { + use crate::util::TxRef; + check_grovedb_v0_with_cost!( "get_query_apply_function", grove_version @@ -325,6 +329,8 @@ impl Element { let original_offset = sized_query.offset; let mut offset = original_offset; + let tx = TxRef::new(storage, transaction); + if sized_query.query.left_to_right { for item in sized_query.query.iter() { cost_return_on_error!( @@ -335,7 +341,7 @@ impl Element { &mut results, path, sized_query, - transaction, + tx.as_ref(), &mut limit, &mut offset, query_options, @@ -358,7 +364,7 @@ impl Element { &mut results, path, sized_query, - transaction, + tx.as_ref(), &mut limit, &mut offset, query_options, @@ -486,7 +492,7 @@ impl Element { if element.is_any_tree() { let mut path_vec = path.to_vec(); let key = cost_return_on_error_no_add!( - &cost, + cost, key.ok_or(Error::MissingParameter( "the key must be provided when using a subquery path", )) @@ -608,7 +614,7 @@ impl Element { } } else if allow_get_raw { cost_return_on_error_no_add!( - &cost, + cost, Element::basic_push( PathQueryPushArgs { storage, @@ -638,7 +644,7 @@ impl Element { } } else { cost_return_on_error_no_add!( - &cost, + cost, Element::basic_push( PathQueryPushArgs { storage, @@ -711,7 +717,7 @@ impl Element { results: &mut Vec, path: &[&[u8]], sized_query: &SizedQuery, - transaction: TransactionArg, + transaction: &Transaction, limit: &mut Option, offset: &mut Option, query_options: QueryOptions, @@ -721,15 +727,13 @@ impl Element { ) -> CostResult<(), Error> { use grovedb_storage::Storage; - use crate::util::{self, TxRef}; + use crate::util::{self}; check_grovedb_v0_with_cost!( "query_item", grove_version.grovedb_versions.element.query_item ); - let tx = TxRef::new(storage, transaction); - let mut cost = OperationCost::default(); let subtree_path: SubtreePath<_> = path.into(); @@ -742,7 +746,7 @@ impl Element { util::open_transactional_merk_at_path( storage, subtree_path, - tx.as_ref(), + transaction, None, grove_version ) @@ -757,7 +761,7 @@ impl Element { match add_element_function( PathQueryPushArgs { storage, - transaction, + transaction: Some(transaction), key: Some(key.as_slice()), element, path, @@ -807,7 +811,7 @@ impl Element { } else { // this is a query on a range let ctx = storage - .get_transactional_storage_context(subtree_path, None, tx.as_ref()) + .get_transactional_storage_context(subtree_path, None, transaction) .unwrap_add_cost(&mut cost); let mut iter = ctx.raw_iter(); @@ -820,7 +824,7 @@ impl Element { .unwrap_add_cost(&mut cost) { let element = cost_return_on_error_no_add!( - &cost, + cost, raw_decode( iter.value() .unwrap_add_cost(&mut cost) @@ -837,7 +841,7 @@ impl Element { let result_with_cost = add_element_function( PathQueryPushArgs { storage, - transaction, + transaction: Some(transaction), key: Some(key), element, path, @@ -1700,7 +1704,7 @@ impl ElementsIterator { .unwrap_add_cost(&mut cost) .zip(self.raw_iter.value().unwrap_add_cost(&mut cost)) { - let element = cost_return_on_error_no_add!(&cost, raw_decode(value, grove_version)); + let element = cost_return_on_error_no_add!(cost, raw_decode(value, grove_version)); let key_vec = key.to_vec(); self.raw_iter.next().unwrap_add_cost(&mut cost); Some((key_vec, element)) diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index cc877ba6..30831bf6 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -85,7 +85,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let flags_size = cost_return_on_error_no_add!( - &cost, + cost, estimated_layer_information .estimated_layer_sizes .layered_flags_size() @@ -173,7 +173,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let flags_size = cost_return_on_error_no_add!( - &cost, + cost, estimated_layer_information .estimated_layer_sizes .layered_flags_size() @@ -235,7 +235,7 @@ impl GroveDb { _ => add_cost_case_merk_insert( &mut cost, key_len, - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32, in_tree_using_sums, ), }; @@ -296,7 +296,7 @@ impl GroveDb { let sum_item_cost_size = if value.is_sum_item() { SUM_ITEM_COST_SIZE } else { - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32 + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32 }; let value_len = sum_item_cost_size + flags_len; add_cost_case_merk_replace_same_size( @@ -309,7 +309,7 @@ impl GroveDb { _ => add_cost_case_merk_replace_same_size( &mut cost, key_len, - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32, in_tree_using_sums, ), }; @@ -351,8 +351,7 @@ impl GroveDb { }); // Items need to be always the same serialized size for this to work let item_cost_size = - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) - as u32; + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32; let value_len = item_cost_size + flags_len; add_cost_case_merk_patch( &mut cost, @@ -394,7 +393,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let value_size = cost_return_on_error_no_add!( - &cost, + cost, estimated_layer_information .estimated_layer_sizes .value_with_feature_and_flags_size() diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index f9c350d2..a6aacb4e 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -221,7 +221,7 @@ impl GroveDb { _ => add_cost_case_merk_insert( &mut cost, key_len, - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32, in_parent_tree_using_sums, ), }; @@ -289,7 +289,7 @@ impl GroveDb { _ => add_cost_case_merk_replace( &mut cost, key_len, - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32, + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32, in_parent_tree_using_sums, ), }; @@ -331,8 +331,7 @@ impl GroveDb { }); // Items need to be always the same serialized size for this to work let sum_item_cost_size = - cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) - as u32; + cost_return_on_error_no_add!(cost, value.serialized_size(grove_version)) as u32; let value_len = sum_item_cost_size + flags_len; add_cost_case_merk_patch( &mut cost, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 1c2638ec..33c96f39 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -404,7 +404,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let mut child_tree = cost_return_on_error_no_add!( - &cost, + cost, merk_cache.remove(path).ok_or(Error::CorruptedCodeExecution( "Merk Cache should always contain the last path", )) @@ -457,7 +457,7 @@ impl GroveDb { let mut cost = OperationCost::default(); let mut child_tree = cost_return_on_error_no_add!( - &cost, + cost, merk_cache .remove(&path) .ok_or(Error::CorruptedCodeExecution( diff --git a/grovedb/src/operations/auxiliary.rs b/grovedb/src/operations/auxiliary.rs index 84c7fee5..22b2d911 100644 --- a/grovedb/src/operations/auxiliary.rs +++ b/grovedb/src/operations/auxiliary.rs @@ -67,6 +67,8 @@ impl GroveDb { .commit_multi_context_batch(batch, Some(tx.as_ref())) .add_cost(cost) .map_err(Into::into) + .map_ok(|_| tx.commit_local()) + .flatten() } /// Delete op for aux storage @@ -96,6 +98,8 @@ impl GroveDb { .commit_multi_context_batch(batch, Some(tx.as_ref())) .add_cost(cost) .map_err(Into::into) + .map_ok(|_| tx.commit_local()) + .flatten() } /// Get op for aux storage diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 986f2b90..56162e8e 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -67,11 +67,11 @@ impl GroveDb { estimated_element_size, is_sum_tree, ) = cost_return_on_error_no_add!( - &cost, + cost, if height == path_len - 1 { if let Some(layer_info) = estimated_layer_info.get(height as u64) { let estimated_value_len = cost_return_on_error_no_add!( - &cost, + cost, layer_info .estimated_layer_sizes .value_with_feature_and_flags_size() @@ -96,7 +96,7 @@ impl GroveDb { used_path = smaller_path; if let Some(layer_info) = estimated_layer_info.get(height as u64) { let estimated_value_len = cost_return_on_error_no_add!( - &cost, + cost, layer_info .estimated_layer_sizes .subtree_with_feature_and_flags_size() @@ -158,7 +158,7 @@ impl GroveDb { if validate { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_average_case_get_merk_at_path::( &mut cost, path, @@ -170,7 +170,7 @@ impl GroveDb { } if check_if_tree { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_average_case_get_raw_cost::( &mut cost, path, diff --git a/grovedb/src/operations/delete/delete_up_tree.rs b/grovedb/src/operations/delete/delete_up_tree.rs index 7ecfce83..c56ed1d4 100644 --- a/grovedb/src/operations/delete/delete_up_tree.rs +++ b/grovedb/src/operations/delete/delete_up_tree.rs @@ -138,7 +138,7 @@ impl GroveDb { ); let ops = cost_return_on_error_no_add!( - &cost, + cost, if let Some(stop_path_height) = options.stop_path_height { maybe_ops.ok_or_else(|| { Error::DeleteUpTreeStopHeightMoreThanInitialPathSize(format!( diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index a3571499..68f964ba 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -11,7 +11,7 @@ use std::collections::{BTreeSet, HashMap}; pub use delete_up_tree::DeleteUpTreeOptions; use grovedb_costs::{ cost_return_on_error, - storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, + storage_cost::removal::StorageRemovedBytes::{self, BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; use grovedb_merk::{proofs::Query, Error as MerkError, KVIterator, Merk, MerkOptions}; @@ -125,11 +125,14 @@ impl GroveDb { ) .map_ok(|_| ()); - collect_costs.flat_map_ok(|_| { - self.db - .commit_multi_context_batch(batch, transaction) - .map_err(Into::into) - }) + collect_costs + .flat_map_ok(|_| { + self.db + .commit_multi_context_batch(batch, transaction) + .map_err(Into::into) + .map_ok(|_| tx.commit_local()) + }) + .flatten() } /// Delete all elements in a specified subtree @@ -146,8 +149,14 @@ impl GroveDb { P: Into>, { let tx = TxRef::new(&self.db, transaction); - self.clear_subtree_with_costs(path, options, tx.as_ref(), grove_version) - .unwrap() + if self + .clear_subtree_with_costs(path, options, tx.as_ref(), grove_version) + .unwrap()? + { + tx.commit_local()?; + return Ok(true); + } + Ok(false) } /// Delete all elements in a specified subtree and get back costs @@ -319,8 +328,10 @@ impl GroveDb { collect_costs.flat_map_ok(|_| { self.db - .commit_multi_context_batch(batch, transaction) + .commit_multi_context_batch(batch, Some(tx.as_ref())) + .map_ok(|_| tx.commit_local()) .map_err(Into::into) + .flatten() }) } @@ -426,6 +437,11 @@ impl GroveDb { batch, grove_version, ) + .map_ok(|r| { + tx.commit_local()?; + Ok(r) + }) + .flatten() } /// Delete operation for delete internal @@ -484,7 +500,7 @@ impl GroveDb { Some(x) => x, }; - if is_subtree { + let result = if is_subtree { let subtree_merk_path = path.derive_owned_with_child(key); let subtree_merk_path_vec = subtree_merk_path.to_vec(); let batch_deleted_keys = current_batch_operations @@ -551,7 +567,14 @@ impl GroveDb { key.to_vec(), ))) .wrap_with_cost(cost) - } + }; + + result + .map_ok(|r| { + tx.commit_local()?; + Ok(r) + }) + .flatten() } } diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 8533cde5..47cfa3e1 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -61,7 +61,7 @@ impl GroveDb { max_element_size, is_sum_tree, ) = cost_return_on_error_no_add!( - &cost, + cost, if height == path_len { if let Some((is_in_sum_tree, _)) = intermediate_tree_info.get(height as u64) { @@ -140,7 +140,7 @@ impl GroveDb { if validate { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, @@ -151,7 +151,7 @@ impl GroveDb { } if check_if_tree { cost_return_on_error_no_add!( - &cost, + cost, GroveDb::add_worst_case_get_raw_cost::( &mut cost, path, diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index fa58f94b..85312176 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -309,7 +309,7 @@ impl GroveDb { }) .unwrap_add_cost(&mut cost); let merk = cost_return_on_error_no_add!( - &cost, + cost, match merk_result { Ok(result) => Ok(Some(result)), Err(Error::PathParentLayerNotFound(_)) | Err(Error::InvalidParentLayerPath(_)) => diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index ebbf04fb..106afa4a 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -133,7 +133,7 @@ where { let mut cost = OperationCost::default(); let query = cost_return_on_error_no_add!( - &cost, + cost, PathQuery::merge(path_queries.to_vec(), grove_version) ); let (result, _) = cost_return_on_error!( @@ -271,7 +271,7 @@ where { }) .collect::, Error>>(); - let results = cost_return_on_error_no_add!(&cost, results_wrapped); + let results = cost_return_on_error_no_add!(cost, results_wrapped); Ok((QueryResultElements { elements: results }, skipped)).wrap_with_cost(cost) } @@ -363,7 +363,7 @@ where { }) .collect::>, Error>>(); - let results = cost_return_on_error_no_add!(&cost, results_wrapped); + let results = cost_return_on_error_no_add!(cost, results_wrapped); Ok((results, skipped)).wrap_with_cost(cost) } @@ -466,7 +466,7 @@ where { }) .collect::, Error>>(); - let results = cost_return_on_error_no_add!(&cost, results_wrapped); + let results = cost_return_on_error_no_add!(cost, results_wrapped); Ok((results, skipped)).wrap_with_cost(cost) } @@ -552,7 +552,7 @@ where { }) .collect::, Error>>(); - let results = cost_return_on_error_no_add!(&cost, results_wrapped); + let results = cost_return_on_error_no_add!(cost, results_wrapped); Ok((results, skipped)).wrap_with_cost(cost) } @@ -617,7 +617,7 @@ where { let mut cost = OperationCost::default(); let terminal_keys = cost_return_on_error_no_add!( - &cost, + cost, path_query.terminal_keys(max_results, grove_version) ); @@ -676,7 +676,7 @@ where { let mut cost = OperationCost::default(); let terminal_keys = cost_return_on_error_no_add!( - &cost, + cost, path_query.terminal_keys(max_results, grove_version) ); diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index cbbb1d54..90adfe3b 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -87,8 +87,10 @@ impl GroveDb { collect_costs.flat_map_ok(|_| { self.db - .commit_multi_context_batch(batch, transaction) + .commit_multi_context_batch(batch, Some(tx.as_ref())) .map_err(Into::into) + .map_ok(|_| tx.commit_local()) + .flatten() }) } @@ -201,7 +203,7 @@ impl GroveDb { } if options.validate_insertion_does_not_override_tree { let element = cost_return_on_error_no_add!( - &cost, + cost, Element::deserialize(element_bytes.as_slice(), grove_version).map_err( |_| { Error::CorruptedData(String::from("unable to deserialize element")) diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 4895402c..0429ff21 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -11,7 +11,7 @@ use grovedb_merk::{ tree::value_hash, Merk, ProofWithoutEncodingResult, }; -use grovedb_storage::{Storage, StorageContext}; +use grovedb_storage::StorageContext; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; @@ -23,7 +23,7 @@ use crate::{ util::hex_to_ascii, GroveDBProof, GroveDBProofV0, LayerProof, ProveOptions, }, reference_path::path_from_reference_path_type, - Element, Error, GroveDb, PathQuery, + Element, Error, GroveDb, PathQuery, Transaction, }; impl GroveDb { @@ -89,7 +89,7 @@ impl GroveDb { .with_big_endian() .with_no_limit(); let encoded_proof = cost_return_on_error_no_add!( - &cost, + cost, bincode::encode_to_vec(proof, config) .map_err(|e| Error::CorruptedData(format!("unable to encode proof {}", e))) ); @@ -121,6 +121,8 @@ impl GroveDb { .wrap_with_cost(cost); } + let tx = self.start_transaction(); + #[cfg(feature = "proof_debug")] { // we want to query raw because we want the references to not be resolved at @@ -134,7 +136,7 @@ impl GroveDb { prove_options.decrease_limit_on_empty_sub_query_result, false, QueryResultType::QueryPathKeyElementTrioResultType, - None, + Some(&tx), grove_version, ) ) @@ -150,7 +152,7 @@ impl GroveDb { prove_options.decrease_limit_on_empty_sub_query_result, false, QueryResultType::QueryPathKeyElementTrioResultType, - None, + Some(&tx), grove_version, ) ) @@ -169,6 +171,7 @@ impl GroveDb { path_query, &mut limit, &prove_options, + &tx, grove_version ) ); @@ -189,12 +192,13 @@ impl GroveDb { path_query: &PathQuery, overall_limit: &mut Option, prove_options: &ProveOptions, + tx: &Transaction, grove_version: &GroveVersion, ) -> CostResult { let mut cost = OperationCost::default(); let query = cost_return_on_error_no_add!( - &cost, + cost, path_query .query_items_at_path(path.as_slice(), grove_version) .and_then(|query_items| { @@ -209,8 +213,6 @@ impl GroveDb { }) ); - let tx = self.db.start_transaction(); - let subtree = cost_return_on_error!( &mut cost, self.open_transactional_merk_at_path(path.as_slice().into(), &tx, None, grove_version) @@ -342,6 +344,7 @@ impl GroveDb { path_query, overall_limit, prove_options, + tx, grove_version, ) ); diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index 7f52b68f..9af44026 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -5,7 +5,7 @@ use grovedb_merk::{ tree::kv::ValueDefinedCostType, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; -use grovedb_storage::StorageBatch; +use grovedb_storage::{Storage, StorageBatch}; use grovedb_version::version::GroveVersion; use crate::{ @@ -354,6 +354,7 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { ) .unwrap() .expect("should insert item"); + let tx = db.start_transaction(); let merk = db .open_transactional_merk_at_path( @@ -431,8 +432,9 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should insert sum tree"); + let sum_tree = db - .get([TEST_LEAF].as_ref(), b"key2", None, grove_version) + .get([TEST_LEAF].as_ref(), b"key2", Some(&tx), grove_version) .unwrap() .expect("should retrieve tree"); assert_eq!(sum_tree.sum_value_or_default(), 0); @@ -861,7 +863,16 @@ fn test_sum_tree_with_batches() { .unwrap() .expect("should apply batch"); + db.db + .commit_multi_context_batch(batch, Some(&tx)) + .unwrap() + .unwrap(); + + db.commit_transaction(tx).unwrap().unwrap(); + let batch = StorageBatch::new(); + let tx = db.start_transaction(); + let sum_tree = db .open_transactional_merk_at_path( [TEST_LEAF, b"key1"].as_ref().into(), diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 3d5283cd..f8761aa3 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -23,6 +23,16 @@ impl<'a, 'db> TxRef<'a, 'db> { Self::Owned(db.start_transaction()) } } + + /// Commit the transaction if it wasn't received from outside + pub(crate) fn commit_local(self) -> Result<(), Error> { + match self { + TxRef::Owned(tx) => tx + .commit() + .map_err(|e| grovedb_storage::Error::from(e).into()), + TxRef::Borrowed(_) => Ok(()), + } + } } impl<'a, 'db> AsRef> for TxRef<'a, 'db> { diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index d5f40c78..134e3e4b 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -378,7 +378,7 @@ where // update pointer to root node cost_return_on_error_no_add!( - &inner_cost, + inner_cost, batch .put_root(ROOT_KEY_KEY, tree_key, costs) .map_err(CostsError) @@ -414,7 +414,7 @@ where for (key, maybe_sum_tree_cost, maybe_value, storage_cost) in to_batch { if let Some((value, left_size, right_size)) = maybe_value { cost_return_on_error_no_add!( - &cost, + cost, batch .put( &key, @@ -432,7 +432,7 @@ where for (key, value, storage_cost) in aux { match value { Op::Put(value, ..) => cost_return_on_error_no_add!( - &cost, + cost, batch .put_aux(key, value, storage_cost.clone()) .map_err(CostsError) @@ -440,7 +440,7 @@ where Op::Delete => batch.delete_aux(key, storage_cost.clone()), _ => { cost_return_on_error_no_add!( - &cost, + cost, Err(Error::InvalidOperation( "only put and delete allowed for aux storage" )) diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index 16655a6d..63ac556c 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -379,11 +379,11 @@ where } for op in ops { - match cost_return_on_error_no_add!(&cost, op) { + match cost_return_on_error_no_add!(cost, op) { Op::Parent => { let (mut parent, child) = ( - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), ); cost_return_on_error!( &mut cost, @@ -400,8 +400,8 @@ where } Op::Child => { let (child, mut parent) = ( - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), ); cost_return_on_error!( &mut cost, @@ -418,8 +418,8 @@ where } Op::ParentInverted => { let (mut parent, child) = ( - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), ); cost_return_on_error!( &mut cost, @@ -436,8 +436,8 @@ where } Op::ChildInverted => { let (child, mut parent) = ( - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), - cost_return_on_error_no_add!(&cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), + cost_return_on_error_no_add!(cost, try_pop(&mut stack)), ); cost_return_on_error!( &mut cost, @@ -470,7 +470,7 @@ where maybe_last_key = Some(key.clone()); } - cost_return_on_error_no_add!(&cost, visit_node(&node)); + cost_return_on_error_no_add!(cost, visit_node(&node)); let tree: Tree = node.into(); stack.push(tree); @@ -493,7 +493,7 @@ where maybe_last_key = Some(key.clone()); } - cost_return_on_error_no_add!(&cost, visit_node(&node)); + cost_return_on_error_no_add!(cost, visit_node(&node)); let tree: Tree = node.into(); stack.push(tree); diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index cd10937d..11bd906a 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -52,7 +52,7 @@ impl TreeNode { let tree_bytes = cost_return_on_error!(&mut cost, storage.get(&key).map_err(StorageError)); let tree_opt = cost_return_on_error_no_add!( - &cost, + cost, tree_bytes .map(|x| TreeNode::decode_raw( &x, diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index 91eebf52..7008f538 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -716,7 +716,7 @@ impl TreeNode { // in this case there is a possibility that the client would want to update the // element flags based on the change of values cost_return_on_error_no_add!( - &cost, + cost, self.just_in_time_tree_node_value_update( old_specialized_cost, get_temp_new_value_with_old_flags, @@ -772,7 +772,7 @@ impl TreeNode { // in this case there is a possibility that the client would want to update the // element flags based on the change of values cost_return_on_error_no_add!( - &cost, + cost, self.just_in_time_tree_node_value_update( old_specialized_cost, get_temp_new_value_with_old_flags, @@ -826,7 +826,7 @@ impl TreeNode { // in this case there is a possibility that the client would want to update the // element flags based on the change of values cost_return_on_error_no_add!( - &cost, + cost, self.just_in_time_tree_node_value_update( old_specialized_cost, get_temp_new_value_with_old_flags, @@ -888,7 +888,7 @@ impl TreeNode { // in this case there is a possibility that the client would want to update the // element flags based on the change of values cost_return_on_error_no_add!( - &cost, + cost, self.just_in_time_tree_node_value_update( old_specialized_cost, get_temp_new_value_with_old_flags, @@ -971,7 +971,7 @@ impl TreeNode { } } - cost_return_on_error_no_add!(&cost, c.write(self, old_specialized_cost,)); + cost_return_on_error_no_add!(cost, c.write(self, old_specialized_cost,)); // println!("done committing {}", std::str::from_utf8(self.key()).unwrap()); diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 3e10b2c8..5fdb625f 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -514,13 +514,13 @@ where Delete => self.tree().inner.kv.value_byte_cost_size(), DeleteLayered | DeleteLayeredMaybeSpecialized => { cost_return_on_error_no_add!( - &cost, + cost, old_specialized_cost(&key_vec, value) ) } DeleteMaybeSpecialized => { cost_return_on_error_no_add!( - &cost, + cost, old_specialized_cost(&key_vec, value) ) } @@ -534,7 +534,7 @@ where prefixed_key_len + prefixed_key_len.required_space() as u32; let value = self.tree().value_ref(); cost_return_on_error_no_add!( - &cost, + cost, section_removal_bytes(value, total_key_len, old_cost) ) }; diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 4b67bb60..8fdf0be4 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -230,7 +230,7 @@ where ) -> CostResult { let mut cost = OperationCost::default(); cost_return_on_error_no_add!( - &cost, + cost, self.tree.own_result(|t| t .put_value( value, @@ -275,7 +275,7 @@ where ) -> CostResult { let mut cost = OperationCost::default(); cost_return_on_error_no_add!( - &cost, + cost, self.tree.own_result(|t| t .put_value_with_fixed_cost( value, @@ -321,7 +321,7 @@ where ) -> CostResult { let mut cost = OperationCost::default(); cost_return_on_error_no_add!( - &cost, + cost, self.tree.own_result(|t| t .put_value_and_reference_value_hash( value, @@ -368,7 +368,7 @@ where ) -> CostResult { let mut cost = OperationCost::default(); cost_return_on_error_no_add!( - &cost, + cost, self.tree.own_result(|t| t .put_value_with_reference_value_hash_and_value_cost( value, diff --git a/storage/src/rocksdb_storage/storage.rs b/storage/src/rocksdb_storage/storage.rs index 8913e9ea..7cedf49d 100644 --- a/storage/src/rocksdb_storage/storage.rs +++ b/storage/src/rocksdb_storage/storage.rs @@ -188,7 +188,7 @@ impl RocksDbStorage { db_batch.put(&key, &value); cost.seek_count += 1; cost_return_on_error_no_add!( - &cost, + cost, pending_costs .add_key_value_storage_costs( key.len() as u32, @@ -207,7 +207,7 @@ impl RocksDbStorage { db_batch.put_cf(cf_aux(&self.db), &key, &value); cost.seek_count += 1; cost_return_on_error_no_add!( - &cost, + cost, pending_costs .add_key_value_storage_costs( key.len() as u32, @@ -228,7 +228,7 @@ impl RocksDbStorage { // We only add costs for put root if they are set, otherwise it is free if cost_info.is_some() { cost_return_on_error_no_add!( - &cost, + cost, pending_costs .add_key_value_storage_costs( key.len() as u32, @@ -248,7 +248,7 @@ impl RocksDbStorage { db_batch.put_cf(cf_meta(&self.db), &key, &value); cost.seek_count += 1; cost_return_on_error_no_add!( - &cost, + cost, pending_costs .add_key_value_storage_costs( key.len() as u32, @@ -272,7 +272,7 @@ impl RocksDbStorage { cost.seek_count += 2; // lets get the values let value_len = cost_return_on_error_no_add!( - &cost, + cost, self.db.get(&key).map_err(RocksDBError) ) .map(|x| x.len() as u32) @@ -299,7 +299,7 @@ impl RocksDbStorage { } else { cost.seek_count += 2; let value_len = cost_return_on_error_no_add!( - &cost, + cost, self.db.get_cf(cf_aux(&self.db), &key).map_err(RocksDBError) ) .map(|x| x.len() as u32) @@ -327,7 +327,7 @@ impl RocksDbStorage { } else { cost.seek_count += 2; let value_len = cost_return_on_error_no_add!( - &cost, + cost, self.db .get_cf(cf_roots(&self.db), &key) .map_err(RocksDBError) @@ -357,7 +357,7 @@ impl RocksDbStorage { } else { cost.seek_count += 2; let value_len = cost_return_on_error_no_add!( - &cost, + cost, self.db .get_cf(cf_meta(&self.db), &key) .map_err(RocksDBError) From 15b3034d6bf51e21e99b6a9cdeff152425a3d85a Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 29 Oct 2024 01:31:46 +0100 Subject: [PATCH 06/27] wip --- grovedb/src/lib.rs | 2 +- grovedb/src/operations/get/mod.rs | 11 ++++++----- grovedb/src/operations/get/query.rs | 19 ++++++++++++------- grovedb/src/operations/insert/mod.rs | 2 +- grovedb/src/operations/proof/generate.rs | 2 +- 5 files changed, 21 insertions(+), 15 deletions(-) diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 33c96f39..42efe8a3 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -901,7 +901,7 @@ impl GroveDb { .follow_reference( (full_path.as_slice()).into(), allow_cache, - Some(transaction), + transaction, grove_version, ) .unwrap()?; diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index 85312176..f23f4335 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -67,6 +67,7 @@ impl GroveDb { ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); match cost_return_on_error!( &mut cost, @@ -74,7 +75,7 @@ impl GroveDb { path.clone(), key, allow_cache, - transaction, + Some(tx.as_ref()), grove_version ) ) { @@ -87,7 +88,7 @@ impl GroveDb { self.follow_reference( path_owned.as_slice().into(), allow_cache, - transaction, + tx.as_ref(), grove_version, ) .add_cost(cost) @@ -99,11 +100,11 @@ impl GroveDb { /// Return the Element that a reference points to. /// If the reference points to another reference, keep following until /// base element is reached. - pub fn follow_reference>( + pub(crate) fn follow_reference>( &self, path: SubtreePath, allow_cache: bool, - transaction: TransactionArg, + transaction: &Transaction, grove_version: &GroveVersion, ) -> CostResult { check_grovedb_v0_with_cost!( @@ -134,7 +135,7 @@ impl GroveDb { path_slice.into(), key, allow_cache, - transaction, + Some(transaction), grove_version ) .map_err(|e| match e { diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 106afa4a..3b387ece 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -15,7 +15,7 @@ use integer_encoding::VarInt; use crate::element::SumValue; use crate::{ element::QueryOptions, operations::proof::ProveOptions, - query_result_type::PathKeyOptionalElementTrio, + query_result_type::PathKeyOptionalElementTrio, util::TxRef, Transaction, }; #[cfg(feature = "full")] use crate::{ @@ -56,6 +56,7 @@ impl GroveDb { ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); let elements = cost_return_on_error!( &mut cost, @@ -83,7 +84,7 @@ impl GroveDb { .follow_reference( absolute_path.as_slice().into(), allow_cache, - transaction, + tx.as_ref(), grove_version, ) .unwrap_add_cost(&mut cost)?; @@ -182,7 +183,7 @@ where { element: Element, allow_cache: bool, cost: &mut OperationCost, - transaction: TransactionArg, + transaction: &Transaction, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -248,6 +249,7 @@ where { grove_version.grovedb_versions.operations.query.query ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); let (elements, skipped) = cost_return_on_error!( &mut cost, @@ -266,7 +268,7 @@ where { .into_iterator() .map(|result_item| { result_item.map_element(|element| { - self.follow_element(element, allow_cache, &mut cost, transaction, grove_version) + self.follow_element(element, allow_cache, &mut cost, tx.as_ref(), grove_version) }) }) .collect::, Error>>(); @@ -295,6 +297,7 @@ where { .query_item_value ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); let (elements, skipped) = cost_return_on_error!( &mut cost, @@ -327,7 +330,7 @@ where { .follow_reference( absolute_path.as_slice().into(), allow_cache, - transaction, + tx.as_ref(), grove_version, ) .unwrap_add_cost(&mut cost)?; @@ -387,6 +390,7 @@ where { .query_item_value_or_sum ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); let (elements, skipped) = cost_return_on_error!( &mut cost, @@ -419,7 +423,7 @@ where { .follow_reference( absolute_path.as_slice().into(), allow_cache, - transaction, + tx.as_ref(), grove_version, ) .unwrap_add_cost(&mut cost)?; @@ -485,6 +489,7 @@ where { grove_version.grovedb_versions.operations.query.query_sums ); let mut cost = OperationCost::default(); + let tx = TxRef::new(&self.db, transaction); let (elements, skipped) = cost_return_on_error!( &mut cost, @@ -517,7 +522,7 @@ where { .follow_reference( absolute_path.as_slice().into(), allow_cache, - transaction, + tx.as_ref(), grove_version, ) .unwrap_add_cost(&mut cost)?; diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 90adfe3b..5a516f51 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -234,7 +234,7 @@ impl GroveDb { self.follow_reference( reference_path.as_slice().into(), false, - Some(transaction), + transaction, grove_version ) ); diff --git a/grovedb/src/operations/proof/generate.rs b/grovedb/src/operations/proof/generate.rs index 0429ff21..7d22b48c 100644 --- a/grovedb/src/operations/proof/generate.rs +++ b/grovedb/src/operations/proof/generate.rs @@ -283,7 +283,7 @@ impl GroveDb { self.follow_reference( absolute_path.as_slice().into(), true, - None, + tx, grove_version ) ); From 6af542bb1b61d4140c2a46d9cf8655b5f095945e Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 29 Oct 2024 11:17:28 +0100 Subject: [PATCH 07/27] remove redundant error --- grovedb/src/element/query.rs | 57 +++++++++++++--------------- grovedb/src/error.rs | 4 -- grovedb/src/operations/delete/mod.rs | 14 +++---- grovedb/src/operations/get/mod.rs | 15 +------- grovedb/src/tests/mod.rs | 2 +- 5 files changed, 37 insertions(+), 55 deletions(-) diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 98502575..8eba0c15 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -741,19 +741,16 @@ impl Element { if !item.is_range() { // this is a query on a key if let QueryItem::Key(key) = item { - let subtree = cost_return_on_error!( - &mut cost, - util::open_transactional_merk_at_path( - storage, - subtree_path, - transaction, - None, - grove_version - ) - ); - let element_res = - Element::get(&subtree, key, query_options.allow_cache, grove_version) - .unwrap_add_cost(&mut cost); + let element_res = util::open_transactional_merk_at_path( + storage, + subtree_path, + transaction, + None, + grove_version, + ) + .flat_map_ok(|s| Element::get(&s, key, query_options.allow_cache, grove_version)) + .unwrap_add_cost(&mut cost); + match element_res { Ok(element) => { let (subquery_path, subquery) = @@ -782,7 +779,7 @@ impl Element { Err(e) => { if !query_options.error_if_intermediate_path_tree_not_present { match e { - Error::PathParentLayerNotFound(_) => Ok(()), + Error::InvalidParentLayerPath(_) => Ok(()), _ => Err(e), } } else { @@ -795,7 +792,7 @@ impl Element { Err(e) => { if !query_options.error_if_intermediate_path_tree_not_present { match e { - Error::PathParentLayerNotFound(_) => Ok(()), + Error::InvalidParentLayerPath(_) => Ok(()), _ => Err(e), } } else { @@ -862,7 +859,7 @@ impl Element { Err(e) => { if !query_options.error_if_intermediate_path_tree_not_present { match e { - Error::PathKeyNotFound(_) | Error::PathParentLayerNotFound(_) => (), + Error::PathKeyNotFound(_) | Error::InvalidParentLayerPath(_) => (), _ => return Err(e).wrap_with_cost(cost), } } else { @@ -1022,13 +1019,13 @@ mod tests { &query, QueryOptions::default(), None, - grove_version + grove_version, ) .unwrap() .expect("expected successful get_query"), vec![ Element::new_item(b"ayya".to_vec()), - Element::new_item(b"ayyc".to_vec()) + Element::new_item(b"ayyc".to_vec()), ] ); @@ -1043,14 +1040,14 @@ mod tests { &query, QueryOptions::default(), None, - grove_version + grove_version, ) .unwrap() .expect("expected successful get_query"), vec![ Element::new_item(b"ayya".to_vec()), Element::new_item(b"ayyb".to_vec()), - Element::new_item(b"ayyc".to_vec()) + Element::new_item(b"ayyc".to_vec()), ] ); @@ -1065,14 +1062,14 @@ mod tests { &query, QueryOptions::default(), None, - grove_version + grove_version, ) .unwrap() .expect("expected successful get_query"), vec![ Element::new_item(b"ayyb".to_vec()), Element::new_item(b"ayyc".to_vec()), - Element::new_item(b"ayyd".to_vec()) + Element::new_item(b"ayyd".to_vec()), ] ); @@ -1088,14 +1085,14 @@ mod tests { &query, QueryOptions::default(), None, - grove_version + grove_version, ) .unwrap() .expect("expected successful get_query"), vec![ Element::new_item(b"ayya".to_vec()), Element::new_item(b"ayyb".to_vec()), - Element::new_item(b"ayyc".to_vec()) + Element::new_item(b"ayyc".to_vec()), ] ); } @@ -1158,7 +1155,7 @@ mod tests { QueryOptions::default(), QueryPathKeyElementTrioResultType, None, - grove_version + grove_version, ) .unwrap() .expect("expected successful get_query") @@ -1173,7 +1170,7 @@ mod tests { vec![TEST_LEAF.to_vec()], b"c".to_vec(), Element::new_item(b"ayyc".to_vec()) - ) + ), ] ); } @@ -1525,7 +1522,7 @@ mod tests { .expect("expected successful get_query"); assert_eq!( elements.to_key_elements(), - vec![(b"c".to_vec(), Element::new_item(b"ayyc".to_vec())),] + vec![(b"c".to_vec(), Element::new_item(b"ayyc".to_vec()))] ); assert_eq!(skipped, 0); @@ -1549,7 +1546,7 @@ mod tests { elements.to_key_elements(), vec![ (b"a".to_vec(), Element::new_item(b"ayya".to_vec())), - (b"b".to_vec(), Element::new_item(b"ayyb".to_vec())) + (b"b".to_vec(), Element::new_item(b"ayyb".to_vec())), ] ); assert_eq!(skipped, 0); @@ -1570,7 +1567,7 @@ mod tests { elements.to_key_elements(), vec![ (b"b".to_vec(), Element::new_item(b"ayyb".to_vec())), - (b"c".to_vec(), Element::new_item(b"ayyc".to_vec())) + (b"c".to_vec(), Element::new_item(b"ayyc".to_vec())), ] ); assert_eq!(skipped, 1); @@ -1596,7 +1593,7 @@ mod tests { elements.to_key_elements(), vec![ (b"b".to_vec(), Element::new_item(b"ayyb".to_vec())), - (b"a".to_vec(), Element::new_item(b"ayya".to_vec())) + (b"a".to_vec(), Element::new_item(b"ayya".to_vec())), ] ); assert_eq!(skipped, 1); diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 92343935..4446f4ab 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -42,10 +42,6 @@ pub enum Error { /// isn't there #[error("path not found: {0}")] PathNotFound(String), - /// The path not found could represent a valid query, just where the parent - /// path merk isn't there - #[error("path parent layer not found: {0}")] - PathParentLayerNotFound(String), /// The path's item by key referenced was not found #[error("corrupted referenced path key not found: {0}")] diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index 68f964ba..fb81532b 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -833,7 +833,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); // assert_eq!(db.subtrees.len().unwrap(), 3); // TEST_LEAF, ANOTHER_TEST_LEAF // TEST_LEAF.key4 stay @@ -902,7 +902,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); transaction.commit().expect("cannot commit transaction"); assert!(matches!( @@ -1024,7 +1024,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); assert!(matches!( @@ -1145,7 +1145,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); assert!(matches!( @@ -1239,7 +1239,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); transaction.commit().expect("cannot commit transaction"); assert!(matches!( @@ -1324,7 +1324,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); assert!(matches!( db.get([TEST_LEAF].as_ref(), b"key1", None, grove_version) @@ -1727,7 +1727,7 @@ mod tests { grove_version ) .unwrap(), - Err(Error::PathParentLayerNotFound(_)) + Err(Error::InvalidParentLayerPath(_)) )); let key1_tree = db .get([TEST_LEAF].as_ref(), b"key1", None, grove_version) diff --git a/grovedb/src/operations/get/mod.rs b/grovedb/src/operations/get/mod.rs index f23f4335..41830873 100644 --- a/grovedb/src/operations/get/mod.rs +++ b/grovedb/src/operations/get/mod.rs @@ -139,7 +139,7 @@ impl GroveDb { grove_version ) .map_err(|e| match e { - Error::PathParentLayerNotFound(p) => { + Error::InvalidParentLayerPath(p) => { Error::CorruptedReferencePathParentLayerNotFound(p) } Error::PathKeyNotFound(p) => { @@ -281,12 +281,6 @@ impl GroveDb { let merk_to_get_from = cost_return_on_error!( &mut cost, self.open_transactional_merk_at_path(path, transaction, None, grove_version) - .map_err(|e| match e { - Error::InvalidParentLayerPath(s) => { - Error::PathParentLayerNotFound(s) - } - _ => e, - }) ); Element::get(&merk_to_get_from, key, allow_cache, grove_version).add_cost(cost) @@ -304,17 +298,12 @@ impl GroveDb { let mut cost = OperationCost::default(); let merk_result = self .open_transactional_merk_at_path(path, transaction, None, grove_version) - .map_err(|e| match e { - Error::InvalidParentLayerPath(s) => Error::PathParentLayerNotFound(s), - _ => e, - }) .unwrap_add_cost(&mut cost); let merk = cost_return_on_error_no_add!( cost, match merk_result { Ok(result) => Ok(Some(result)), - Err(Error::PathParentLayerNotFound(_)) | Err(Error::InvalidParentLayerPath(_)) => - Ok(None), + Err(Error::InvalidParentLayerPath(_)) => Ok(None), Err(e) => Err(e), } ); diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index b0540e72..69fb55f3 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -3929,7 +3929,7 @@ mod tests { assert!(elem_result.is_err()); assert!(matches!( elem_result, - Err(Error::PathParentLayerNotFound(..)) + Err(Error::InvalidParentLayerPath(..)) )); } From e8385ae620dcbd0a5245fc1db654848811e1ee94 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 1 Nov 2024 09:45:45 +0100 Subject: [PATCH 08/27] wip --- .../src/version/grovedb_versions.rs | 1 + grovedb-version/src/version/v1.rs | 1 + grovedb/src/element/insert.rs | 26 +++++- grovedb/src/element/mod.rs | 4 +- grovedb/src/element/serialize.rs | 84 +++++++++++++++++++ grovedb/src/tests/mod.rs | 18 ++++ 6 files changed, 130 insertions(+), 4 deletions(-) diff --git a/grovedb-version/src/version/grovedb_versions.rs b/grovedb-version/src/version/grovedb_versions.rs index 598fa178..9ff5ee11 100644 --- a/grovedb-version/src/version/grovedb_versions.rs +++ b/grovedb-version/src/version/grovedb_versions.rs @@ -214,6 +214,7 @@ pub struct GroveDBElementMethodVersions { pub query_item: FeatureVersion, pub basic_push: FeatureVersion, pub serialize: FeatureVersion, + pub serialize_for_value_hash: FeatureVersion, pub serialized_size: FeatureVersion, pub deserialize: FeatureVersion, } diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 97cfb38b..e3eeaed9 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -63,6 +63,7 @@ pub const GROVE_V1: GroveVersion = GroveVersion { query_item: 0, basic_push: 0, serialize: 0, + serialize_for_value_hash: 0, serialized_size: 0, deserialize: 0, }, diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index c8e5e189..41c18e5c 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -12,7 +12,9 @@ use grovedb_version::{ }; use integer_encoding::VarInt; -use crate::{Element, Element::SumItem, Error, Hash}; +use crate::{reference_path::ReferencePathType, Element, Error, Hash}; + +use super::CascadeOnUpdate; impl Element { #[cfg(feature = "full")] @@ -39,7 +41,7 @@ impl Element { let merk_feature_type = cost_return_on_error_default!(self.get_feature_type(merk.is_sum_tree)); - let batch_operations = if matches!(self, SumItem(..)) { + let batch_operations = if matches!(self, Element::SumItem(..)) { let value_cost = cost_return_on_error_default!(self.get_specialized_cost(grove_version)); @@ -94,7 +96,7 @@ impl Element { Err(e) => return Err(e).wrap_with_cost(Default::default()), }; - let entry = if matches!(self, SumItem(..)) { + let entry = if matches!(self, Element::SumItem(..)) { let value_cost = cost_return_on_error_default!(self.get_specialized_cost(grove_version)); @@ -463,6 +465,24 @@ impl Element { batch_operations.push(entry); Ok(()).wrap_with_cost(Default::default()) } + + /// Adds info on reference that points to this element. + fn referenced_from( + &mut self, + reference: ReferencePathType, + cascade_on_update: CascadeOnUpdate, + ) -> Result<(), Error> { + match self { + Element::Item(_, _) => todo!(), + Element::ItemWithBackwardsReferences(_, _, _) => todo!(), + Element::Reference(_, _, _) => todo!(), + Element::BidirectionalReference(_, _, _) => todo!(), + Element::SumItem(_, _) => todo!(), + Element::SumItemWithBackwardsReferences(_, _, _) => todo!(), + Element::Tree(_, _) => todo!(), + Element::SumTree(_, _, _) => todo!(), + } + } } #[cfg(feature = "full")] diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 321b37af..2327422d 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -295,7 +295,9 @@ impl Element { &self, grove_version: &grovedb_version::version::GroveVersion, ) -> grovedb_costs::CostResult { - let bytes = grovedb_costs::cost_return_on_error_default!(self.serialize(grove_version)); + let bytes = grovedb_costs::cost_return_on_error_default!( + self.serialize_for_value_hash(grove_version) + ); crate::value_hash(&bytes).map(Result::Ok) } } diff --git a/grovedb/src/element/serialize.rs b/grovedb/src/element/serialize.rs index 395fea8d..71447f1c 100644 --- a/grovedb/src/element/serialize.rs +++ b/grovedb/src/element/serialize.rs @@ -20,6 +20,25 @@ impl Element { .map_err(|e| Error::CorruptedData(format!("unable to serialize element {}", e))) } + #[cfg(feature = "full")] + /// Serializes Element without including backreferences. + /// This allows for identical serialization regardless of whether the item is referenced or not. + pub(crate) fn serialize_for_value_hash( + &self, + grove_version: &GroveVersion, + ) -> Result, Error> { + check_grovedb_v0!( + "Element::serialize_for_value_hash", + grove_version + .grovedb_versions + .element + .serialize_for_value_hash + ); + let config = config::standard().with_big_endian().with_no_limit(); + bincode::encode_to_vec(SerializeElementData(self), config) + .map_err(|e| Error::CorruptedData(format!("unable to serialize element {}", e))) + } + #[cfg(feature = "full")] /// Serializes self. Returns usize. pub fn serialized_size(&self, grove_version: &GroveVersion) -> Result { @@ -154,4 +173,69 @@ mod tests { ); assert_eq!(hex::encode(serialized), "010003010002abcd0105000103010203"); } + + #[test] + fn regular_items_and_referenced_items_shall_hash_equally() { + let element_one = Element::new_item(b"hello".to_vec()); + let element_two = Element::ItemWithBackwardsReferences(b"hello".to_vec(), Vec::new(), None); + + assert_eq!( + element_one + .value_hash(&GroveVersion::latest()) + .unwrap() + .unwrap(), + element_two + .value_hash(&GroveVersion::latest()) + .unwrap() + .unwrap() + ); + } +} + +/// Wrapper type to alter [Element]'s bincode serialization for variants with backward references, +/// practically ignoring that part. +struct SerializeElementData<'a>(&'a Element); + +impl bincode::Encode for SerializeElementData<'_> { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + match self.0 { + Element::Item(data, flags) | Element::ItemWithBackwardsReferences(data, _, flags) => { + ::encode(&(0u32), encoder)?; + bincode::Encode::encode(data, encoder)?; + bincode::Encode::encode(flags, encoder)?; + Ok(()) + } + Element::Reference(ref_path, max_hops, flags) + | Element::BidirectionalReference(ref_path, max_hops, flags) => { + ::encode(&(1u32), encoder)?; + bincode::Encode::encode(ref_path, encoder)?; + bincode::Encode::encode(max_hops, encoder)?; + bincode::Encode::encode(flags, encoder)?; + Ok(()) + } + Element::Tree(root_key, flags) => { + ::encode(&(2u32), encoder)?; + bincode::Encode::encode(root_key, encoder)?; + bincode::Encode::encode(flags, encoder)?; + Ok(()) + } + Element::SumItem(sum_value, flags) + | Element::SumItemWithBackwardsReferences(sum_value, _, flags) => { + ::encode(&(3u32), encoder)?; + bincode::Encode::encode(sum_value, encoder)?; + bincode::Encode::encode(flags, encoder)?; + Ok(()) + } + Element::SumTree(root_key, sum_value, flags) => { + ::encode(&(4u32), encoder)?; + bincode::Encode::encode(root_key, encoder)?; + bincode::Encode::encode(sum_value, encoder)?; + bincode::Encode::encode(flags, encoder)?; + Ok(()) + } + } + } } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 69fb55f3..568bc6a8 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -4144,3 +4144,21 @@ mod tests { .is_empty()); } } + +#[test] +fn subtrees_cant_be_referenced() { + let db = make_deep_tree(&GroveVersion::latest()); + assert!(db + .insert( + SubtreePath::empty(), + b"test_ref", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + TEST_LEAF.to_vec() + ])), + None, + None, + &GroveVersion::latest(), + ) + .unwrap() + .is_err()); +} From 4af11cfa693a8d0e30d20bbcb98ac80794ed2bf9 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 1 Nov 2024 18:57:05 +0100 Subject: [PATCH 09/27] wip --- grovedb/src/batch/mod.rs | 1 + grovedb/src/debugger.rs | 7 ++++ grovedb/src/element/helpers.rs | 8 ++--- grovedb/src/element/insert.rs | 51 +++++++++++++++++++++------- grovedb/src/element/mod.rs | 10 ++++-- grovedb/src/element/serialize.rs | 9 ++--- grovedb/src/operations/insert/mod.rs | 27 ++++++++++++++- 7 files changed, 90 insertions(+), 23 deletions(-) diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 82b272fc..4e0a7bd0 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -1315,6 +1315,7 @@ where Element::Reference(path_reference, element_max_reference_hop, _) | Element::BidirectionalReference( path_reference, + _, element_max_reference_hop, .., ) => { diff --git a/grovedb/src/debugger.rs b/grovedb/src/debugger.rs index 32beec15..ea0afefc 100644 --- a/grovedb/src/debugger.rs +++ b/grovedb/src/debugger.rs @@ -538,6 +538,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::AbsolutePathReference(path), _, + _, element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::AbsolutePathReference { path, @@ -551,6 +552,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::UpstreamRootHeightReference(n_keep, path_append), _, + _, element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightReference { @@ -573,6 +575,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { path_append, ), _, + _, element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamRootHeightWithParentPathAdditionReference { @@ -589,6 +592,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::UpstreamFromElementHeightReference(n_remove, path_append), _, + _, element_flags, ) => grovedbg_types::Element::Reference( grovedbg_types::Reference::UpstreamFromElementHeightReference { @@ -605,6 +609,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::CousinReference(swap_parent), _, + _, element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::CousinReference { swap_parent, @@ -618,6 +623,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::RemovedCousinReference(swap_parent), _, + _, element_flags, ) => { grovedbg_types::Element::Reference(grovedbg_types::Reference::RemovedCousinReference { @@ -633,6 +639,7 @@ fn element_to_grovedbg(element: crate::Element) -> grovedbg_types::Element { | crate::Element::BidirectionalReference( ReferencePathType::SiblingReference(sibling_key), _, + _, element_flags, ) => grovedbg_types::Element::Reference(grovedbg_types::Reference::SiblingReference { sibling_key, diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 0142d1fe..7f444926 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -165,7 +165,7 @@ impl Element { | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) | Element::SumItemWithBackwardsReferences(_, _, flags) - | Element::BidirectionalReference(_, _, flags) => flags, + | Element::BidirectionalReference(_, _, _, flags) => flags, } } @@ -180,7 +180,7 @@ impl Element { | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) | Element::SumItemWithBackwardsReferences(_, _, flags) - | Element::BidirectionalReference(_, _, flags) => flags, + | Element::BidirectionalReference(_, _, _, flags) => flags, } } @@ -195,7 +195,7 @@ impl Element { | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) | Element::SumItemWithBackwardsReferences(_, _, flags) - | Element::BidirectionalReference(_, _, flags) => flags, + | Element::BidirectionalReference(_, _, _, flags) => flags, } } @@ -210,7 +210,7 @@ impl Element { | Element::SumItem(_, flags) | Element::ItemWithBackwardsReferences(_, _, flags) | Element::SumItemWithBackwardsReferences(_, _, flags) - | Element::BidirectionalReference(_, _, flags) => *flags = new_flags, + | Element::BidirectionalReference(_, _, _, flags) => *flags = new_flags, } } diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 41c18e5c..432ff2f0 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -12,9 +12,8 @@ use grovedb_version::{ }; use integer_encoding::VarInt; -use crate::{reference_path::ReferencePathType, Element, Error, Hash}; - use super::CascadeOnUpdate; +use crate::{reference_path::ReferencePathType, Element, Error, Hash}; impl Element { #[cfg(feature = "full")] @@ -468,19 +467,47 @@ impl Element { /// Adds info on reference that points to this element. fn referenced_from( - &mut self, + mut self, reference: ReferencePathType, cascade_on_update: CascadeOnUpdate, - ) -> Result<(), Error> { + ) -> Result { match self { - Element::Item(_, _) => todo!(), - Element::ItemWithBackwardsReferences(_, _, _) => todo!(), - Element::Reference(_, _, _) => todo!(), - Element::BidirectionalReference(_, _, _) => todo!(), - Element::SumItem(_, _) => todo!(), - Element::SumItemWithBackwardsReferences(_, _, _) => todo!(), - Element::Tree(_, _) => todo!(), - Element::SumTree(_, _, _) => todo!(), + Element::Item(data, flags) => Ok(Element::ItemWithBackwardsReferences( + data, + vec![(reference, cascade_on_update)], + flags, + )), + Element::ItemWithBackwardsReferences(_, ref mut backward_references, _) => { + backward_references.push((reference, cascade_on_update)); + Ok(self) + } + Element::Reference(reference_path, max_hops, flags) => { + Ok(Element::BidirectionalReference( + reference_path, + vec![(reference, cascade_on_update)], + max_hops, + flags, + )) + } + Element::BidirectionalReference(_, ref mut backward_references, ..) => { + backward_references.push((reference, cascade_on_update)); + Ok(self) + } + Element::SumItem(value, flags) => Ok(Element::SumItemWithBackwardsReferences( + value, + vec![(reference, cascade_on_update)], + flags, + )), + Element::SumItemWithBackwardsReferences(_, ref mut backward_references, _) => { + backward_references.push((reference, cascade_on_update)); + Ok(self) + } + Element::Tree(..) => Err(Error::NotSupported( + "Cannot add references pointing to subtrees".to_owned(), + )), + Element::SumTree(..) => Err(Error::NotSupported( + "Cannot add references pointing to sumtrees".to_owned(), + )), } } } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 2327422d..691a15a4 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -90,7 +90,12 @@ pub enum Element { /// nodes SumTree(Option>, SumValue, Option), /// A reference to an object by its path - BidirectionalReference(ReferencePathType, MaxReferenceHop, Option), + BidirectionalReference( + ReferencePathType, + Vec<(ReferencePathType, CascadeOnUpdate)>, + MaxReferenceHop, + Option, + ), /// An ordinary value that has a backwards reference ItemWithBackwardsReferences( Vec, @@ -130,7 +135,8 @@ impl fmt::Display for Element { .map_or(String::new(), |f| format!(", flags: {:?}", f)) ) } - Element::BidirectionalReference(path, max_hop, flags) => { + Element::BidirectionalReference(path, backward_references, max_hop, flags) => { + // TODO: print something on backward_references write!( f, "BidirectionalReference({}, max_hop: {}{})", diff --git a/grovedb/src/element/serialize.rs b/grovedb/src/element/serialize.rs index 71447f1c..24114ca2 100644 --- a/grovedb/src/element/serialize.rs +++ b/grovedb/src/element/serialize.rs @@ -22,7 +22,8 @@ impl Element { #[cfg(feature = "full")] /// Serializes Element without including backreferences. - /// This allows for identical serialization regardless of whether the item is referenced or not. + /// This allows for identical serialization regardless of whether the item + /// is referenced or not. pub(crate) fn serialize_for_value_hash( &self, grove_version: &GroveVersion, @@ -192,8 +193,8 @@ mod tests { } } -/// Wrapper type to alter [Element]'s bincode serialization for variants with backward references, -/// practically ignoring that part. +/// Wrapper type to alter [Element]'s bincode serialization for variants with +/// backward references, practically ignoring that part. struct SerializeElementData<'a>(&'a Element); impl bincode::Encode for SerializeElementData<'_> { @@ -209,7 +210,7 @@ impl bincode::Encode for SerializeElementData<'_> { Ok(()) } Element::Reference(ref_path, max_hops, flags) - | Element::BidirectionalReference(ref_path, max_hops, flags) => { + | Element::BidirectionalReference(ref_path, _, max_hops, flags) => { ::encode(&(1u32), encoder)?; bincode::Encode::encode(ref_path, encoder)?; bincode::Encode::encode(max_hops, encoder)?; diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 5a516f51..14bc1a34 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -221,7 +221,12 @@ impl GroveDb { } match element { - Element::Reference(ref reference_path, ..) => { + Element::Reference(ref reference_path, ..) + | Element::BidirectionalReference(ref reference_path, ..) => { + // TODO: 1. turn referenced value into a backreference kind + // 2. if referencing another reference first, that one shall become + // bidirectional, the rest shall be bidirectional already + let path = path.to_vec(); // TODO: need for support for references in path library let reference_path = cost_return_on_error!( &mut cost, @@ -239,6 +244,16 @@ impl GroveDb { ) ); + if matches!( + referenced_item, + Element::Tree(_, _) | Element::SumTree(_, _, _) + ) { + return Err(Error::NotSupported( + "References cannot point to subtrees".to_owned(), + )) + .wrap_with_cost(cost); + } + let referenced_element_value_hash = cost_return_on_error!(&mut cost, referenced_item.value_hash(grove_version)); @@ -273,6 +288,16 @@ impl GroveDb { } } _ => { + // TODO: Check if overwriting an backreference-flavored item: + // 1. Item/SumItem -- update reference chains + // 2. Replace bidirectional reference with item: Previous in chain -- + // update hashes Next in chain reference -- make one directional + // reference if the last one Next is an actual item -- downgrade from + // backreferenced-flavor if the last one + // 3. Replace bidirectional reference with another reference: Old target + // reference becomes one directional if the last one Old target item is + // downgraded if the last one Update previous in chain with a new + // target cost_return_on_error!( &mut cost, element.insert( From ac4b627723cf49a480408af4ab7f89b902a1d835 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Mon, 4 Nov 2024 15:05:28 +0100 Subject: [PATCH 10/27] wip --- grovedb/src/lib.rs | 2 + grovedb/src/merk_cache.rs | 205 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 207 insertions(+) create mode 100644 grovedb/src/merk_cache.rs diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 42efe8a3..5b0a3e3c 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -135,6 +135,8 @@ pub mod element; pub mod error; #[cfg(feature = "estimated_costs")] mod estimated_costs; +#[cfg(feature = "full")] +mod merk_cache; #[cfg(any(feature = "full", feature = "verify"))] pub mod operations; #[cfg(any(feature = "full", feature = "verify"))] diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs new file mode 100644 index 00000000..b691fbd0 --- /dev/null +++ b/grovedb/src/merk_cache.rs @@ -0,0 +1,205 @@ +//! Module dedicated to keep necessary Merks in memory and solve propagation +//! after usage automatically. + +use std::{ + collections::{hash_map::Entry, HashMap, HashSet}, + mem::{self, MaybeUninit}, + ops::Deref, +}; + +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt}; +use grovedb_merk::Merk; +use grovedb_path::SubtreePath; +use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; +use grovedb_version::version::GroveVersion; + +use crate::{Error, GroveDb, Transaction}; + +type TxMerk<'db> = Merk>; + +/// Merk caching structure. +/// +/// Since we usually postpone all writes to the very end with a single RocksDB +/// batch all intermediate changes to subtrees might not be tracked if we reopen +/// those Merks, so it's better to have them cached and proceed through the same +/// structure. Eventually we'll have enough info at the same place to perform +/// necessary propagations as well. +pub(crate) struct MerkCache<'db, 'b, B> { + db: &'db GroveDb, + tx: &'db Transaction<'db>, + batch: &'db StorageBatch, + version: &'db GroveVersion, + inner: HashMap, TxMerk<'db>>, +} + +impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { + pub(crate) fn new( + db: &'db GroveDb, + tx: &'db Transaction<'db>, + batch: &'db StorageBatch, + version: &'db GroveVersion, + ) -> Self { + MerkCache { + db, + tx, + batch, + version, + inner: Default::default(), + } + } + + /// Get a mutable Merk reference from the cache. + /// If it doesn't present then it will be opened. + /// Returns `None` if there is no Merk under this path. + fn get_merk_mut_internal<'s>( + &'s mut self, + path: SubtreePath<'b, B>, + ) -> CostResult<&'s mut TxMerk<'db>, Error> { + let mut cost = Default::default(); + + match self.inner.entry(path) { + Entry::Occupied(e) => Ok(e.into_mut()).wrap_with_cost(cost), + Entry::Vacant(e) => { + let merk = cost_return_on_error!( + &mut cost, + self.db.open_transactional_merk_at_path( + e.key().clone(), + self.tx, + Some(self.batch), + self.version + ) + ); + Ok(e.insert(merk)).wrap_with_cost(cost) + } + } + } + + /// Returns an array of mutable references to different Merks, where each + /// element in the array corresponds to a unique Merk based on its + /// position in the input paths array. + /// + /// # Panics + /// All input paths *must* be unique, otherwise it could provide multiple + /// mutable references to the same memory which is strictly prohibited. + pub(crate) fn get_multi_mut<'s, const N: usize>( + &'s mut self, + paths: [SubtreePath<'b, B>; N], + ) -> CostResult<[MerkHandle<'db, 's>; N], Error> { + let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + let mut cost = Default::default(); + + let unique_args: HashSet<_> = paths.iter().collect(); + if unique_args.len() != N { + panic!("`get_multi_mut` keys must be unique"); + } + + for (i, path) in paths.into_iter().enumerate() { + // SAFETY is ensured by tying the lifetime of mutable references to the + // collection itself, preventing them from outliving the collection and + // ensuring exclusive access to the collection's layout through other + // mutable references. The mandatory keys' uniqueness check above makes + // sure no overlapping memory will be referenced. + let merk_ref = unsafe { + MerkHandle( + (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) + as *mut TxMerk<'db>) + .as_mut::<'s>() + .expect("not a null pointer"), + ) + }; + result_uninit[i].write(merk_ref); + } + + // SAFETY: An array of `MaybeUninit` references takes the same size as an array + // of references as long as they both have the same number of elements, + // N in our case. `mem::transmute` would represent it better, however, + // due to poor support of const generics in stable Rust we bypass + // compile-time size checks with pointer casts. + let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; + mem::forget(result_uninit); + + Ok(result).wrap_with_cost(cost) + } +} + +/// Handle to a cached Merk. +pub(crate) struct MerkHandle<'db, 'c>(&'c mut TxMerk<'db>); + +/// It is allowed to dereference `MerkHandle` to regular Merks but in a +/// non-mutable way since we want to track what have been done to those Merks. +impl<'db, 'c> Deref for MerkHandle<'db, 'c> { + type Target = TxMerk<'db>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'db, 'c> MerkHandle<'db, 'c> { + pub(crate) fn insert(&mut self) { + todo!() + } +} + +#[cfg(test)] +mod tests { + use grovedb_costs::OperationCost; + use grovedb_path::SubtreePath; + use grovedb_storage::StorageBatch; + use grovedb_version::version::GroveVersion; + + use super::MerkCache; + use crate::tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}; + + #[test] + fn cached_subtrees_are_free() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let mut cache = MerkCache::new(&db, &tx, &batch, version); + + let mut cost: OperationCost = Default::default(); + let [test1, test2] = cache + .get_multi_mut([ + SubtreePath::from(&[TEST_LEAF]), + SubtreePath::from(&[ANOTHER_TEST_LEAF]), + ]) + .unwrap_add_cost(&mut cost) + .expect("unable to get subtrees"); + + // Assert trees aren't empty + assert!(test1.root_hash().unwrap() != [0; 32]); + assert!(test2.root_hash().unwrap() != [0; 32]); + + // Assert some cost been paid + assert!(!cost.is_nothing()); + + let mut next_cost: OperationCost = Default::default(); + let [_test1, _test2] = cache + .get_multi_mut([ + SubtreePath::from(&[TEST_LEAF]), + SubtreePath::from(&[ANOTHER_TEST_LEAF]), + ]) + .unwrap_add_cost(&mut next_cost) + .expect("unable to get subtrees"); + + // Assert it was for free now + assert!(next_cost.is_nothing()); + } + + #[test] + #[should_panic] + fn overlapping_references_should_panic() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + let batch = StorageBatch::new(); + let mut cache = MerkCache::new(&db, &tx, &batch, version); + + let _ = cache.get_multi_mut([ + SubtreePath::from(&[TEST_LEAF]), + SubtreePath::from(&[TEST_LEAF]), + ]); + } +} From 740d97d5c9c60d483dade0df9f091e94c96d863c Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 5 Nov 2024 11:34:46 +0100 Subject: [PATCH 11/27] wip --- costs/src/context.rs | 9 +++++ grovedb/src/merk_cache.rs | 37 +++++++++++++------- path/src/subtree_path.rs | 72 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 14 deletions(-) diff --git a/costs/src/context.rs b/costs/src/context.rs index 8c0faffd..374466ef 100644 --- a/costs/src/context.rs +++ b/costs/src/context.rs @@ -116,6 +116,15 @@ impl CostResult { pub fn cost_as_result(self) -> Result { self.value.map(|_| self.cost) } + + /// Call the provided function on success without altering result or cost. + pub fn for_ok(self, f: impl FnOnce(&T)) -> CostResult { + if let Ok(x) = &self.value { + f(x) + } + + self + } } impl CostResult, E> { diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index b691fbd0..6053a833 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -2,18 +2,18 @@ //! after usage automatically. use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{btree_map::Entry, BTreeMap, HashSet}, mem::{self, MaybeUninit}, ops::Deref, }; use grovedb_costs::{cost_return_on_error, CostResult, CostsExt}; -use grovedb_merk::Merk; +use grovedb_merk::{Merk, MerkOptions}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; use grovedb_version::version::GroveVersion; -use crate::{Error, GroveDb, Transaction}; +use crate::{Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; @@ -29,7 +29,7 @@ pub(crate) struct MerkCache<'db, 'b, B> { tx: &'db Transaction<'db>, batch: &'db StorageBatch, version: &'db GroveVersion, - inner: HashMap, TxMerk<'db>>, + merks: BTreeMap, TxMerk<'db>>, } impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { @@ -44,7 +44,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { tx, batch, version, - inner: Default::default(), + merks: Default::default(), } } @@ -57,7 +57,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { ) -> CostResult<&'s mut TxMerk<'db>, Error> { let mut cost = Default::default(); - match self.inner.entry(path) { + match self.merks.entry(path) { Entry::Occupied(e) => Ok(e.into_mut()).wrap_with_cost(cost), Entry::Vacant(e) => { let merk = cost_return_on_error!( @@ -100,12 +100,13 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // mutable references. The mandatory keys' uniqueness check above makes // sure no overlapping memory will be referenced. let merk_ref = unsafe { - MerkHandle( - (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) + MerkHandle { + merk: (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) as *mut TxMerk<'db>) .as_mut::<'s>() .expect("not a null pointer"), - ) + version: &self.version, + } }; result_uninit[i].write(merk_ref); } @@ -123,7 +124,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } /// Handle to a cached Merk. -pub(crate) struct MerkHandle<'db, 'c>(&'c mut TxMerk<'db>); +pub(crate) struct MerkHandle<'db, 'c> { + merk: &'c mut TxMerk<'db>, + version: &'db GroveVersion, +} /// It is allowed to dereference `MerkHandle` to regular Merks but in a /// non-mutable way since we want to track what have been done to those Merks. @@ -131,13 +135,20 @@ impl<'db, 'c> Deref for MerkHandle<'db, 'c> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { - &self.0 + &self.merk } } impl<'db, 'c> MerkHandle<'db, 'c> { - pub(crate) fn insert(&mut self) { - todo!() + pub(crate) fn insert( + &mut self, + key: impl AsRef<[u8]>, + element: Element, + options: Option, + ) -> CostResult<(), Error> { + element + .insert(self.merk, key, options, self.version) + .for_ok(|_| todo!()) } } diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index 437f911a..b6132531 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -34,7 +34,10 @@ //! combined with it's various `From` implementations it can cover slices, owned //! subtree paths and other path references if use as generic [Into]. -use std::hash::{Hash, Hasher}; +use std::{ + cmp, + hash::{Hash, Hasher}, +}; use crate::{ subtree_path_builder::{SubtreePathBuilder, SubtreePathRelative}, @@ -78,6 +81,48 @@ where } } +/// First and foremost, the order of subtree paths is dictated by their lengths. +/// Therefore, those subtrees closer to the root will come first. The rest it +/// can guarantee is to be free of false equality; however, seemingly unrelated +/// subtrees can come one after another if they share the same length, which was +/// (not) done for performance reasons. +impl<'bl, 'br, BL, BR> PartialOrd> for SubtreePath<'bl, BL> +where + BL: AsRef<[u8]>, + BR: AsRef<[u8]>, +{ + fn partial_cmp(&self, other: &SubtreePath<'br, BR>) -> Option { + let iter_a = self.clone().into_reverse_iter(); + let iter_b = other.clone().into_reverse_iter(); + + Some( + iter_a + .len() + .cmp(&iter_b.len()) + .then_with(|| iter_a.cmp(iter_b)), + ) + } +} + +impl<'bl, 'br, BL, BR> PartialOrd> for SubtreePath<'bl, BL> +where + BL: AsRef<[u8]>, + BR: AsRef<[u8]>, +{ + fn partial_cmp(&self, other: &SubtreePathBuilder<'br, BR>) -> Option { + self.partial_cmp(&SubtreePath::from(other)) + } +} + +impl<'bl, BL> Ord for SubtreePath<'bl, BL> +where + BL: AsRef<[u8]>, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.partial_cmp(other).expect("order is totally defined") + } +} + impl<'b, B: AsRef<[u8]>> Eq for SubtreePath<'b, B> {} impl<'b, B> From> for SubtreePath<'b, B> { @@ -274,4 +319,29 @@ mod tests { assert_eq!(as_vec, reference_vec); assert_eq!(parent.len(), reference_vec.len()); } + + #[test] + fn ordering() { + let path_a: SubtreePath<_> = (&[b"one" as &[u8], b"two", b"three"]).into(); + let path_b = path_a.derive_owned_with_child(b"four"); + let path_c = path_a.derive_owned_with_child(b"notfour"); + let (path_d_parent, _) = path_a.derive_parent().unwrap(); + let path_d = path_d_parent.derive_owned_with_child(b"three"); + + // Same lengths for different paths don't make them equal: + assert!(!matches!( + SubtreePath::from(&path_b).cmp(&SubtreePath::from(&path_c)), + cmp::Ordering::Equal + )); + + // Equal paths made the same way are equal: + assert!(matches!( + path_a.cmp(&SubtreePath::from(&path_d)), + cmp::Ordering::Equal + )); + + // Longer paths are "greater than": + assert!(path_a < path_b); + assert!(path_a < path_c); + } } From eb7b6b2d372edf28947351964efc2d2377db45e7 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 5 Nov 2024 17:50:48 +0100 Subject: [PATCH 12/27] wip --- grovedb/src/merk_cache.rs | 55 ++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 6053a833..7804fad3 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -17,6 +17,11 @@ use crate::{Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; +struct CachedMerk<'db> { + updated: bool, + merk: TxMerk<'db>, +} + /// Merk caching structure. /// /// Since we usually postpone all writes to the very end with a single RocksDB @@ -29,7 +34,7 @@ pub(crate) struct MerkCache<'db, 'b, B> { tx: &'db Transaction<'db>, batch: &'db StorageBatch, version: &'db GroveVersion, - merks: BTreeMap, TxMerk<'db>>, + merks: BTreeMap, CachedMerk<'db>>, } impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { @@ -54,7 +59,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { fn get_merk_mut_internal<'s>( &'s mut self, path: SubtreePath<'b, B>, - ) -> CostResult<&'s mut TxMerk<'db>, Error> { + ) -> CostResult<&'s mut CachedMerk<'db>, Error> { let mut cost = Default::default(); match self.merks.entry(path) { @@ -69,7 +74,11 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { self.version ) ); - Ok(e.insert(merk)).wrap_with_cost(cost) + Ok(e.insert(CachedMerk { + merk, + updated: false, + })) + .wrap_with_cost(cost) } } } @@ -84,8 +93,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { pub(crate) fn get_multi_mut<'s, const N: usize>( &'s mut self, paths: [SubtreePath<'b, B>; N], - ) -> CostResult<[MerkHandle<'db, 's>; N], Error> { - let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + ) -> CostResult<[MerkHandle<'db, 's, 'b, B>; N], Error> { + let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; let mut cost = Default::default(); let unique_args: HashSet<_> = paths.iter().collect(); @@ -100,13 +109,14 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // mutable references. The mandatory keys' uniqueness check above makes // sure no overlapping memory will be referenced. let merk_ref = unsafe { - MerkHandle { - merk: (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) - as *mut TxMerk<'db>) + MerkHandle::new( + path.clone(), + (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) + as *mut CachedMerk<'db>) .as_mut::<'s>() .expect("not a null pointer"), - version: &self.version, - } + &self.version, + ) }; result_uninit[i].write(merk_ref); } @@ -116,7 +126,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // N in our case. `mem::transmute` would represent it better, however, // due to poor support of const generics in stable Rust we bypass // compile-time size checks with pointer casts. - let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; + let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; mem::forget(result_uninit); Ok(result).wrap_with_cost(cost) @@ -124,14 +134,16 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } /// Handle to a cached Merk. -pub(crate) struct MerkHandle<'db, 'c> { +pub(crate) struct MerkHandle<'db, 'c, 'b, B> { + path: SubtreePath<'b, B>, merk: &'c mut TxMerk<'db>, version: &'db GroveVersion, + updated: &'c mut bool, } /// It is allowed to dereference `MerkHandle` to regular Merks but in a /// non-mutable way since we want to track what have been done to those Merks. -impl<'db, 'c> Deref for MerkHandle<'db, 'c> { +impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { @@ -139,7 +151,7 @@ impl<'db, 'c> Deref for MerkHandle<'db, 'c> { } } -impl<'db, 'c> MerkHandle<'db, 'c> { +impl<'db, 'c, 'b, B> MerkHandle<'db, 'c, 'b, B> { pub(crate) fn insert( &mut self, key: impl AsRef<[u8]>, @@ -148,7 +160,20 @@ impl<'db, 'c> MerkHandle<'db, 'c> { ) -> CostResult<(), Error> { element .insert(self.merk, key, options, self.version) - .for_ok(|_| todo!()) + .for_ok(|_| *self.updated = true) + } + + fn new( + path: SubtreePath<'b, B>, + cached_merk: &'c mut CachedMerk<'db>, + version: &'db GroveVersion, + ) -> Self { + Self { + path, + merk: &mut cached_merk.merk, + version, + updated: &mut cached_merk.updated, + } } } From ea61c390e69524ecff701a8b0f9c812978717463 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 5 Nov 2024 18:49:37 +0100 Subject: [PATCH 13/27] wip --- grovedb/src/merk_cache.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 7804fad3..31219ee0 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -30,25 +30,24 @@ struct CachedMerk<'db> { /// structure. Eventually we'll have enough info at the same place to perform /// necessary propagations as well. pub(crate) struct MerkCache<'db, 'b, B> { + merks: BTreeMap, CachedMerk<'db>>, db: &'db GroveDb, tx: &'db Transaction<'db>, - batch: &'db StorageBatch, + batch: &'static StorageBatch, version: &'db GroveVersion, - merks: BTreeMap, CachedMerk<'db>>, } impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { pub(crate) fn new( db: &'db GroveDb, tx: &'db Transaction<'db>, - batch: &'db StorageBatch, version: &'db GroveVersion, ) -> Self { MerkCache { db, tx, - batch, version, + batch: Box::leak(Box::new(StorageBatch::default())), merks: Default::default(), } } @@ -131,6 +130,19 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { Ok(result).wrap_with_cost(cost) } + + /// Summarizes all performed operations on this `MerkCache` with necessary + /// propagations into a `Storagebatch`. + pub(crate) fn finalize(self) -> Box { + let batch_ptr = self.batch as *const _; + // Remove all possible usages of the `StorageBatch`: + drop(self); + + // SAFETY: The batch reference was created by `Box::leak` and restored into a + // `Box` form. No shared usage of that memory exists because the + // `MerkCache` structure was dropped above. + unsafe { Box::from_raw(batch_ptr as *mut _) } + } } /// Handle to a cached Merk. @@ -193,7 +205,7 @@ mod tests { let db = make_deep_tree(&version); let tx = db.start_transaction(); let batch = StorageBatch::new(); - let mut cache = MerkCache::new(&db, &tx, &batch, version); + let mut cache = MerkCache::new(&db, &tx, version); let mut cost: OperationCost = Default::default(); let [test1, test2] = cache @@ -231,7 +243,7 @@ mod tests { let db = make_deep_tree(&version); let tx = db.start_transaction(); let batch = StorageBatch::new(); - let mut cache = MerkCache::new(&db, &tx, &batch, version); + let mut cache = MerkCache::new(&db, &tx, version); let _ = cache.get_multi_mut([ SubtreePath::from(&[TEST_LEAF]), From 2f134ac2473272f6d94ce894279c71f27a5f698c Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 6 Nov 2024 17:26:26 +0100 Subject: [PATCH 14/27] wip --- grovedb/src/merk_cache.rs | 140 +++++++++++++++++++++++++++++--------- path/src/subtree_path.rs | 7 +- storage/src/storage.rs | 4 +- 3 files changed, 112 insertions(+), 39 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 31219ee0..42e48ce2 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -18,7 +18,7 @@ use crate::{Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; struct CachedMerk<'db> { - updated: bool, + to_propagate: bool, merk: TxMerk<'db>, } @@ -29,6 +29,7 @@ struct CachedMerk<'db> { /// those Merks, so it's better to have them cached and proceed through the same /// structure. Eventually we'll have enough info at the same place to perform /// necessary propagations as well. +// SAFETY: please consult with other safety docs here before doing any changes pub(crate) struct MerkCache<'db, 'b, B> { merks: BTreeMap, CachedMerk<'db>>, db: &'db GroveDb, @@ -38,14 +39,14 @@ pub(crate) struct MerkCache<'db, 'b, B> { } impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { - pub(crate) fn new( + pub(crate) fn new<'tx>( db: &'db GroveDb, tx: &'db Transaction<'db>, version: &'db GroveVersion, ) -> Self { MerkCache { db, - tx, + tx: &tx, version, batch: Box::leak(Box::new(StorageBatch::default())), merks: Default::default(), @@ -55,10 +56,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// Get a mutable Merk reference from the cache. /// If it doesn't present then it will be opened. /// Returns `None` if there is no Merk under this path. - fn get_merk_mut_internal<'s>( - &'s mut self, + fn get_merk_mut_internal<'c>( + &'c mut self, path: SubtreePath<'b, B>, - ) -> CostResult<&'s mut CachedMerk<'db>, Error> { + ) -> CostResult<&'c mut CachedMerk<'db>, Error> { let mut cost = Default::default(); match self.merks.entry(path) { @@ -75,7 +76,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { ); Ok(e.insert(CachedMerk { merk, - updated: false, + to_propagate: false, })) .wrap_with_cost(cost) } @@ -89,11 +90,11 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// # Panics /// All input paths *must* be unique, otherwise it could provide multiple /// mutable references to the same memory which is strictly prohibited. - pub(crate) fn get_multi_mut<'s, const N: usize>( - &'s mut self, + pub(crate) fn get_multi_mut<'c, const N: usize>( + &'c mut self, paths: [SubtreePath<'b, B>; N], - ) -> CostResult<[MerkHandle<'db, 's, 'b, B>; N], Error> { - let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + ) -> CostResult<[MerkHandle<'db, 'c>; N], Error> { + let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; let mut cost = Default::default(); let unique_args: HashSet<_> = paths.iter().collect(); @@ -109,10 +110,9 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // sure no overlapping memory will be referenced. let merk_ref = unsafe { MerkHandle::new( - path.clone(), (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) as *mut CachedMerk<'db>) - .as_mut::<'s>() + .as_mut::<'c>() .expect("not a null pointer"), &self.version, ) @@ -125,7 +125,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // N in our case. `mem::transmute` would represent it better, however, // due to poor support of const generics in stable Rust we bypass // compile-time size checks with pointer casts. - let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; + let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; mem::forget(result_uninit); Ok(result).wrap_with_cost(cost) @@ -133,29 +133,76 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// Summarizes all performed operations on this `MerkCache` with necessary /// propagations into a `Storagebatch`. - pub(crate) fn finalize(self) -> Box { + pub(crate) fn finalize(mut self) -> CostResult, Error> { let batch_ptr = self.batch as *const _; - // Remove all possible usages of the `StorageBatch`: - drop(self); + + // Propagate updated subtrees' hashes up to the root and dropping all possible + // batch users: + let propagation_result = self.propagate_updated_merks(); // SAFETY: The batch reference was created by `Box::leak` and restored into a // `Box` form. No shared usage of that memory exists because the // `MerkCache` structure was dropped above. - unsafe { Box::from_raw(batch_ptr as *mut _) } + let result_batch = unsafe { Box::from_raw(batch_ptr as *mut _) }; + + // The batch's unsafety cleanup happens regardless of propagation results, so no + // early termination allowed. We do the mapping afterwards: + propagation_result.map_ok(|_| result_batch) + } + + /// Finalizes each Merk starting from the deepest subtrees, updating hashes + /// up to the root. + fn propagate_updated_merks(mut self) -> CostResult<(), Error> { + let mut cost = Default::default(); + + let version: &'db GroveVersion = self.version; + + // Picking Merks one by one as long as they have a parent + while let Some((parent_path, parent_key, subtree)) = + self.merks.pop_first().and_then(|(path, subtree)| { + path.derive_parent() + .map(|(parent_path, parent_key)| (parent_path, parent_key, subtree)) + }) + { + // If a cached Merk wasn't changed, we don't need to propagate it: + if !subtree.to_propagate { + continue; + } + + let parent_subtree = + cost_return_on_error!(&mut cost, self.get_merk_mut_internal(parent_path)); + parent_subtree.to_propagate = true; + let (root_hash, root_key, root_sum) = cost_return_on_error!( + &mut cost, + subtree.merk.root_hash_key_and_sum().map_err(|e| e.into()) + ); + cost_return_on_error!( + &mut cost, + GroveDb::update_tree_item_preserve_flag( + &mut parent_subtree.merk, + parent_key, + root_key, + root_hash, + root_sum, + version + ) + ); + } + + Ok(()).wrap_with_cost(cost) } } /// Handle to a cached Merk. -pub(crate) struct MerkHandle<'db, 'c, 'b, B> { - path: SubtreePath<'b, B>, +pub(crate) struct MerkHandle<'db, 'c> { merk: &'c mut TxMerk<'db>, version: &'db GroveVersion, - updated: &'c mut bool, + to_propagate: &'c mut bool, } /// It is allowed to dereference `MerkHandle` to regular Merks but in a /// non-mutable way since we want to track what have been done to those Merks. -impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { +impl<'db, 'c> Deref for MerkHandle<'db, 'c> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { @@ -163,7 +210,7 @@ impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { } } -impl<'db, 'c, 'b, B> MerkHandle<'db, 'c, 'b, B> { +impl<'db, 'c> MerkHandle<'db, 'c> { pub(crate) fn insert( &mut self, key: impl AsRef<[u8]>, @@ -172,19 +219,14 @@ impl<'db, 'c, 'b, B> MerkHandle<'db, 'c, 'b, B> { ) -> CostResult<(), Error> { element .insert(self.merk, key, options, self.version) - .for_ok(|_| *self.updated = true) + .for_ok(|_| *self.to_propagate = true) } - fn new( - path: SubtreePath<'b, B>, - cached_merk: &'c mut CachedMerk<'db>, - version: &'db GroveVersion, - ) -> Self { + fn new(cached_merk: &'c mut CachedMerk<'db>, version: &'db GroveVersion) -> Self { Self { - path, merk: &mut cached_merk.merk, version, - updated: &mut cached_merk.updated, + to_propagate: &mut cached_merk.to_propagate, } } } @@ -197,14 +239,16 @@ mod tests { use grovedb_version::version::GroveVersion; use super::MerkCache; - use crate::tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}; + use crate::{ + tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}, + Element, + }; #[test] fn cached_subtrees_are_free() { let version = GroveVersion::latest(); let db = make_deep_tree(&version); let tx = db.start_transaction(); - let batch = StorageBatch::new(); let mut cache = MerkCache::new(&db, &tx, version); let mut cost: OperationCost = Default::default(); @@ -242,7 +286,6 @@ mod tests { let version = GroveVersion::latest(); let db = make_deep_tree(&version); let tx = db.start_transaction(); - let batch = StorageBatch::new(); let mut cache = MerkCache::new(&db, &tx, version); let _ = cache.get_multi_mut([ @@ -250,4 +293,33 @@ mod tests { SubtreePath::from(&[TEST_LEAF]), ]); } + + #[test] + fn changes_to_merk_cache_provide_non_empty_batch() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + + let mut cache = MerkCache::new(&db, &tx, version); + let [_subtree] = cache + .get_multi_mut([SubtreePath::from(&[TEST_LEAF])]) + .unwrap() + .unwrap(); + + // Do nothing and finalize a batch: + assert!(cache.finalize().unwrap().unwrap().len() == 0); + + let mut cache = MerkCache::new(&db, &tx, version); + let [mut subtree] = cache + .get_multi_mut([SubtreePath::from(&[TEST_LEAF])]) + .unwrap() + .unwrap(); + subtree + .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .unwrap() + .unwrap(); + + // Do something and finalize another batch: + assert!(cache.finalize().unwrap().unwrap().len() > 0); + } } diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index b6132531..502a74c9 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -99,6 +99,7 @@ where iter_a .len() .cmp(&iter_b.len()) + .reverse() .then_with(|| iter_a.cmp(iter_b)), ) } @@ -340,8 +341,8 @@ mod tests { cmp::Ordering::Equal )); - // Longer paths are "greater than": - assert!(path_a < path_b); - assert!(path_a < path_c); + // Longer paths come first + assert!(path_a > path_b); + assert!(path_a > path_c); } } diff --git a/storage/src/storage.rs b/storage/src/storage.rs index 13f8d083..7b0435b6 100644 --- a/storage/src/storage.rs +++ b/storage/src/storage.rs @@ -305,8 +305,8 @@ impl StorageBatch { } } - #[cfg(test)] - pub(crate) fn len(&self) -> usize { + /// Returns a number of operations in the batch. + pub fn len(&self) -> usize { let operations = self.operations.borrow(); operations.data.len() + operations.roots.len() From 72f519f5c1d716e0124d2d8a0a54e64362a57bac Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 6 Nov 2024 20:39:56 +0100 Subject: [PATCH 15/27] wip --- grovedb/src/merk_cache.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 42e48ce2..3ed25653 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -235,7 +235,7 @@ impl<'db, 'c> MerkHandle<'db, 'c> { mod tests { use grovedb_costs::OperationCost; use grovedb_path::SubtreePath; - use grovedb_storage::StorageBatch; + use grovedb_storage::{Storage, StorageBatch}; use grovedb_version::version::GroveVersion; use super::MerkCache; @@ -322,4 +322,35 @@ mod tests { // Do something and finalize another batch: assert!(cache.finalize().unwrap().unwrap().len() > 0); } + + #[test] + fn changes_to_merk_are_propagated() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + + let pre_hash = db.root_hash(Some(&tx), &version).unwrap().unwrap(); + + let mut cache = MerkCache::new(&db, &tx, version); + let [mut subtree] = cache + .get_multi_mut([SubtreePath::from(&[TEST_LEAF, b"innertree"])]) + .unwrap() + .unwrap(); + subtree + .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .unwrap() + .unwrap(); + + let batch = cache.finalize().unwrap().unwrap(); + + db.db + .commit_multi_context_batch(*batch, Some(&tx)) + .unwrap() + .unwrap(); + + assert_ne!( + pre_hash, + db.root_hash(Some(&tx), &version).unwrap().unwrap() + ) + } } From cd475898fa275bcf46c448f7a15c272fba13c5e0 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 7 Nov 2024 13:12:57 +0100 Subject: [PATCH 16/27] wip --- costs/src/context.rs | 5 +- grovedb/src/element/insert.rs | 50 ++++++++++- grovedb/src/element/mod.rs | 52 +++++++++++ grovedb/src/merk_cache.rs | 159 +++++++++++++++++++++++++--------- 4 files changed, 221 insertions(+), 45 deletions(-) diff --git a/costs/src/context.rs b/costs/src/context.rs index 374466ef..8224cb88 100644 --- a/costs/src/context.rs +++ b/costs/src/context.rs @@ -179,8 +179,9 @@ impl CostsExt for T {} /// 1. Early termination on error; /// 2. Because of 1, `Result` is removed from the equation; /// 3. `CostContext` is removed too because it is added to external cost -/// accumulator; 4. Early termination uses external cost accumulator so previous -/// costs won't be lost. +/// accumulator; +/// 4. Early termination uses external cost accumulator so previous costs won't +/// be lost. #[macro_export] macro_rules! cost_return_on_error { ( &mut $cost:ident, $($body:tt)+ ) => { diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 432ff2f0..5c381e52 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -190,6 +190,38 @@ impl Element { } } + #[cfg(feature = "full")] + /// Promote `Element` to referenced variant in case the old one was already + /// referenced. + fn promote_to_referenced_variant(self, old_element: &mut Element) -> Result { + if let Some(refs) = old_element.take_backward_references() { + // Since variants with backward references are publicly available, we still have + // to address them, meaning filling in the actual information about references + // from the database by discarding user input. + + match self { + Element::Item(value, flags) + | Element::ItemWithBackwardsReferences(value, _, flags) => { + Ok(Element::ItemWithBackwardsReferences(value, refs, flags)) + } + Element::Reference(ref_path, max_hops, flags) + | Element::BidirectionalReference(ref_path, _, max_hops, flags) => Ok( + Element::BidirectionalReference(ref_path, refs, max_hops, flags), + ), + Element::SumItem(sum, flags) + | Element::SumItemWithBackwardsReferences(sum, _, flags) => { + Ok(Element::SumItemWithBackwardsReferences(sum, refs, flags)) + } + + Element::Tree(..) | Element::SumTree(..) => Err(Error::NotSupported( + "cannot insert subtree in place of a referenced item".to_owned(), + )), + } + } else { + Ok(self) + } + } + #[cfg(feature = "full")] /// Insert an element in Merk under a key if the value is different from /// what already exists; path should be resolved and proper Merk should @@ -198,8 +230,10 @@ impl Element { /// will be committed on the transaction commit. /// The bool represents if we indeed inserted. /// If the value changed we return the old element. + // TODO: a combo of `bool` and `Option::Some` in case `bool` equals + // to true could be covered just by `Option`. pub fn insert_if_changed_value<'db, S: StorageContext<'db>>( - &self, + self, merk: &mut Merk, key: &[u8], options: Option, @@ -214,18 +248,26 @@ impl Element { ); let mut cost = OperationCost::default(); - let previous_element = cost_return_on_error!( + let mut previous_element = cost_return_on_error!( &mut cost, Self::get_optional_from_storage(&merk.storage, key, grove_version) ); + let to_insert = if let Some(prev) = previous_element.as_mut() { + cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(prev)) + } else { + self + }; let needs_insert = match &previous_element { None => true, - Some(previous_element) => previous_element != self, + Some(previous_element) => !previous_element.eq_no_backreferences(&to_insert), }; if !needs_insert { Ok((false, None)).wrap_with_cost(cost) } else { - cost_return_on_error!(&mut cost, self.insert(merk, key, options, grove_version)); + cost_return_on_error!( + &mut cost, + to_insert.insert(merk, key, options, grove_version) + ); Ok((true, previous_element)).wrap_with_cost(cost) } } diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 691a15a4..25ec964f 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -22,6 +22,7 @@ pub use query::QueryOptions; mod serialize; #[cfg(any(feature = "full", feature = "verify"))] use std::fmt; +use std::mem; use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] @@ -306,6 +307,57 @@ impl Element { ); crate::value_hash(&bytes).map(Result::Ok) } + + /// Returns backward references if the `Element` in question participates in + /// bidirectional referencing machinery. + pub(crate) fn take_backward_references( + &mut self, + ) -> Option> { + match self { + Element::BidirectionalReference(_, refs, ..) + | Element::ItemWithBackwardsReferences(_, refs, ..) + | Element::SumItemWithBackwardsReferences(_, refs, ..) + if !refs.is_empty() => + { + Some(mem::take(refs)) + } + _ => None, + } + } + + /// Checks elements for equality ignoring backreferences part. + pub(crate) fn eq_no_backreferences(&self, other: &Self) -> bool { + use Element::*; + + match (self, other) { + ( + Item(value_left, flags_left) + | ItemWithBackwardsReferences(value_left, _, flags_left), + Item(value_right, flags_right) + | ItemWithBackwardsReferences(value_right, _, flags_right), + ) => value_left == value_right && flags_left == flags_right, + ( + SumItem(sum_left, flags_left) + | SumItemWithBackwardsReferences(sum_left, _, flags_left), + SumItem(sum_right, flags_right) + | SumItemWithBackwardsReferences(sum_right, _, flags_right), + ) => sum_left == sum_right && flags_left == flags_right, + ( + Reference(ref_path_left, max_hops_left, flags_left) + | BidirectionalReference(ref_path_left, _, max_hops_left, flags_left), + Reference(ref_path_right, max_hops_right, flags_right) + | BidirectionalReference(ref_path_right, _, max_hops_right, flags_right), + ) => { + ref_path_left == ref_path_right + && max_hops_left == max_hops_right + && flags_left == flags_right + } + (left @ Tree(..), right @ Tree(..)) => left == right, + (left @ SumTree(..), right @ SumTree(..)) => left == right, + + _ => false, + } + } } #[cfg(any(feature = "full", feature = "visualize"))] diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 3ed25653..6353a430 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -2,6 +2,8 @@ //! after usage automatically. use std::{ + borrow::Cow, + cell::RefCell, collections::{btree_map::Entry, BTreeMap, HashSet}, mem::{self, MaybeUninit}, ops::Deref, @@ -17,39 +19,39 @@ use crate::{Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; +type Key = Vec; + struct CachedMerk<'db> { to_propagate: bool, merk: TxMerk<'db>, } -/// Merk caching structure. -/// -/// Since we usually postpone all writes to the very end with a single RocksDB -/// batch all intermediate changes to subtrees might not be tracked if we reopen -/// those Merks, so it's better to have them cached and proceed through the same -/// structure. Eventually we'll have enough info at the same place to perform -/// necessary propagations as well. -// SAFETY: please consult with other safety docs here before doing any changes -pub(crate) struct MerkCache<'db, 'b, B> { +type UpdatedReferences<'b, B> = RefCell, Vec)>>; + +/// Helper struct to split `MerkCache` into independent parts to allow splitting +/// borrows, meaning a dependency on the storage part that shall have a certain +/// lifetime won't clash with another dependency unrelated to the storage part. +struct MerkCacheStorage<'db, 'b, B> { + /// Subtrees opened during usage of this cache structure, the wrapper also + /// marks those that were changed. merks: BTreeMap, CachedMerk<'db>>, + /// GroveDb provides a storage to open Merks against. db: &'db GroveDb, + /// Nowadays GroveDb operates solely on transactional storage contexts. tx: &'db Transaction<'db>, + /// The `MerkCache` finalization result is a `StorageBatch` of operations. + /// It's then up to the user what actions to take on that result until + /// no further changes can be made to the storage. batch: &'static StorageBatch, - version: &'db GroveVersion, } -impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { - pub(crate) fn new<'tx>( - db: &'db GroveDb, - tx: &'db Transaction<'db>, - version: &'db GroveVersion, - ) -> Self { - MerkCache { +impl<'db, 'b, B: AsRef<[u8]>> MerkCacheStorage<'db, 'b, B> { + fn new(db: &'db GroveDb, tx: &'db Transaction<'db>) -> Self { + MerkCacheStorage { + merks: Default::default(), db, - tx: &tx, - version, + tx, batch: Box::leak(Box::new(StorageBatch::default())), - merks: Default::default(), } } @@ -59,6 +61,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { fn get_merk_mut_internal<'c>( &'c mut self, path: SubtreePath<'b, B>, + version: &GroveVersion, ) -> CostResult<&'c mut CachedMerk<'db>, Error> { let mut cost = Default::default(); @@ -71,7 +74,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { e.key().clone(), self.tx, Some(self.batch), - self.version + version ) ); Ok(e.insert(CachedMerk { @@ -82,6 +85,36 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } } } +} + +/// Merk caching structure. +/// +/// Since we usually postpone all writes to the very end with a single RocksDB +/// batch all intermediate changes to subtrees might not be tracked if we reopen +/// those Merks, so it's better to have them cached and proceed through the same +/// structure. Eventually we'll have enough info at the same place to perform +/// necessary propagations as well. +// SAFETY: please consult with other safety docs here before doing any changes +pub(crate) struct MerkCache<'db, 'b, B> { + storage: MerkCacheStorage<'db, 'b, B>, + /// References require different kind of propagation and we track pointed to + /// values to update references. + updated_references: UpdatedReferences<'b, B>, + version: &'db GroveVersion, +} + +impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { + pub(crate) fn new<'tx>( + db: &'db GroveDb, + tx: &'db Transaction<'db>, + version: &'db GroveVersion, + ) -> Self { + MerkCache { + storage: MerkCacheStorage::new(db, tx), + version, + updated_references: Default::default(), + } + } /// Returns an array of mutable references to different Merks, where each /// element in the array corresponds to a unique Merk based on its @@ -93,8 +126,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { pub(crate) fn get_multi_mut<'c, const N: usize>( &'c mut self, paths: [SubtreePath<'b, B>; N], - ) -> CostResult<[MerkHandle<'db, 'c>; N], Error> { - let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + ) -> CostResult<[MerkHandle<'db, 'c, 'b, B>; N], Error> { + let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; let mut cost = Default::default(); let unique_args: HashSet<_> = paths.iter().collect(); @@ -110,10 +143,17 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // sure no overlapping memory will be referenced. let merk_ref = unsafe { MerkHandle::new( - (cost_return_on_error!(&mut cost, self.get_merk_mut_internal(path)) - as *mut CachedMerk<'db>) + (cost_return_on_error!( + &mut cost, + self.storage + .get_merk_mut_internal(path.clone(), self.version) + ) as *mut CachedMerk<'db>) .as_mut::<'c>() .expect("not a null pointer"), + UpdatedReferenceHandle { + path, + updated_references: &self.updated_references, + }, &self.version, ) }; @@ -125,7 +165,9 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // N in our case. `mem::transmute` would represent it better, however, // due to poor support of const generics in stable Rust we bypass // compile-time size checks with pointer casts. - let result = unsafe { (&result_uninit as *const _ as *const [MerkHandle; N]).read() }; + let result = unsafe { + (&result_uninit as *const _ as *const [MerkHandle<'db, 'c, 'b, B>; N]).read() + }; mem::forget(result_uninit); Ok(result).wrap_with_cost(cost) @@ -134,7 +176,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// Summarizes all performed operations on this `MerkCache` with necessary /// propagations into a `Storagebatch`. pub(crate) fn finalize(mut self) -> CostResult, Error> { - let batch_ptr = self.batch as *const _; + let batch_ptr = self.storage.batch as *const _; // Propagate updated subtrees' hashes up to the root and dropping all possible // batch users: @@ -150,6 +192,11 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { propagation_result.map_ok(|_| result_batch) } + /// Perform propagation of references' chains marked as changed. + fn propagate_updated_references(&mut self) -> CostResult<(), Error> { + todo!() + } + /// Finalizes each Merk starting from the deepest subtrees, updating hashes /// up to the root. fn propagate_updated_merks(mut self) -> CostResult<(), Error> { @@ -159,7 +206,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // Picking Merks one by one as long as they have a parent while let Some((parent_path, parent_key, subtree)) = - self.merks.pop_first().and_then(|(path, subtree)| { + self.storage.merks.pop_first().and_then(|(path, subtree)| { path.derive_parent() .map(|(parent_path, parent_key)| (parent_path, parent_key, subtree)) }) @@ -169,8 +216,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { continue; } - let parent_subtree = - cost_return_on_error!(&mut cost, self.get_merk_mut_internal(parent_path)); + let parent_subtree = cost_return_on_error!( + &mut cost, + self.storage.get_merk_mut_internal(parent_path, version) + ); parent_subtree.to_propagate = true; let (root_hash, root_key, root_sum) = cost_return_on_error!( &mut cost, @@ -194,15 +243,30 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } /// Handle to a cached Merk. -pub(crate) struct MerkHandle<'db, 'c> { +pub(crate) struct MerkHandle<'db, 'c, 'b, B> { merk: &'c mut TxMerk<'db>, version: &'db GroveVersion, to_propagate: &'c mut bool, + updated_reference_handle: UpdatedReferenceHandle<'c, 'b, B>, +} + +/// Helper struct to signal `MerkCache` about updated references. +struct UpdatedReferenceHandle<'c, 'b, B> { + path: SubtreePath<'b, B>, + updated_references: &'c UpdatedReferences<'b, B>, +} + +impl<'c, 'b, B: AsRef<[u8]>> UpdatedReferenceHandle<'c, 'b, B> { + fn mark_updated_reference(&self, key: Key) { + self.updated_references + .borrow_mut() + .insert((self.path.clone(), key)); + } } /// It is allowed to dereference `MerkHandle` to regular Merks but in a /// non-mutable way since we want to track what have been done to those Merks. -impl<'db, 'c> Deref for MerkHandle<'db, 'c> { +impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { @@ -210,23 +274,40 @@ impl<'db, 'c> Deref for MerkHandle<'db, 'c> { } } -impl<'db, 'c> MerkHandle<'db, 'c> { +impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { pub(crate) fn insert( &mut self, - key: impl AsRef<[u8]>, + key: &[u8], element: Element, options: Option, ) -> CostResult<(), Error> { - element - .insert(self.merk, key, options, self.version) - .for_ok(|_| *self.to_propagate = true) + let mut costs = Default::default(); + if let (_, Some(mut old_element)) = cost_return_on_error!( + &mut costs, + element.insert_if_changed_value(self.merk, key, options, self.version) + ) { + // In case the item that was changed has been referenced, we indicate that + // references should be propagated after + if old_element.take_backward_references().is_some() { + self.updated_reference_handle + .mark_updated_reference(key.to_vec()); + } + } + *self.to_propagate = true; + + Ok(()).wrap_with_cost(costs) } - fn new(cached_merk: &'c mut CachedMerk<'db>, version: &'db GroveVersion) -> Self { + fn new( + cached_merk: &'c mut CachedMerk<'db>, + updated_reference_handle: UpdatedReferenceHandle<'c, 'b, B>, + version: &'db GroveVersion, + ) -> Self { Self { merk: &mut cached_merk.merk, version, to_propagate: &mut cached_merk.to_propagate, + updated_reference_handle, } } } @@ -235,7 +316,7 @@ impl<'db, 'c> MerkHandle<'db, 'c> { mod tests { use grovedb_costs::OperationCost; use grovedb_path::SubtreePath; - use grovedb_storage::{Storage, StorageBatch}; + use grovedb_storage::Storage; use grovedb_version::version::GroveVersion; use super::MerkCache; From 644efb3f71586f1e5d7de1e8ca061d191bf3bf13 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 7 Nov 2024 15:54:47 +0100 Subject: [PATCH 17/27] wip --- grovedb/src/element/insert.rs | 86 ++++++++++++++++++++++++----------- grovedb/src/element/mod.rs | 7 +++ grovedb/src/merk_cache.rs | 17 +++---- 3 files changed, 75 insertions(+), 35 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 5c381e52..030d1510 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -23,7 +23,7 @@ impl Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. pub fn insert<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( - &self, + self, merk: &mut Merk, key: K, options: Option, @@ -31,21 +31,40 @@ impl Element { ) -> CostResult<(), Error> { check_grovedb_v0_with_cost!("insert", grove_version.grovedb_versions.element.insert); - let serialized = cost_return_on_error_default!(self.serialize(grove_version)); + let mut original_cost = Default::default(); + + let to_insert = if let Some(mut prev) = cost_return_on_error!( + &mut original_cost, + Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) + ) { + cost_return_on_error_no_add!( + original_cost, + self.promote_to_referenced_variant(&mut prev) + ) + } else { + self + }; + + let serialized = cost_return_on_error_default!(to_insert.serialize(grove_version)); - if !merk.is_sum_tree && self.is_sum_item() { + if !merk.is_sum_tree && to_insert.is_sum_item() { return Err(Error::InvalidInput("cannot add sum item to non sum tree")) - .wrap_with_cost(Default::default()); + .wrap_with_cost(original_cost); } let merk_feature_type = - cost_return_on_error_default!(self.get_feature_type(merk.is_sum_tree)); - let batch_operations = if matches!(self, Element::SumItem(..)) { - let value_cost = - cost_return_on_error_default!(self.get_specialized_cost(grove_version)); + cost_return_on_error_default!(to_insert.get_feature_type(merk.is_sum_tree)); + let batch_operations = if matches!( + to_insert, + Element::SumItem(..) | Element::SumItemWithBackwardsReferences(..) + ) { + let value_cost = cost_return_on_error_no_add!( + original_cost, + to_insert.get_specialized_cost(grove_version) + ); let cost = value_cost - + self.get_flags().as_ref().map_or(0, |flags| { + + to_insert.get_flags().as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); @@ -70,6 +89,7 @@ impl Element { grove_version, ) .map_err(|e| Error::CorruptedData(e.to_string())) + .add_cost(original_cost) } #[cfg(feature = "full")] @@ -122,7 +142,7 @@ impl Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. pub fn insert_if_not_exists<'db, S: StorageContext<'db>>( - &self, + self, merk: &mut Merk, key: &[u8], options: Option, @@ -228,10 +248,8 @@ impl Element { /// be loaded by this moment If transaction is not passed, the batch /// will be written immediately. If transaction is passed, the operation /// will be committed on the transaction commit. - /// The bool represents if we indeed inserted. - /// If the value changed we return the old element. - // TODO: a combo of `bool` and `Option::Some` in case `bool` equals - // to true could be covered just by `Option`. + /// The bool represents whether a propagation of references is needed. + /// If the value changed, it returns the old element under `Some`. pub fn insert_if_changed_value<'db, S: StorageContext<'db>>( self, merk: &mut Merk, @@ -257,18 +275,21 @@ impl Element { } else { self }; - let needs_insert = match &previous_element { - None => true, - Some(previous_element) => !previous_element.eq_no_backreferences(&to_insert), - }; - if !needs_insert { - Ok((false, None)).wrap_with_cost(cost) - } else { + + let changed = previous_element + .as_ref() + .map(|p| !p.eq_no_backreferences(&to_insert)) + .unwrap_or_default(); + + if changed { + let has_references = to_insert.has_backward_references(); cost_return_on_error!( &mut cost, to_insert.insert(merk, key, options, grove_version) ); - Ok((true, previous_element)).wrap_with_cost(cost) + Ok((has_references, previous_element)).wrap_with_cost(cost) + } else { + Ok((false, None)).wrap_with_cost(cost) } } @@ -329,7 +350,7 @@ impl Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. pub fn insert_reference<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( - &self, + self, merk: &mut Merk, key: K, referenced_value: Hash, @@ -341,15 +362,26 @@ impl Element { grove_version.grovedb_versions.element.insert_reference ); - let serialized = match self.serialize(grove_version) { + let mut cost = Default::default(); + + let to_insert = if let Some(mut prev) = cost_return_on_error!( + &mut cost, + Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) + ) { + cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)) + } else { + self + }; + + let serialized = match to_insert.serialize(grove_version) { Ok(s) => s, - Err(e) => return Err(e).wrap_with_cost(Default::default()), + Err(e) => return Err(e).wrap_with_cost(cost), }; - let mut cost = OperationCost::default(); let merk_feature_type = cost_return_on_error!( &mut cost, - self.get_feature_type(merk.is_sum_tree) + to_insert + .get_feature_type(merk.is_sum_tree) .wrap_with_cost(OperationCost::default()) ); diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 25ec964f..4c65da51 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -325,6 +325,13 @@ impl Element { } } + /// Returns true if there are references to this `Element`. + pub(crate) fn has_backward_references(&self) -> bool { + matches!(self, Element::BidirectionalReference(_, refs, ..) + | Element::ItemWithBackwardsReferences(_, refs, ..) + | Element::SumItemWithBackwardsReferences(_, refs, ..) if !refs.is_empty()) + } + /// Checks elements for equality ignoring backreferences part. pub(crate) fn eq_no_backreferences(&self, other: &Self) -> bool { use Element::*; diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 6353a430..70be0ea0 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -282,16 +282,17 @@ impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { options: Option, ) -> CostResult<(), Error> { let mut costs = Default::default(); - if let (_, Some(mut old_element)) = cost_return_on_error!( + + // In case the item that was changed has been referenced, we indicate that + // references should be propagated after + if cost_return_on_error!( &mut costs, element.insert_if_changed_value(self.merk, key, options, self.version) - ) { - // In case the item that was changed has been referenced, we indicate that - // references should be propagated after - if old_element.take_backward_references().is_some() { - self.updated_reference_handle - .mark_updated_reference(key.to_vec()); - } + ) + .0 + { + self.updated_reference_handle + .mark_updated_reference(key.to_vec()); } *self.to_propagate = true; From be17531ba89329cfa85a49b260198bec388ab749 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 7 Nov 2024 16:06:07 +0100 Subject: [PATCH 18/27] wip --- grovedb/src/element/insert.rs | 8 ++---- grovedb/src/operations/insert/mod.rs | 43 ++++++++++++++-------------- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 030d1510..ae231766 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -279,7 +279,7 @@ impl Element { let changed = previous_element .as_ref() .map(|p| !p.eq_no_backreferences(&to_insert)) - .unwrap_or_default(); + .unwrap_or(true); if changed { let has_references = to_insert.has_backward_references(); @@ -673,12 +673,11 @@ mod tests { let batch = StorageBatch::new(); let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); - let (inserted, previous) = Element::new_item(b"value2".to_vec()) + let (_, previous) = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); - assert!(inserted); assert_eq!(previous, Some(Element::new_item(b"value".to_vec())),); storage @@ -703,12 +702,11 @@ mod tests { .insert(&mut merk, b"mykey", None, grove_version) .unwrap() .expect("expected successful insertion"); - let (inserted, previous) = Element::new_item(b"value2".to_vec()) + let (_, previous) = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); - assert!(inserted); assert_eq!(previous, None); assert_eq!( diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 14bc1a34..6a07b41b 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -850,7 +850,8 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 3, // 1 to get tree, 1 to insert, 1 to insert into root tree + seek_count: 4, /* 1 to get tree, 1 to check prev value for ref propagation, 1 to + * insert, 1 to insert into root tree */ storage_cost: StorageCost { added_bytes: 149, replaced_bytes: 0, @@ -917,7 +918,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 5, + seek_count: 6, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 84, // todo: verify @@ -1000,7 +1001,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, + seek_count: 8, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 217, @@ -1079,7 +1080,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, + seek_count: 8, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 217, // todo: verify @@ -1140,7 +1141,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 3, // 1 to get tree, 1 to insert, 1 to insert into root tree + seek_count: 4, // 1 to get tree, 1 to insert, 1 to insert into root tree storage_cost: StorageCost { added_bytes: 153, replaced_bytes: 0, @@ -1405,7 +1406,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 5, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 150, replaced_bytes: 78, @@ -1470,7 +1471,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 3, // todo: verify this + seek_count: 4, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 0, @@ -1564,7 +1565,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 5, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 78, @@ -1660,7 +1661,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 5, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 82, @@ -1729,13 +1730,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 3, // todo: verify this + seek_count: 4, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 112, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 77, + storage_loaded_bytes: 124, hash_node_calls: 2, } ); @@ -1783,13 +1784,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 7, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 190, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 230, // todo verify this + storage_loaded_bytes: 277, // todo verify this hash_node_calls: 8, } ); @@ -1837,13 +1838,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 7, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 248, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 266, // todo verify this + storage_loaded_bytes: 411, // todo verify this hash_node_calls: 9, } ); @@ -1903,13 +1904,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 9, // todo: verify this + seek_count: 10, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 409, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 487, // todo verify this + storage_loaded_bytes: 632, // todo verify this hash_node_calls: 11, } ); @@ -1957,13 +1958,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 7, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 248, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 276, // todo verify this + storage_loaded_bytes: 421, // todo verify this hash_node_calls: 9, } ); @@ -2011,13 +2012,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 7, // todo: verify this storage_cost: StorageCost { added_bytes: 1, replaced_bytes: 191, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 231, + storage_loaded_bytes: 279, hash_node_calls: 8, } ); From 0d92cebd434410bce284f52c0840c3bd4fba7c02 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 8 Nov 2024 15:30:02 +0100 Subject: [PATCH 19/27] wip --- grovedb/src/element/insert.rs | 15 ++- grovedb/src/element/mod.rs | 13 ++ grovedb/src/merk_cache.rs | 244 ++++++++++++++++++++++++++++++++-- 3 files changed, 257 insertions(+), 15 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index ae231766..1d418d56 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -349,6 +349,7 @@ impl Element { /// If transaction is not passed, the batch will be written immediately. /// If transaction is passed, the operation will be committed on the /// transaction commit. + /// Returns `bool` that indicates if a reference propagation is required. pub fn insert_reference<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( self, merk: &mut Merk, @@ -356,7 +357,7 @@ impl Element { referenced_value: Hash, options: Option, grove_version: &GroveVersion, - ) -> CostResult<(), Error> { + ) -> CostResult { check_grovedb_v0_with_cost!( "insert_reference", grove_version.grovedb_versions.element.insert_reference @@ -364,13 +365,16 @@ impl Element { let mut cost = Default::default(); - let to_insert = if let Some(mut prev) = cost_return_on_error!( + let (ref_updated, to_insert) = if let Some(mut prev) = cost_return_on_error!( &mut cost, Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) ) { - cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)) + ( + !prev.eq_no_backreferences(&self) && prev.has_backward_references(), + cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)), + ) } else { - self + (false, self) }; let serialized = match to_insert.serialize(grove_version) { @@ -402,6 +406,7 @@ impl Element { grove_version, ) .map_err(|e| Error::CorruptedData(e.to_string())) + .map_ok(|_| ref_updated) } #[cfg(feature = "full")] @@ -540,7 +545,7 @@ impl Element { } /// Adds info on reference that points to this element. - fn referenced_from( + pub(crate) fn referenced_from( mut self, reference: ReferencePathType, cascade_on_update: CascadeOnUpdate, diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index 4c65da51..d607d35e 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -365,6 +365,19 @@ impl Element { _ => false, } } + + /// Downgrades `Element` variants with backward references to regular + /// variants. + pub(crate) fn cut_backreferences(self) -> Self { + match self { + Element::ItemWithBackwardsReferences(value, _, flags) => Element::Item(value, flags), + Element::SumItemWithBackwardsReferences(sum, _, flags) => Element::SumItem(sum, flags), + Element::BidirectionalReference(ref_path, _, max_hops, flags) => { + Element::Reference(ref_path, max_hops, flags) + } + x => x, + } + } } #[cfg(any(feature = "full", feature = "visualize"))] diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 70be0ea0..bdd6bc05 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -2,20 +2,19 @@ //! after usage automatically. use std::{ - borrow::Cow, cell::RefCell, collections::{btree_map::Entry, BTreeMap, HashSet}, mem::{self, MaybeUninit}, ops::Deref, }; -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt}; -use grovedb_merk::{Merk, MerkOptions}; +use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; +use grovedb_merk::{tree::NULL_HASH, CryptoHash, Merk, MerkOptions}; use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; use grovedb_version::version::GroveVersion; -use crate::{Element, Error, GroveDb, Transaction}; +use crate::{reference_path::ReferencePathType, Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; @@ -192,6 +191,87 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { propagation_result.map_ok(|_| result_batch) } + /// Inserts a reference into a cached Merk. + /// The reason why this has to be a whole `MerkCache` method is that + /// references involve opening and modifying several Merks at once, this + /// makes it out of scope for a single `MerkHandle`. + pub(crate) fn insert_reference<'c>( + &'c mut self, + path: SubtreePath<'b, B>, + key: &[u8], + ref_element: ReferenceElement, + cascade_on_update: bool, + options: Option, + ) -> CostResult<(), Error> { + let mut cost = Default::default(); + + let follow_reference_result = cost_return_on_error!( + &mut cost, + self.follow_reference(path.clone(), key, ref_element.get_ref_path()) + ); + + let value_hash = cost_return_on_error!( + &mut cost, + follow_reference_result + .last_element + .value_hash(&self.version) + ); + + // The insertion of a reference requires that its hash value be equal to the + // hash value of the item in the end of the reference's chain: + let [mut merk] = cost_return_on_error!(&mut cost, self.get_multi_mut([path])); + cost_return_on_error!( + &mut cost, + merk.insert_internal(key, ref_element.0.clone(), options, Some(value_hash)) + ); + + let version = self.version.clone(); // TODO + + // The closest referenced item's backward references list of the chain shall be + // updated with a new entry: + let [mut closest_merk] = cost_return_on_error!( + &mut cost, + self.get_multi_mut([follow_reference_result.first_path]) + ); + let mut closest_element = cost_return_on_error!( + &mut cost, + Element::get( + closest_merk.deref(), + &follow_reference_result.first_key, + true, + &version + ) + ); + // Updated backward references information: + closest_element = cost_return_on_error_no_add!( + cost, + closest_element.referenced_from(ref_element.to_ref_path(), cascade_on_update) + ); + // And write it back: + cost_return_on_error!( + &mut cost, + closest_merk.insert_internal( + &follow_reference_result.first_key, + closest_element, + None, + Some(value_hash), + ) + ); + + todo!() + } + + /// Follows a reference returning the first link in the references chain and + /// also the referenced item at the end of it. + fn follow_reference( + &mut self, + self_path: SubtreePath<'b, B>, + self_key: &[u8], + reference_path: &ReferencePathType, + ) -> CostResult, Error> { + todo!() + } + /// Perform propagation of references' chains marked as changed. fn propagate_updated_references(&mut self) -> CostResult<(), Error> { todo!() @@ -242,6 +322,62 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } } +struct FollowReferenceResult<'b, B> { + first_path: SubtreePath<'b, B>, + first_key: Vec, + last_element: Element, +} + +/// A wrapper type to ensure `Element` is a reference wherever it is required. +pub(crate) struct ReferenceElement(Element); + +impl ReferenceElement { + fn get_ref_path(&self) -> &ReferencePathType { + match &self.0 { + Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { + ref_path + } + _ => unreachable!(), + } + } + + fn to_ref_path(self) -> ReferencePathType { + match self.0 { + Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { + ref_path + } + _ => unreachable!(), + } + } +} + +impl TryFrom for ReferenceElement { + type Error = (); + + fn try_from(value: Element) -> Result { + match value { + element @ Element::Reference(..) | element @ Element::BidirectionalReference(..) => { + Ok(Self(element)) + } + _ => Err(()), + } + } +} + +/// A wrapper type to ensure `Element` is not a reference. +pub(crate) struct NonReferenceElement(Element); + +impl TryFrom for NonReferenceElement { + type Error = (); + + fn try_from(value: Element) -> Result { + match value { + Element::Reference(..) | Element::BidirectionalReference(..) => Err(()), + element => Ok(Self(element)), + } + } +} + /// Handle to a cached Merk. pub(crate) struct MerkHandle<'db, 'c, 'b, B> { merk: &'c mut TxMerk<'db>, @@ -276,10 +412,20 @@ impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { pub(crate) fn insert( + &mut self, + key: &[u8], + NonReferenceElement(element): NonReferenceElement, + options: Option, + ) -> CostResult<(), Error> { + self.insert_internal(key, element, options, None) + } + + fn insert_internal( &mut self, key: &[u8], element: Element, options: Option, + referenced_value_hash: Option, ) -> CostResult<(), Error> { let mut costs = Default::default(); @@ -287,13 +433,39 @@ impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { // references should be propagated after if cost_return_on_error!( &mut costs, - element.insert_if_changed_value(self.merk, key, options, self.version) - ) - .0 - { + match element { + Element::Item(..) + | Element::SumItem(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItemWithBackwardsReferences(..) => element + .insert_if_changed_value(self.merk, key, options, self.version) + .map_ok(|r| r.0), + Element::Reference(..) | Element::BidirectionalReference(..) => element + .insert_reference( + self.merk, + key, + referenced_value_hash.expect("todo"), + options, + self.version + ), + Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => + if value.is_some() { + Err(Error::InvalidCodeExecution( + "a tree should be empty at the moment of insertion when not using \ + batches", + )) + .wrap_with_cost(Default::default()) + } else { + element + .insert_subtree(self.merk, key, NULL_HASH, options, self.version) + .map_ok(|_| false) + }, + } + ) { self.updated_reference_handle .mark_updated_reference(key.to_vec()); } + *self.to_propagate = true; Ok(()).wrap_with_cost(costs) @@ -322,6 +494,7 @@ mod tests { use super::MerkCache; use crate::{ + reference_path::ReferencePathType, tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}, Element, }; @@ -397,7 +570,11 @@ mod tests { .unwrap() .unwrap(); subtree - .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .insert( + b"ayy", + Element::new_item(b"lmao".to_vec()).try_into().unwrap(), + None, + ) .unwrap() .unwrap(); @@ -419,7 +596,11 @@ mod tests { .unwrap() .unwrap(); subtree - .insert(b"ayy", Element::new_item(b"lmao".to_vec()), None) + .insert( + b"ayy", + Element::new_item(b"lmao".to_vec()).try_into().unwrap(), + None, + ) .unwrap() .unwrap(); @@ -435,4 +616,47 @@ mod tests { db.root_hash(Some(&tx), &version).unwrap().unwrap() ) } + + #[test] + fn changes_to_referenced_values_are_marked_uncommitted() { + let version = GroveVersion::latest(); + let db = make_deep_tree(&version); + let tx = db.start_transaction(); + + let mut cache = MerkCache::new(&db, &tx, version); + cache + .insert_reference( + SubtreePath::from(&[TEST_LEAF, b"innertree"]), + b"ayy", + Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + ANOTHER_TEST_LEAF.to_vec(), + b"innertree2".to_vec(), + b"k3".to_vec(), + ])) + .try_into() + .unwrap(), + false, + None, + ) + .unwrap() + .unwrap(); + + assert!(cache.updated_references.borrow().is_empty()); + + let [mut subtree] = cache + .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"])]) + .unwrap() + .unwrap(); + + subtree + .insert( + b"k3", + Element::new_item(b"huh".to_vec()).try_into().unwrap(), + None, + ) + .unwrap() + .unwrap(); + + assert!(!cache.updated_references.borrow().is_empty()); + } } From 131e9a5ba0b9cec4c675f5422e63c1fe7a11a6dc Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 26 Nov 2024 23:07:09 +0100 Subject: [PATCH 20/27] wip --- grovedb/src/element/insert.rs | 15 +- grovedb/src/merk_cache.rs | 669 ++++++++++++++++++++++------------ 2 files changed, 434 insertions(+), 250 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 1d418d56..17a44d4d 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -248,7 +248,6 @@ impl Element { /// be loaded by this moment If transaction is not passed, the batch /// will be written immediately. If transaction is passed, the operation /// will be committed on the transaction commit. - /// The bool represents whether a propagation of references is needed. /// If the value changed, it returns the old element under `Some`. pub fn insert_if_changed_value<'db, S: StorageContext<'db>>( self, @@ -256,7 +255,7 @@ impl Element { key: &[u8], options: Option, grove_version: &GroveVersion, - ) -> CostResult<(bool, Option), Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "insert_if_changed_value", grove_version @@ -282,14 +281,13 @@ impl Element { .unwrap_or(true); if changed { - let has_references = to_insert.has_backward_references(); cost_return_on_error!( &mut cost, to_insert.insert(merk, key, options, grove_version) ); - Ok((has_references, previous_element)).wrap_with_cost(cost) + Ok(previous_element).wrap_with_cost(cost) } else { - Ok((false, None)).wrap_with_cost(cost) + Ok(None).wrap_with_cost(cost) } } @@ -636,14 +634,13 @@ mod tests { merk.commit(grove_version); - let (inserted, previous) = Element::new_item(b"value".to_vec()) + let previous = Element::new_item(b"value".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); merk.commit(grove_version); - assert!(!inserted); assert_eq!(previous, None); assert_eq!( Element::get(&merk, b"another-key", true, grove_version) @@ -678,7 +675,7 @@ mod tests { let batch = StorageBatch::new(); let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); - let (_, previous) = Element::new_item(b"value2".to_vec()) + let previous = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); @@ -707,7 +704,7 @@ mod tests { .insert(&mut merk, b"mykey", None, grove_version) .unwrap() .expect("expected successful insertion"); - let (_, previous) = Element::new_item(b"value2".to_vec()) + let previous = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index bdd6bc05..27aca594 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -14,7 +14,10 @@ use grovedb_path::SubtreePath; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; use grovedb_version::version::GroveVersion; -use crate::{reference_path::ReferencePathType, Element, Error, GroveDb, Transaction}; +use crate::{ + element::CascadeOnUpdate, reference_path::ReferencePathType, Element, Error, GroveDb, + Transaction, +}; type TxMerk<'db> = Merk>; @@ -25,7 +28,7 @@ struct CachedMerk<'db> { merk: TxMerk<'db>, } -type UpdatedReferences<'b, B> = RefCell, Vec)>>; +// type UpdatedReferences<'b, B> = RefCell, Vec)>>; /// Helper struct to split `MerkCache` into independent parts to allow splitting /// borrows, meaning a dependency on the storage part that shall have a certain @@ -96,9 +99,6 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCacheStorage<'db, 'b, B> { // SAFETY: please consult with other safety docs here before doing any changes pub(crate) struct MerkCache<'db, 'b, B> { storage: MerkCacheStorage<'db, 'b, B>, - /// References require different kind of propagation and we track pointed to - /// values to update references. - updated_references: UpdatedReferences<'b, B>, version: &'db GroveVersion, } @@ -111,7 +111,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { MerkCache { storage: MerkCacheStorage::new(db, tx), version, - updated_references: Default::default(), + // updated_references: Default::default(), } } @@ -125,8 +125,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { pub(crate) fn get_multi_mut<'c, const N: usize>( &'c mut self, paths: [SubtreePath<'b, B>; N], - ) -> CostResult<[MerkHandle<'db, 'c, 'b, B>; N], Error> { - let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + ) -> CostResult<[MerkHandle<'db, 'c>; N], Error> { + let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; let mut cost = Default::default(); let unique_args: HashSet<_> = paths.iter().collect(); @@ -149,10 +149,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { ) as *mut CachedMerk<'db>) .as_mut::<'c>() .expect("not a null pointer"), - UpdatedReferenceHandle { - path, - updated_references: &self.updated_references, - }, + // UpdatedReferenceHandle { + // path, + // updated_references: &self.updated_references, + // }, &self.version, ) }; @@ -164,9 +164,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // N in our case. `mem::transmute` would represent it better, however, // due to poor support of const generics in stable Rust we bypass // compile-time size checks with pointer casts. - let result = unsafe { - (&result_uninit as *const _ as *const [MerkHandle<'db, 'c, 'b, B>; N]).read() - }; + let result = + unsafe { (&result_uninit as *const _ as *const [MerkHandle<'db, 'c>; N]).read() }; mem::forget(result_uninit); Ok(result).wrap_with_cost(cost) @@ -179,7 +178,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // Propagate updated subtrees' hashes up to the root and dropping all possible // batch users: - let propagation_result = self.propagate_updated_merks(); + let propagation_result = { + self.propagate_updated_references() + .flat_map_ok(|_| self.propagate_updated_merks()) + }; // SAFETY: The batch reference was created by `Box::leak` and restored into a // `Box` form. No shared usage of that memory exists because the @@ -191,86 +193,86 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { propagation_result.map_ok(|_| result_batch) } - /// Inserts a reference into a cached Merk. - /// The reason why this has to be a whole `MerkCache` method is that - /// references involve opening and modifying several Merks at once, this - /// makes it out of scope for a single `MerkHandle`. - pub(crate) fn insert_reference<'c>( - &'c mut self, - path: SubtreePath<'b, B>, - key: &[u8], - ref_element: ReferenceElement, - cascade_on_update: bool, - options: Option, - ) -> CostResult<(), Error> { - let mut cost = Default::default(); - - let follow_reference_result = cost_return_on_error!( - &mut cost, - self.follow_reference(path.clone(), key, ref_element.get_ref_path()) - ); - - let value_hash = cost_return_on_error!( - &mut cost, - follow_reference_result - .last_element - .value_hash(&self.version) - ); - - // The insertion of a reference requires that its hash value be equal to the - // hash value of the item in the end of the reference's chain: - let [mut merk] = cost_return_on_error!(&mut cost, self.get_multi_mut([path])); - cost_return_on_error!( - &mut cost, - merk.insert_internal(key, ref_element.0.clone(), options, Some(value_hash)) - ); - - let version = self.version.clone(); // TODO - - // The closest referenced item's backward references list of the chain shall be - // updated with a new entry: - let [mut closest_merk] = cost_return_on_error!( - &mut cost, - self.get_multi_mut([follow_reference_result.first_path]) - ); - let mut closest_element = cost_return_on_error!( - &mut cost, - Element::get( - closest_merk.deref(), - &follow_reference_result.first_key, - true, - &version - ) - ); - // Updated backward references information: - closest_element = cost_return_on_error_no_add!( - cost, - closest_element.referenced_from(ref_element.to_ref_path(), cascade_on_update) - ); - // And write it back: - cost_return_on_error!( - &mut cost, - closest_merk.insert_internal( - &follow_reference_result.first_key, - closest_element, - None, - Some(value_hash), - ) - ); - - todo!() - } - - /// Follows a reference returning the first link in the references chain and - /// also the referenced item at the end of it. - fn follow_reference( - &mut self, - self_path: SubtreePath<'b, B>, - self_key: &[u8], - reference_path: &ReferencePathType, - ) -> CostResult, Error> { - todo!() - } + // /// Inserts a reference into a cached Merk. + // /// The reason why this has to be a whole `MerkCache` method is that + // /// references involve opening and modifying several Merks at once, this + // /// makes it out of scope for a single `MerkHandle`. + // pub(crate) fn insert_reference<'c>( + // &'c mut self, + // path: SubtreePath<'b, B>, + // key: &[u8], + // ref_element: ReferenceElement, + // cascade_on_update: bool, + // options: Option, + // ) -> CostResult<(), Error> { + // let mut cost = Default::default(); + + // let follow_reference_result = cost_return_on_error!( + // &mut cost, + // self.follow_reference(path.clone(), key, ref_element.get_ref_path()) + // ); + + // let value_hash = cost_return_on_error!( + // &mut cost, + // follow_reference_result + // .last_element + // .value_hash(&self.version) + // ); + + // // The insertion of a reference requires that its hash value be equal to the + // // hash value of the item in the end of the reference's chain: + // let [mut merk] = cost_return_on_error!(&mut cost, self.get_multi_mut([path])); + // cost_return_on_error!( + // &mut cost, + // merk.insert_internal(key, ref_element.0.clone(), options, Some(value_hash)) + // ); + + // let version = self.version.clone(); // TODO + + // // The closest referenced item's backward references list of the chain shall be + // // updated with a new entry: + // let [mut closest_merk] = cost_return_on_error!( + // &mut cost, + // self.get_multi_mut([follow_reference_result.first_path]) + // ); + // let mut closest_element = cost_return_on_error!( + // &mut cost, + // Element::get( + // closest_merk.deref(), + // &follow_reference_result.first_key, + // true, + // &version + // ) + // ); + // // Updated backward references information: + // closest_element = cost_return_on_error_no_add!( + // cost, + // closest_element.referenced_from(ref_element.to_ref_path(), cascade_on_update) + // ); + // // And write it back: + // cost_return_on_error!( + // &mut cost, + // closest_merk.insert_internal( + // &follow_reference_result.first_key, + // closest_element, + // None, + // Some(value_hash), + // ) + // ); + + // todo!() + // } + + // /// Follows a reference returning the first link in the references chain and + // /// also the referenced item at the end of it. + // fn follow_reference( + // &mut self, + // self_path: SubtreePath<'b, B>, + // self_key: &[u8], + // reference_path: &ReferencePathType, + // ) -> CostResult, Error> { + // todo!() + // } /// Perform propagation of references' chains marked as changed. fn propagate_updated_references(&mut self) -> CostResult<(), Error> { @@ -328,81 +330,82 @@ struct FollowReferenceResult<'b, B> { last_element: Element, } -/// A wrapper type to ensure `Element` is a reference wherever it is required. -pub(crate) struct ReferenceElement(Element); - -impl ReferenceElement { - fn get_ref_path(&self) -> &ReferencePathType { - match &self.0 { - Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { - ref_path - } - _ => unreachable!(), - } - } - - fn to_ref_path(self) -> ReferencePathType { - match self.0 { - Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { - ref_path - } - _ => unreachable!(), - } - } -} - -impl TryFrom for ReferenceElement { - type Error = (); - - fn try_from(value: Element) -> Result { - match value { - element @ Element::Reference(..) | element @ Element::BidirectionalReference(..) => { - Ok(Self(element)) - } - _ => Err(()), - } - } -} - -/// A wrapper type to ensure `Element` is not a reference. -pub(crate) struct NonReferenceElement(Element); - -impl TryFrom for NonReferenceElement { - type Error = (); - - fn try_from(value: Element) -> Result { - match value { - Element::Reference(..) | Element::BidirectionalReference(..) => Err(()), - element => Ok(Self(element)), - } - } -} +// /// A wrapper type to ensure `Element` is a reference wherever it is required. +// pub(crate) struct ReferenceElement(Element); + +// impl ReferenceElement { +// fn get_ref_path(&self) -> &ReferencePathType { +// match &self.0 { +// Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { +// ref_path +// } +// _ => unreachable!(), +// } +// } + +// fn to_ref_path(self) -> ReferencePathType { +// match self.0 { +// Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { +// ref_path +// } +// _ => unreachable!(), +// } +// } +// } + +// impl TryFrom for ReferenceElement { +// type Error = (); + +// fn try_from(value: Element) -> Result { +// match value { +// element @ Element::Reference(..) | element @ Element::BidirectionalReference(..) => { +// Ok(Self(element)) +// } +// _ => Err(()), +// } +// } +// } + +// /// A wrapper type to ensure `Element` is not a reference. +// pub(crate) struct NonReferenceElement(Element); + +// impl TryFrom for NonReferenceElement { +// type Error = (); + +// fn try_from(value: Element) -> Result { +// match value { +// Element::Reference(..) | Element::BidirectionalReference(..) => Err(()), +// element => Ok(Self(element)), +// } +// } +// } /// Handle to a cached Merk. -pub(crate) struct MerkHandle<'db, 'c, 'b, B> { +pub(crate) struct MerkHandle<'db, 'c> { + /// Reference to an opened Merk tree with transactional batch storage context merk: &'c mut TxMerk<'db>, version: &'db GroveVersion, + /// Mark this subtree as a subject to propagate to_propagate: &'c mut bool, - updated_reference_handle: UpdatedReferenceHandle<'c, 'b, B>, } -/// Helper struct to signal `MerkCache` about updated references. -struct UpdatedReferenceHandle<'c, 'b, B> { - path: SubtreePath<'b, B>, - updated_references: &'c UpdatedReferences<'b, B>, -} +// /// Helper struct to signal `MerkCache` about updated references. +// struct UpdatedReferenceHandle<'c, 'b, B> { +// path: SubtreePath<'b, B>, +// updated_references: &'c UpdatedReferences<'b, B>, +// } -impl<'c, 'b, B: AsRef<[u8]>> UpdatedReferenceHandle<'c, 'b, B> { - fn mark_updated_reference(&self, key: Key) { - self.updated_references - .borrow_mut() - .insert((self.path.clone(), key)); - } -} +// impl<'c, 'b, B: AsRef<[u8]>> UpdatedReferenceHandle<'c, 'b, B> { +// fn mark_updated_reference(&self, key: Key) { +// self.updated_references +// .borrow_mut() +// .insert((self.path.clone(), key)); +// } +// } /// It is allowed to dereference `MerkHandle` to regular Merks but in a /// non-mutable way since we want to track what have been done to those Merks. -impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { +impl<'db, 'c> Deref for MerkHandle<'db, 'c> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { @@ -410,77 +413,261 @@ impl<'db, 'c, 'b, B> Deref for MerkHandle<'db, 'c, 'b, B> { } } -impl<'db, 'c, 'b, B: AsRef<[u8]>> MerkHandle<'db, 'c, 'b, B> { - pub(crate) fn insert( - &mut self, - key: &[u8], - NonReferenceElement(element): NonReferenceElement, - options: Option, - ) -> CostResult<(), Error> { - self.insert_internal(key, element, options, None) +/// This type represents changes made to a particular element, a difference between its initial +/// state and eventual state +enum Delta { + AddItem, + RemoveItem { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + }, + UpdateItemToReference { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + new_forward_reference: ReferencePathType, + }, + UpdateItemToItem { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + }, + AddReference { + new_forward_reference: ReferencePathType, + }, + RemoveReference { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + ex_forward_reference: ReferencePathType, + }, + UpdateReferenceToItem { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + ex_forward_reference: ReferencePathType, + }, + UpdateReferenceToReference { + backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, + new_forward_reference: ReferencePathType, + ex_forward_reference: ReferencePathType, + }, +} + +impl Delta { + fn new(old_element: Option, new_element: Element) -> Option { + match (old_element, new_element) { + ( + None | Some(Element::Tree(..) | Element::SumTree(..)), + Element::Item(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..), + ) => Some(Self::AddItem), + ( + None | Some(Element::Tree(..) | Element::SumTree(..)), + Element::Reference(new_forward_reference, ..) + | Element::BidirectionalReference(new_forward_reference, ..), + ) => Some(Self::AddReference { + new_forward_reference, + }), + (_, Element::Tree(..) | Element::SumTree(..)) => None, + + ( + Some( + Element::ItemWithBackwardsReferences(_, backward_references, _) + | Element::SumItemWithBackwardsReferences(_, backward_references, _), + ), + Element::Item(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..), + ) => Some(Self::UpdateItemToItem { + backward_references, + }), + ( + Some(Element::Item(..) | Element::SumItem(..)), + Element::Item(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..), + ) => Some(Self::UpdateItemToItem { + backward_references: Vec::new(), + }), + ( + Some( + Element::ItemWithBackwardsReferences(_, backward_references, _) + | Element::SumItemWithBackwardsReferences(_, backward_references, _), + ), + Element::Reference(new_forward_reference, ..) + | Element::BidirectionalReference(new_forward_reference, ..), + ) => Some(Self::UpdateItemToReference { + backward_references, + new_forward_reference, + }), + ( + Some(Element::Item(..) | Element::SumItem(..)), + Element::Reference(new_forward_reference, ..) + | Element::BidirectionalReference(new_forward_reference, ..), + ) => Some(Self::UpdateItemToReference { + backward_references: Vec::new(), + new_forward_reference, + }), + ( + Some(Element::Reference(ex_forward_reference, ..)), + Element::Item(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..), + ) => Some(Self::UpdateReferenceToItem { + backward_references: Vec::new(), + ex_forward_reference, + }), + ( + Some(Element::BidirectionalReference( + ex_forward_reference, + backward_references, + .., + )), + Element::Item(..) + | Element::ItemWithBackwardsReferences(..) + | Element::SumItem(..) + | Element::SumItemWithBackwardsReferences(..), + ) => Some(Self::UpdateReferenceToItem { + backward_references, + ex_forward_reference, + }), + ( + Some(Element::Reference(ex_forward_reference, ..)), + Element::Reference(new_forward_reference, ..) + | Element::BidirectionalReference(new_forward_reference, ..), + ) => Some(Self::UpdateReferenceToReference { + backward_references: Vec::new(), + new_forward_reference, + ex_forward_reference, + }), + ( + Some(Element::BidirectionalReference( + ex_forward_reference, + backward_references, + .., + )), + Element::Reference(new_forward_reference, ..) + | Element::BidirectionalReference(new_forward_reference, ..), + ) => Some(Self::UpdateReferenceToReference { + backward_references, + new_forward_reference, + ex_forward_reference, + }), + } } - fn insert_internal( + fn new_from_delete(element: Element) -> Option { + match element { + Element::Item(..) | Element::SumItem(..) => Some(Self::RemoveItem { + backward_references: Vec::new(), + }), + Element::ItemWithBackwardsReferences(_, backward_references, ..) + | Element::SumItemWithBackwardsReferences(_, backward_references, ..) => { + Some(Self::RemoveItem { + backward_references, + }) + } + Element::Reference(ex_forward_reference, ..) => Some(Self::RemoveReference { + backward_references: Vec::new(), + ex_forward_reference, + }), + Element::BidirectionalReference(ex_forward_reference, backward_references, ..) => { + Some(Self::RemoveReference { + backward_references, + ex_forward_reference, + }) + } + Element::Tree(..) | Element::SumTree(..) => None, + } + } +} + +impl<'db, 'c> MerkHandle<'db, 'c> { + pub(crate) fn insert( &mut self, key: &[u8], element: Element, options: Option, - referenced_value_hash: Option, ) -> CostResult<(), Error> { let mut costs = Default::default(); - // In case the item that was changed has been referenced, we indicate that - // references should be propagated after - if cost_return_on_error!( + let old_value = cost_return_on_error!( &mut costs, match element { - Element::Item(..) - | Element::SumItem(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItemWithBackwardsReferences(..) => element - .insert_if_changed_value(self.merk, key, options, self.version) - .map_ok(|r| r.0), - Element::Reference(..) | Element::BidirectionalReference(..) => element - .insert_reference( - self.merk, - key, - referenced_value_hash.expect("todo"), - options, - self.version - ), Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => if value.is_some() { Err(Error::InvalidCodeExecution( "a tree should be empty at the moment of insertion when not using \ - batches", + batches", )) .wrap_with_cost(Default::default()) } else { element .insert_subtree(self.merk, key, NULL_HASH, options, self.version) - .map_ok(|_| false) + .map_ok(|_| None) }, + _ => element.insert_if_changed_value(self.merk, key, options, self.version), } - ) { - self.updated_reference_handle - .mark_updated_reference(key.to_vec()); - } + ); *self.to_propagate = true; Ok(()).wrap_with_cost(costs) } - fn new( - cached_merk: &'c mut CachedMerk<'db>, - updated_reference_handle: UpdatedReferenceHandle<'c, 'b, B>, - version: &'db GroveVersion, - ) -> Self { + // fn insert_internal( + // &mut self, + // key: &[u8], + // element: Element, + // options: Option, + // referenced_value_hash: Option, + // ) -> CostResult<(), Error> { + // let mut costs = Default::default(); + + // // In case the item that was changed has been referenced, we indicate that + // // references should be propagated after + // if cost_return_on_error!( + // &mut costs, + // match element { + // Element::Item(..) + // | Element::SumItem(..) + // | Element::ItemWithBackwardsReferences(..) + // | Element::SumItemWithBackwardsReferences(..) => element + // .insert_if_changed_value(self.merk, key, options, self.version) + // .map_ok(|r| r.0), + // Element::Reference(..) | Element::BidirectionalReference(..) => element + // .insert_reference( + // self.merk, + // key, + // referenced_value_hash.expect("todo"), + // options, + // self.version + // ), + // Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => + // if value.is_some() { + // Err(Error::InvalidCodeExecution( + // "a tree should be empty at the moment of insertion when not using \ + // batches", + // )) + // .wrap_with_cost(Default::default()) + // } else { + // element + // .insert_subtree(self.merk, key, NULL_HASH, options, self.version) + // .map_ok(|_| false) + // }, + // } + // ) { + // self.updated_reference_handle + // .mark_updated_reference(key.to_vec()); + // } + + // *self.to_propagate = true; + + // Ok(()).wrap_with_cost(costs) + // } + + fn new(cached_merk: &'c mut CachedMerk<'db>, version: &'db GroveVersion) -> Self { Self { merk: &mut cached_merk.merk, version, to_propagate: &mut cached_merk.to_propagate, - updated_reference_handle, } } } @@ -617,46 +804,46 @@ mod tests { ) } - #[test] - fn changes_to_referenced_values_are_marked_uncommitted() { - let version = GroveVersion::latest(); - let db = make_deep_tree(&version); - let tx = db.start_transaction(); - - let mut cache = MerkCache::new(&db, &tx, version); - cache - .insert_reference( - SubtreePath::from(&[TEST_LEAF, b"innertree"]), - b"ayy", - Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ - ANOTHER_TEST_LEAF.to_vec(), - b"innertree2".to_vec(), - b"k3".to_vec(), - ])) - .try_into() - .unwrap(), - false, - None, - ) - .unwrap() - .unwrap(); - - assert!(cache.updated_references.borrow().is_empty()); - - let [mut subtree] = cache - .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"])]) - .unwrap() - .unwrap(); - - subtree - .insert( - b"k3", - Element::new_item(b"huh".to_vec()).try_into().unwrap(), - None, - ) - .unwrap() - .unwrap(); - - assert!(!cache.updated_references.borrow().is_empty()); - } + // #[test] + // fn changes_to_referenced_values_are_marked_uncommitted() { + // let version = GroveVersion::latest(); + // let db = make_deep_tree(&version); + // let tx = db.start_transaction(); + + // let mut cache = MerkCache::new(&db, &tx, version); + // cache + // .insert_reference( + // SubtreePath::from(&[TEST_LEAF, b"innertree"]), + // b"ayy", + // Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + // ANOTHER_TEST_LEAF.to_vec(), + // b"innertree2".to_vec(), + // b"k3".to_vec(), + // ])) + // .try_into() + // .unwrap(), + // false, + // None, + // ) + // .unwrap() + // .unwrap(); + + // assert!(cache.updated_references.borrow().is_empty()); + + // let [mut subtree] = cache + // .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"])]) + // .unwrap() + // .unwrap(); + + // subtree + // .insert( + // b"k3", + // Element::new_item(b"huh".to_vec()).try_into().unwrap(), + // None, + // ) + // .unwrap() + // .unwrap(); + + // assert!(!cache.updated_references.borrow().is_empty()); + // } } From 8da0d5a409f68ce605169614503d54cb6d4cdbcf Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 28 Nov 2024 19:18:14 +0100 Subject: [PATCH 21/27] upd --- grovedb/src/merk_cache.rs | 79 +++++---- grovedb/src/reference_path.rs | 321 +++++++++++++++++++++++++++++++++- 2 files changed, 361 insertions(+), 39 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 27aca594..a01a46c3 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -28,7 +28,8 @@ struct CachedMerk<'db> { merk: TxMerk<'db>, } -// type UpdatedReferences<'b, B> = RefCell, Vec)>>; +// type UpdatedReferences<'b, B> = RefCell, +// Vec)>>; /// Helper struct to split `MerkCache` into independent parts to allow splitting /// borrows, meaning a dependency on the storage part that shall have a certain @@ -219,18 +220,18 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // .value_hash(&self.version) // ); - // // The insertion of a reference requires that its hash value be equal to the - // // hash value of the item in the end of the reference's chain: - // let [mut merk] = cost_return_on_error!(&mut cost, self.get_multi_mut([path])); - // cost_return_on_error!( + // // The insertion of a reference requires that its hash value be equal to + // the // hash value of the item in the end of the reference's chain: + // let [mut merk] = cost_return_on_error!(&mut cost, + // self.get_multi_mut([path])); cost_return_on_error!( // &mut cost, - // merk.insert_internal(key, ref_element.0.clone(), options, Some(value_hash)) - // ); + // merk.insert_internal(key, ref_element.0.clone(), options, + // Some(value_hash)) ); // let version = self.version.clone(); // TODO - // // The closest referenced item's backward references list of the chain shall be - // // updated with a new entry: + // // The closest referenced item's backward references list of the chain + // shall be // updated with a new entry: // let [mut closest_merk] = cost_return_on_error!( // &mut cost, // self.get_multi_mut([follow_reference_result.first_path]) @@ -247,8 +248,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // // Updated backward references information: // closest_element = cost_return_on_error_no_add!( // cost, - // closest_element.referenced_from(ref_element.to_ref_path(), cascade_on_update) - // ); + // closest_element.referenced_from(ref_element.to_ref_path(), + // cascade_on_update) ); // // And write it back: // cost_return_on_error!( // &mut cost, @@ -330,14 +331,14 @@ struct FollowReferenceResult<'b, B> { last_element: Element, } -// /// A wrapper type to ensure `Element` is a reference wherever it is required. -// pub(crate) struct ReferenceElement(Element); +// /// A wrapper type to ensure `Element` is a reference wherever it is +// required. pub(crate) struct ReferenceElement(Element); // impl ReferenceElement { // fn get_ref_path(&self) -> &ReferencePathType { // match &self.0 { -// Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { -// ref_path +// Element::Reference(ref_path, ..) | +// Element::BidirectionalReference(ref_path, ..) => { ref_path // } // _ => unreachable!(), // } @@ -345,8 +346,8 @@ struct FollowReferenceResult<'b, B> { // fn to_ref_path(self) -> ReferencePathType { // match self.0 { -// Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { -// ref_path +// Element::Reference(ref_path, ..) | +// Element::BidirectionalReference(ref_path, ..) => { ref_path // } // _ => unreachable!(), // } @@ -358,8 +359,8 @@ struct FollowReferenceResult<'b, B> { // fn try_from(value: Element) -> Result { // match value { -// element @ Element::Reference(..) | element @ Element::BidirectionalReference(..) => { -// Ok(Self(element)) +// element @ Element::Reference(..) | element @ +// Element::BidirectionalReference(..) => { Ok(Self(element)) // } // _ => Err(()), // } @@ -374,15 +375,16 @@ struct FollowReferenceResult<'b, B> { // fn try_from(value: Element) -> Result { // match value { -// Element::Reference(..) | Element::BidirectionalReference(..) => Err(()), -// element => Ok(Self(element)), +// Element::Reference(..) | Element::BidirectionalReference(..) => +// Err(()), element => Ok(Self(element)), // } // } // } /// Handle to a cached Merk. pub(crate) struct MerkHandle<'db, 'c> { - /// Reference to an opened Merk tree with transactional batch storage context + /// Reference to an opened Merk tree with transactional batch storage + /// context merk: &'c mut TxMerk<'db>, version: &'db GroveVersion, /// Mark this subtree as a subject to propagate @@ -413,8 +415,8 @@ impl<'db, 'c> Deref for MerkHandle<'db, 'c> { } } -/// This type represents changes made to a particular element, a difference between its initial -/// state and eventual state +/// This type represents changes made to a particular element, a difference +/// between its initial state and eventual state enum Delta { AddItem, RemoveItem { @@ -595,7 +597,7 @@ impl<'db, 'c> MerkHandle<'db, 'c> { if value.is_some() { Err(Error::InvalidCodeExecution( "a tree should be empty at the moment of insertion when not using \ - batches", + batches", )) .wrap_with_cost(Default::default()) } else { @@ -621,8 +623,8 @@ impl<'db, 'c> MerkHandle<'db, 'c> { // ) -> CostResult<(), Error> { // let mut costs = Default::default(); - // // In case the item that was changed has been referenced, we indicate that - // // references should be propagated after + // // In case the item that was changed has been referenced, we indicate + // that // references should be propagated after // if cost_return_on_error!( // &mut costs, // match element { @@ -630,10 +632,10 @@ impl<'db, 'c> MerkHandle<'db, 'c> { // | Element::SumItem(..) // | Element::ItemWithBackwardsReferences(..) // | Element::SumItemWithBackwardsReferences(..) => element - // .insert_if_changed_value(self.merk, key, options, self.version) - // .map_ok(|r| r.0), - // Element::Reference(..) | Element::BidirectionalReference(..) => element - // .insert_reference( + // .insert_if_changed_value(self.merk, key, options, + // self.version) .map_ok(|r| r.0), + // Element::Reference(..) | Element::BidirectionalReference(..) => + // element .insert_reference( // self.merk, // key, // referenced_value_hash.expect("todo"), @@ -643,14 +645,14 @@ impl<'db, 'c> MerkHandle<'db, 'c> { // Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => // if value.is_some() { // Err(Error::InvalidCodeExecution( - // "a tree should be empty at the moment of insertion when not using \ - // batches", + // "a tree should be empty at the moment of insertion + // when not using \ batches", // )) // .wrap_with_cost(Default::default()) // } else { // element - // .insert_subtree(self.merk, key, NULL_HASH, options, self.version) - // .map_ok(|_| false) + // .insert_subtree(self.merk, key, NULL_HASH, options, + // self.version) .map_ok(|_| false) // }, // } // ) { @@ -815,7 +817,8 @@ mod tests { // .insert_reference( // SubtreePath::from(&[TEST_LEAF, b"innertree"]), // b"ayy", - // Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ + // + // Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ // ANOTHER_TEST_LEAF.to_vec(), // b"innertree2".to_vec(), // b"k3".to_vec(), @@ -831,8 +834,8 @@ mod tests { // assert!(cache.updated_references.borrow().is_empty()); // let [mut subtree] = cache - // .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, b"innertree2"])]) - // .unwrap() + // .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, + // b"innertree2"])]) .unwrap() // .unwrap(); // subtree diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index fcfeee6e..4a489aa9 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -2,8 +2,10 @@ #[cfg(any(feature = "full", feature = "verify"))] use std::fmt; +use std::iter; use bincode::{Decode, Encode}; +use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; #[cfg(feature = "full")] @@ -59,6 +61,76 @@ pub enum ReferencePathType { SiblingReference(Vec), } +impl ReferencePathType { + /// Get an inverted reference + fn invert>(&self, path: SubtreePath, key: &[u8]) -> Option { + Some(match self { + // Absolute path shall point to a fully qualified path of the reference's origin + ReferencePathType::AbsolutePathReference(_) => { + let mut qualified_path = path.to_vec(); + qualified_path.push(key.to_vec()); + ReferencePathType::AbsolutePathReference(qualified_path) + } + // Since both reference origin and path share N first segments, the backward reference + // can do the same, key we shall persist for a qualified path as the output + ReferencePathType::UpstreamRootHeightReference(n, _) => { + let relative_path: Vec<_> = path + .to_vec() + .into_iter() + .skip(*n as usize) + .chain(iter::once(key.to_vec())) + .collect(); + ReferencePathType::UpstreamRootHeightReference(*n, relative_path) + } + // Since it uses some parent information it get's complicated, so falling back to the + // preivous type of reference + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference(n, _) => { + let relative_path: Vec<_> = path + .to_vec() + .into_iter() + .skip(*n as usize) + .chain(iter::once(key.to_vec())) + .collect(); + ReferencePathType::UpstreamRootHeightReference(*n, relative_path) + } + // Discarding N latest segments is relative to the previously appended path, so it would + // be easier to discard appended paths both ways and have a shared prefix. + ReferencePathType::UpstreamFromElementHeightReference(n, append_path) => { + let mut relative_path: Vec> = path + .into_reverse_iter() + .take(*n as usize) + .map(|x| x.to_vec()) + .collect(); + relative_path.reverse(); + relative_path.push(key.to_vec()); + ReferencePathType::UpstreamFromElementHeightReference( + append_path.len() as u8 - 1, + relative_path, + ) + } + // Cousin is relative to cousin, key will remain the same + ReferencePathType::CousinReference(_) => ReferencePathType::CousinReference( + path.into_reverse_iter().next().map(|x| x.to_vec())?, + ), + // Here since any number of segments could've been added we need to resort to a more + // specific option + ReferencePathType::RemovedCousinReference(append_path) => { + let mut relative_path = + vec![path.into_reverse_iter().next().map(|x| x.to_vec())?]; + relative_path.push(key.to_vec()); + ReferencePathType::UpstreamFromElementHeightReference( + append_path.len() as u8, + relative_path, + ) + } + // The closest way back would be just to use the key + ReferencePathType::SiblingReference(_) => { + ReferencePathType::SiblingReference(key.to_vec()) + } + }) + } +} + // Helper function to display paths fn display_path(path: &[Vec]) -> String { path.iter() @@ -170,7 +242,6 @@ pub fn path_from_reference_path_type>( current_key: Option<&[u8]>, ) -> Result>, Error> { match reference_path_type { - // No computation required, we already know the absolute path ReferencePathType::AbsolutePathReference(path) => Ok(path), // Take the first n elements from current path, append new path to subpath @@ -324,6 +395,7 @@ impl ReferencePathType { #[cfg(test)] mod tests { use grovedb_merk::proofs::Query; + use grovedb_path::SubtreePath; use grovedb_version::version::GroveVersion; use crate::{ @@ -515,4 +587,251 @@ mod tests { assert_eq!(hash, db.root_hash(None, grove_version).unwrap().unwrap()); assert_eq!(result.len(), 5); } + + #[test] + fn inverted_absolute_path() { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + + let reference = + ReferencePathType::AbsolutePathReference(vec![b"m".to_vec(), b"n".to_vec()]); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + + assert_eq!( + reference, + inverse + .invert(pointed_to_path.into(), pointed_to_key) + .unwrap() + ); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path + ); + } + + #[test] + fn inverted_upstream_root_height() { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + + let reference = + ReferencePathType::UpstreamRootHeightReference(2, vec![b"m".to_vec(), b"n".to_vec()]); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), None) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + + assert_eq!( + reference, + inverse + .invert(pointed_to_path.into(), pointed_to_key) + .unwrap() + ); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path.to_vec(), + ); + } + + #[test] + fn inverted_upstream_root_height_with_parent_path_addition() { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + let reference = ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + 2, + vec![b"m".to_vec(), b"n".to_vec()], + ); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path.to_vec(), + ); + } + + #[test] + fn inverted_upstream_from_element_height() { + { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + let reference = ReferencePathType::UpstreamFromElementHeightReference( + 1, + vec![b"m".to_vec(), b"n".to_vec()], + ); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + + assert_eq!( + reference, + inverse + .invert(pointed_to_path.into(), pointed_to_key) + .unwrap() + ); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path.to_vec(), + ); + } + + { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + let reference = ReferencePathType::UpstreamFromElementHeightReference( + 3, + vec![b"m".to_vec(), b"n".to_vec()], + ); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + + assert_eq!( + reference, + inverse + .invert(pointed_to_path.into(), pointed_to_key) + .unwrap() + ); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path.to_vec(), + ); + } + } + + #[test] + fn inverted_cousin_reference() { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + let reference = + ReferencePathType::RemovedCousinReference(vec![b"m".to_vec(), b"n".to_vec()]); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path + ); + } + + #[test] + fn inverted_sibling_reference() { + let current_path: SubtreePath<_> = (&[b"a", b"b", b"c", b"d"]).into(); + let current_key = b"e"; + let current_qualified_path = { + let mut p = current_path.to_vec(); + p.push(current_key.to_vec()); + p + }; + let reference = ReferencePathType::SiblingReference(b"yeet".to_vec()); + + let pointed_to_qualified_path = reference + .clone() + .absolute_path(¤t_path.to_vec(), Some(current_key)) + .unwrap(); + let (pointed_to_key, pointed_to_path) = pointed_to_qualified_path.split_last().unwrap(); + + let inverse = reference.invert(current_path.clone(), current_key).unwrap(); + + assert_ne!(reference, inverse); + assert_eq!( + reference, + inverse + .invert(pointed_to_path.into(), pointed_to_key) + .unwrap() + ); + assert_eq!( + inverse + .absolute_path(&pointed_to_path, Some(pointed_to_key)) + .unwrap(), + current_qualified_path + ); + } } From fda42a1a61210fcd7f44692ba005c1e520dc24a4 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Thu, 5 Dec 2024 16:15:52 +0100 Subject: [PATCH 22/27] revert obsolete changes and simplify merk cache --- grovedb/src/element/insert.rs | 193 ++---- grovedb/src/element/mod.rs | 27 +- grovedb/src/lib.rs | 53 -- grovedb/src/merk_cache.rs | 916 +++++---------------------- grovedb/src/operations/delete/mod.rs | 6 +- grovedb/src/operations/insert/mod.rs | 60 +- 6 files changed, 226 insertions(+), 1029 deletions(-) diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 17a44d4d..c8e5e189 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -12,8 +12,7 @@ use grovedb_version::{ }; use integer_encoding::VarInt; -use super::CascadeOnUpdate; -use crate::{reference_path::ReferencePathType, Element, Error, Hash}; +use crate::{Element, Element::SumItem, Error, Hash}; impl Element { #[cfg(feature = "full")] @@ -23,7 +22,7 @@ impl Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. pub fn insert<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( - self, + &self, merk: &mut Merk, key: K, options: Option, @@ -31,40 +30,21 @@ impl Element { ) -> CostResult<(), Error> { check_grovedb_v0_with_cost!("insert", grove_version.grovedb_versions.element.insert); - let mut original_cost = Default::default(); - - let to_insert = if let Some(mut prev) = cost_return_on_error!( - &mut original_cost, - Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) - ) { - cost_return_on_error_no_add!( - original_cost, - self.promote_to_referenced_variant(&mut prev) - ) - } else { - self - }; - - let serialized = cost_return_on_error_default!(to_insert.serialize(grove_version)); + let serialized = cost_return_on_error_default!(self.serialize(grove_version)); - if !merk.is_sum_tree && to_insert.is_sum_item() { + if !merk.is_sum_tree && self.is_sum_item() { return Err(Error::InvalidInput("cannot add sum item to non sum tree")) - .wrap_with_cost(original_cost); + .wrap_with_cost(Default::default()); } let merk_feature_type = - cost_return_on_error_default!(to_insert.get_feature_type(merk.is_sum_tree)); - let batch_operations = if matches!( - to_insert, - Element::SumItem(..) | Element::SumItemWithBackwardsReferences(..) - ) { - let value_cost = cost_return_on_error_no_add!( - original_cost, - to_insert.get_specialized_cost(grove_version) - ); + cost_return_on_error_default!(self.get_feature_type(merk.is_sum_tree)); + let batch_operations = if matches!(self, SumItem(..)) { + let value_cost = + cost_return_on_error_default!(self.get_specialized_cost(grove_version)); let cost = value_cost - + to_insert.get_flags().as_ref().map_or(0, |flags| { + + self.get_flags().as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); @@ -89,7 +69,6 @@ impl Element { grove_version, ) .map_err(|e| Error::CorruptedData(e.to_string())) - .add_cost(original_cost) } #[cfg(feature = "full")] @@ -115,7 +94,7 @@ impl Element { Err(e) => return Err(e).wrap_with_cost(Default::default()), }; - let entry = if matches!(self, Element::SumItem(..)) { + let entry = if matches!(self, SumItem(..)) { let value_cost = cost_return_on_error_default!(self.get_specialized_cost(grove_version)); @@ -142,7 +121,7 @@ impl Element { /// If transaction is passed, the operation will be committed on the /// transaction commit. pub fn insert_if_not_exists<'db, S: StorageContext<'db>>( - self, + &self, merk: &mut Merk, key: &[u8], options: Option, @@ -210,52 +189,21 @@ impl Element { } } - #[cfg(feature = "full")] - /// Promote `Element` to referenced variant in case the old one was already - /// referenced. - fn promote_to_referenced_variant(self, old_element: &mut Element) -> Result { - if let Some(refs) = old_element.take_backward_references() { - // Since variants with backward references are publicly available, we still have - // to address them, meaning filling in the actual information about references - // from the database by discarding user input. - - match self { - Element::Item(value, flags) - | Element::ItemWithBackwardsReferences(value, _, flags) => { - Ok(Element::ItemWithBackwardsReferences(value, refs, flags)) - } - Element::Reference(ref_path, max_hops, flags) - | Element::BidirectionalReference(ref_path, _, max_hops, flags) => Ok( - Element::BidirectionalReference(ref_path, refs, max_hops, flags), - ), - Element::SumItem(sum, flags) - | Element::SumItemWithBackwardsReferences(sum, _, flags) => { - Ok(Element::SumItemWithBackwardsReferences(sum, refs, flags)) - } - - Element::Tree(..) | Element::SumTree(..) => Err(Error::NotSupported( - "cannot insert subtree in place of a referenced item".to_owned(), - )), - } - } else { - Ok(self) - } - } - #[cfg(feature = "full")] /// Insert an element in Merk under a key if the value is different from /// what already exists; path should be resolved and proper Merk should /// be loaded by this moment If transaction is not passed, the batch /// will be written immediately. If transaction is passed, the operation /// will be committed on the transaction commit. - /// If the value changed, it returns the old element under `Some`. + /// The bool represents if we indeed inserted. + /// If the value changed we return the old element. pub fn insert_if_changed_value<'db, S: StorageContext<'db>>( - self, + &self, merk: &mut Merk, key: &[u8], options: Option, grove_version: &GroveVersion, - ) -> CostResult, Error> { + ) -> CostResult<(bool, Option), Error> { check_grovedb_v0_with_cost!( "insert_if_changed_value", grove_version @@ -265,29 +213,19 @@ impl Element { ); let mut cost = OperationCost::default(); - let mut previous_element = cost_return_on_error!( + let previous_element = cost_return_on_error!( &mut cost, Self::get_optional_from_storage(&merk.storage, key, grove_version) ); - let to_insert = if let Some(prev) = previous_element.as_mut() { - cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(prev)) - } else { - self + let needs_insert = match &previous_element { + None => true, + Some(previous_element) => previous_element != self, }; - - let changed = previous_element - .as_ref() - .map(|p| !p.eq_no_backreferences(&to_insert)) - .unwrap_or(true); - - if changed { - cost_return_on_error!( - &mut cost, - to_insert.insert(merk, key, options, grove_version) - ); - Ok(previous_element).wrap_with_cost(cost) + if !needs_insert { + Ok((false, None)).wrap_with_cost(cost) } else { - Ok(None).wrap_with_cost(cost) + cost_return_on_error!(&mut cost, self.insert(merk, key, options, grove_version)); + Ok((true, previous_element)).wrap_with_cost(cost) } } @@ -347,43 +285,28 @@ impl Element { /// If transaction is not passed, the batch will be written immediately. /// If transaction is passed, the operation will be committed on the /// transaction commit. - /// Returns `bool` that indicates if a reference propagation is required. pub fn insert_reference<'db, K: AsRef<[u8]>, S: StorageContext<'db>>( - self, + &self, merk: &mut Merk, key: K, referenced_value: Hash, options: Option, grove_version: &GroveVersion, - ) -> CostResult { + ) -> CostResult<(), Error> { check_grovedb_v0_with_cost!( "insert_reference", grove_version.grovedb_versions.element.insert_reference ); - let mut cost = Default::default(); - - let (ref_updated, to_insert) = if let Some(mut prev) = cost_return_on_error!( - &mut cost, - Self::get_optional_from_storage(&merk.storage, key.as_ref(), grove_version) - ) { - ( - !prev.eq_no_backreferences(&self) && prev.has_backward_references(), - cost_return_on_error_no_add!(cost, self.promote_to_referenced_variant(&mut prev)), - ) - } else { - (false, self) - }; - - let serialized = match to_insert.serialize(grove_version) { + let serialized = match self.serialize(grove_version) { Ok(s) => s, - Err(e) => return Err(e).wrap_with_cost(cost), + Err(e) => return Err(e).wrap_with_cost(Default::default()), }; + let mut cost = OperationCost::default(); let merk_feature_type = cost_return_on_error!( &mut cost, - to_insert - .get_feature_type(merk.is_sum_tree) + self.get_feature_type(merk.is_sum_tree) .wrap_with_cost(OperationCost::default()) ); @@ -404,7 +327,6 @@ impl Element { grove_version, ) .map_err(|e| Error::CorruptedData(e.to_string())) - .map_ok(|_| ref_updated) } #[cfg(feature = "full")] @@ -541,52 +463,6 @@ impl Element { batch_operations.push(entry); Ok(()).wrap_with_cost(Default::default()) } - - /// Adds info on reference that points to this element. - pub(crate) fn referenced_from( - mut self, - reference: ReferencePathType, - cascade_on_update: CascadeOnUpdate, - ) -> Result { - match self { - Element::Item(data, flags) => Ok(Element::ItemWithBackwardsReferences( - data, - vec![(reference, cascade_on_update)], - flags, - )), - Element::ItemWithBackwardsReferences(_, ref mut backward_references, _) => { - backward_references.push((reference, cascade_on_update)); - Ok(self) - } - Element::Reference(reference_path, max_hops, flags) => { - Ok(Element::BidirectionalReference( - reference_path, - vec![(reference, cascade_on_update)], - max_hops, - flags, - )) - } - Element::BidirectionalReference(_, ref mut backward_references, ..) => { - backward_references.push((reference, cascade_on_update)); - Ok(self) - } - Element::SumItem(value, flags) => Ok(Element::SumItemWithBackwardsReferences( - value, - vec![(reference, cascade_on_update)], - flags, - )), - Element::SumItemWithBackwardsReferences(_, ref mut backward_references, _) => { - backward_references.push((reference, cascade_on_update)); - Ok(self) - } - Element::Tree(..) => Err(Error::NotSupported( - "Cannot add references pointing to subtrees".to_owned(), - )), - Element::SumTree(..) => Err(Error::NotSupported( - "Cannot add references pointing to sumtrees".to_owned(), - )), - } - } } #[cfg(feature = "full")] @@ -634,13 +510,14 @@ mod tests { merk.commit(grove_version); - let previous = Element::new_item(b"value".to_vec()) + let (inserted, previous) = Element::new_item(b"value".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); merk.commit(grove_version); + assert!(!inserted); assert_eq!(previous, None); assert_eq!( Element::get(&merk, b"another-key", true, grove_version) @@ -675,11 +552,12 @@ mod tests { let batch = StorageBatch::new(); let mut merk = empty_path_merk(&*storage, &batch, &tx, grove_version); - let previous = Element::new_item(b"value2".to_vec()) + let (inserted, previous) = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); + assert!(inserted); assert_eq!(previous, Some(Element::new_item(b"value".to_vec())),); storage @@ -704,11 +582,12 @@ mod tests { .insert(&mut merk, b"mykey", None, grove_version) .unwrap() .expect("expected successful insertion"); - let previous = Element::new_item(b"value2".to_vec()) + let (inserted, previous) = Element::new_item(b"value2".to_vec()) .insert_if_changed_value(&mut merk, b"another-key", None, grove_version) .unwrap() .expect("expected successful insertion 2"); + assert!(inserted); assert_eq!(previous, None); assert_eq!( diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index d607d35e..bbbc6b47 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -22,7 +22,6 @@ pub use query::QueryOptions; mod serialize; #[cfg(any(feature = "full", feature = "verify"))] use std::fmt; -use std::mem; use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] @@ -93,7 +92,7 @@ pub enum Element { /// A reference to an object by its path BidirectionalReference( ReferencePathType, - Vec<(ReferencePathType, CascadeOnUpdate)>, + Option<(ReferencePathType, CascadeOnUpdate)>, MaxReferenceHop, Option, ), @@ -308,30 +307,6 @@ impl Element { crate::value_hash(&bytes).map(Result::Ok) } - /// Returns backward references if the `Element` in question participates in - /// bidirectional referencing machinery. - pub(crate) fn take_backward_references( - &mut self, - ) -> Option> { - match self { - Element::BidirectionalReference(_, refs, ..) - | Element::ItemWithBackwardsReferences(_, refs, ..) - | Element::SumItemWithBackwardsReferences(_, refs, ..) - if !refs.is_empty() => - { - Some(mem::take(refs)) - } - _ => None, - } - } - - /// Returns true if there are references to this `Element`. - pub(crate) fn has_backward_references(&self) -> bool { - matches!(self, Element::BidirectionalReference(_, refs, ..) - | Element::ItemWithBackwardsReferences(_, refs, ..) - | Element::SumItemWithBackwardsReferences(_, refs, ..) if !refs.is_empty()) - } - /// Checks elements for equality ignoring backreferences part. pub(crate) fn eq_no_backreferences(&self, other: &Self) -> bool { use Element::*; diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index 5b0a3e3c..52bea01e 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -393,59 +393,6 @@ impl GroveDb { subtree.root_hash().map(Ok).add_cost(cost) } - /// Method to propagate updated subtree key changes one level up inside a - /// transaction - fn propagate_changes_with_batch_transaction<'b, B: AsRef<[u8]>>( - &self, - storage_batch: &StorageBatch, - mut merk_cache: HashMap, Merk>, - path: &SubtreePath<'b, B>, - transaction: &Transaction, - grove_version: &GroveVersion, - ) -> CostResult<(), Error> { - let mut cost = OperationCost::default(); - - let mut child_tree = cost_return_on_error_no_add!( - cost, - merk_cache.remove(path).ok_or(Error::CorruptedCodeExecution( - "Merk Cache should always contain the last path", - )) - ); - - let mut current_path = path.clone(); - - while let Some((parent_path, parent_key)) = current_path.derive_parent() { - let mut parent_tree = cost_return_on_error!( - &mut cost, - self.open_batch_transactional_merk_at_path( - storage_batch, - parent_path.clone(), - transaction, - false, - grove_version, - ) - ); - let (root_hash, root_key, sum) = cost_return_on_error!( - &mut cost, - child_tree.root_hash_key_and_sum().map_err(Error::MerkError) - ); - cost_return_on_error!( - &mut cost, - Self::update_tree_item_preserve_flag( - &mut parent_tree, - parent_key, - root_key, - root_hash, - sum, - grove_version, - ) - ); - child_tree = parent_tree; - current_path = parent_path; - } - Ok(()).wrap_with_cost(cost) - } - /// Method to propagate updated subtree key changes one level up inside a /// transaction fn propagate_changes_with_transaction<'b, B: AsRef<[u8]>>( diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index a01a46c3..ab0afc21 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -1,852 +1,264 @@ -//! Module dedicated to keep necessary Merks in memory and solve propagation -//! after usage automatically. +//! Module dedicated to keep necessary Merks in memory. use std::{ - cell::RefCell, - collections::{btree_map::Entry, BTreeMap, HashSet}, - mem::{self, MaybeUninit}, - ops::Deref, + cell::{Cell, UnsafeCell}, + collections::{btree_map::Entry, BTreeMap}, + ops::{Deref, DerefMut}, }; use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; -use grovedb_merk::{tree::NULL_HASH, CryptoHash, Merk, MerkOptions}; +use grovedb_merk::Merk; use grovedb_path::SubtreePath; -use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; +use grovedb_storage::{ + rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, + StorageBatch, +}; use grovedb_version::version::GroveVersion; -use crate::{ - element::CascadeOnUpdate, reference_path::ReferencePathType, Element, Error, GroveDb, - Transaction, -}; +use crate::{Element, Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; -type Key = Vec; - -struct CachedMerk<'db> { - to_propagate: bool, - merk: TxMerk<'db>, -} - -// type UpdatedReferences<'b, B> = RefCell, -// Vec)>>; - -/// Helper struct to split `MerkCache` into independent parts to allow splitting -/// borrows, meaning a dependency on the storage part that shall have a certain -/// lifetime won't clash with another dependency unrelated to the storage part. -struct MerkCacheStorage<'db, 'b, B> { - /// Subtrees opened during usage of this cache structure, the wrapper also - /// marks those that were changed. - merks: BTreeMap, CachedMerk<'db>>, - /// GroveDb provides a storage to open Merks against. +/// Structure to keep subtrees open in memory for repeated access. +pub(crate) struct MerkCache<'db, 'b, B: AsRef<[u8]>> { db: &'db GroveDb, - /// Nowadays GroveDb operates solely on transactional storage contexts. + version: &'db GroveVersion, + batch: Box, tx: &'db Transaction<'db>, - /// The `MerkCache` finalization result is a `StorageBatch` of operations. - /// It's then up to the user what actions to take on that result until - /// no further changes can be made to the storage. - batch: &'static StorageBatch, + merks: UnsafeCell, Box<(Cell, TxMerk<'db>)>>>, } -impl<'db, 'b, B: AsRef<[u8]>> MerkCacheStorage<'db, 'b, B> { - fn new(db: &'db GroveDb, tx: &'db Transaction<'db>) -> Self { - MerkCacheStorage { - merks: Default::default(), +impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { + /// Initialize a new `MerkCache` instance + pub(crate) fn new( + db: &'db GroveDb, + tx: &'db Transaction<'db>, + version: &'db GroveVersion, + ) -> Self { + MerkCache { db, tx, - batch: Box::leak(Box::new(StorageBatch::default())), + version, + merks: Default::default(), + batch: Default::default(), } } - /// Get a mutable Merk reference from the cache. - /// If it doesn't present then it will be opened. - /// Returns `None` if there is no Merk under this path. - fn get_merk_mut_internal<'c>( - &'c mut self, + /// Returns cached Merk reference or opens one if needed. + /// + /// # Panics + /// Borrowing one Merk several times will cause a panic, previous borrow + /// shall reach end of the scope or to be `drop`ped manually. + pub(crate) fn get_merk<'c>( + &'c self, path: SubtreePath<'b, B>, - version: &GroveVersion, - ) -> CostResult<&'c mut CachedMerk<'db>, Error> { + ) -> CostResult, Error> { let mut cost = Default::default(); - match self.merks.entry(path) { - Entry::Occupied(e) => Ok(e.into_mut()).wrap_with_cost(cost), + // SAFETY: there are no other references to `merks` memory at the same time. + let boxed_flag_merk = match unsafe { + self.merks + .get() + .as_mut() + .expect("`UnsafeCell` is never null") + } + .entry(path.clone()) + { Entry::Vacant(e) => { let merk = cost_return_on_error!( &mut cost, self.db.open_transactional_merk_at_path( e.key().clone(), self.tx, - Some(self.batch), - version + // SAFETY: batch is allocated on the heap and we use only shared + // references, so as long as the `Box` allocation + // outlives those references we're safe, + // and it will outlive because Merks are dropped first. + Some(unsafe { + (&*self.batch as *const StorageBatch) + .as_ref() + .expect("`Box` is never null") + }), + self.version ) ); - Ok(e.insert(CachedMerk { - merk, - to_propagate: false, - })) - .wrap_with_cost(cost) + e.insert(Box::new((true.into(), merk))) } - } - } -} - -/// Merk caching structure. -/// -/// Since we usually postpone all writes to the very end with a single RocksDB -/// batch all intermediate changes to subtrees might not be tracked if we reopen -/// those Merks, so it's better to have them cached and proceed through the same -/// structure. Eventually we'll have enough info at the same place to perform -/// necessary propagations as well. -// SAFETY: please consult with other safety docs here before doing any changes -pub(crate) struct MerkCache<'db, 'b, B> { - storage: MerkCacheStorage<'db, 'b, B>, - version: &'db GroveVersion, -} + Entry::Occupied(e) => { + if e.get().0.get() { + e.into_mut() + } else { + panic!("Double borrow of a cached Merk") + } + } + }; -impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { - pub(crate) fn new<'tx>( - db: &'db GroveDb, - tx: &'db Transaction<'db>, - version: &'db GroveVersion, - ) -> Self { - MerkCache { - storage: MerkCacheStorage::new(db, tx), - version, - // updated_references: Default::default(), - } + let taken_handle_ref: *const Cell = &boxed_flag_merk.0 as *const _; + let merk_ptr: *mut TxMerk<'db> = &mut boxed_flag_merk.1 as *mut _; + + // SAFETY: `MerkHandle` contains two references to the heap allocated memory, + // and we want to be sure that the referenced data will outlive those + // references plus borrowing rules aren't violated (one `&mut` or many + // `&` with no `&mut` at a time). + // + // To make sure changes to the internal structure won't affect existing borrows + // we have an indirection in a form of `Box`, that allows us to move and update + // `MerkCache` with new subtrees and possible reallocations without breaking + // `MerkHandle`'s references. We use `UnsafeCell` to connect lifetimes and check + // in compile time that `MerkHandle`s won't outlive the cache, even though we + // don't hold any references to it, but `&mut` reference would make this borrow + // exclusive for the whole time of `MerkHandle`, so it shall go intially through + // a shared reference. + // + // Borrowing rules are covered slightly more complicated way: + // 1. Of a pair behind heap allocation only Merk is uniquely borrowed by + // `MerkHandle` + // 2. Borrow flag is referenced by `MerkHandle` to be updated on `Drop` and is + // referenced while taking a new `MerkHandle` to check if it was already + // borrowed, that gives us two shared references to the same memory and + // that's allowed, note we're not referring to the Merk part of the pair + // 3. Borrow flag's reference points to a heap allocated memory and will remain + // valid just as the Merk reference to the memory right after the flag + Ok(unsafe { + MerkHandle { + merk: merk_ptr.as_mut().expect("`Box` contents are never null"), + taken_handle: taken_handle_ref + .as_ref() + .expect("`Box` contents are never null"), + } + }) + .wrap_with_cost(cost) } - /// Returns an array of mutable references to different Merks, where each - /// element in the array corresponds to a unique Merk based on its - /// position in the input paths array. - /// - /// # Panics - /// All input paths *must* be unique, otherwise it could provide multiple - /// mutable references to the same memory which is strictly prohibited. - pub(crate) fn get_multi_mut<'c, const N: usize>( - &'c mut self, - paths: [SubtreePath<'b, B>; N], - ) -> CostResult<[MerkHandle<'db, 'c>; N], Error> { - let mut result_uninit = [const { MaybeUninit::>::uninit() }; N]; + /// Consumes `MerkCache` into accumulated batch of uncommited operations + /// with subtrees' root hash propagation done. + pub(crate) fn into_batch(mut self) -> CostResult, Error> { let mut cost = Default::default(); + cost_return_on_error!(&mut cost, self.propagate_subtrees()); - let unique_args: HashSet<_> = paths.iter().collect(); - if unique_args.len() != N { - panic!("`get_multi_mut` keys must be unique"); - } - - for (i, path) in paths.into_iter().enumerate() { - // SAFETY is ensured by tying the lifetime of mutable references to the - // collection itself, preventing them from outliving the collection and - // ensuring exclusive access to the collection's layout through other - // mutable references. The mandatory keys' uniqueness check above makes - // sure no overlapping memory will be referenced. - let merk_ref = unsafe { - MerkHandle::new( - (cost_return_on_error!( - &mut cost, - self.storage - .get_merk_mut_internal(path.clone(), self.version) - ) as *mut CachedMerk<'db>) - .as_mut::<'c>() - .expect("not a null pointer"), - // UpdatedReferenceHandle { - // path, - // updated_references: &self.updated_references, - // }, - &self.version, - ) - }; - result_uninit[i].write(merk_ref); - } - - // SAFETY: An array of `MaybeUninit` references takes the same size as an array - // of references as long as they both have the same number of elements, - // N in our case. `mem::transmute` would represent it better, however, - // due to poor support of const generics in stable Rust we bypass - // compile-time size checks with pointer casts. - let result = - unsafe { (&result_uninit as *const _ as *const [MerkHandle<'db, 'c>; N]).read() }; - mem::forget(result_uninit); - - Ok(result).wrap_with_cost(cost) + // SAFETY: By this time all subtrees are taken and dropped during + // propagation, so there are no more references to the batch and in can be + // safely released into the world. + Ok(self.batch).wrap_with_cost(cost) } - /// Summarizes all performed operations on this `MerkCache` with necessary - /// propagations into a `Storagebatch`. - pub(crate) fn finalize(mut self) -> CostResult, Error> { - let batch_ptr = self.storage.batch as *const _; - - // Propagate updated subtrees' hashes up to the root and dropping all possible - // batch users: - let propagation_result = { - self.propagate_updated_references() - .flat_map_ok(|_| self.propagate_updated_merks()) - }; - - // SAFETY: The batch reference was created by `Box::leak` and restored into a - // `Box` form. No shared usage of that memory exists because the - // `MerkCache` structure was dropped above. - let result_batch = unsafe { Box::from_raw(batch_ptr as *mut _) }; - - // The batch's unsafety cleanup happens regardless of propagation results, so no - // early termination allowed. We do the mapping afterwards: - propagation_result.map_ok(|_| result_batch) - } - - // /// Inserts a reference into a cached Merk. - // /// The reason why this has to be a whole `MerkCache` method is that - // /// references involve opening and modifying several Merks at once, this - // /// makes it out of scope for a single `MerkHandle`. - // pub(crate) fn insert_reference<'c>( - // &'c mut self, - // path: SubtreePath<'b, B>, - // key: &[u8], - // ref_element: ReferenceElement, - // cascade_on_update: bool, - // options: Option, - // ) -> CostResult<(), Error> { - // let mut cost = Default::default(); - - // let follow_reference_result = cost_return_on_error!( - // &mut cost, - // self.follow_reference(path.clone(), key, ref_element.get_ref_path()) - // ); - - // let value_hash = cost_return_on_error!( - // &mut cost, - // follow_reference_result - // .last_element - // .value_hash(&self.version) - // ); - - // // The insertion of a reference requires that its hash value be equal to - // the // hash value of the item in the end of the reference's chain: - // let [mut merk] = cost_return_on_error!(&mut cost, - // self.get_multi_mut([path])); cost_return_on_error!( - // &mut cost, - // merk.insert_internal(key, ref_element.0.clone(), options, - // Some(value_hash)) ); - - // let version = self.version.clone(); // TODO - - // // The closest referenced item's backward references list of the chain - // shall be // updated with a new entry: - // let [mut closest_merk] = cost_return_on_error!( - // &mut cost, - // self.get_multi_mut([follow_reference_result.first_path]) - // ); - // let mut closest_element = cost_return_on_error!( - // &mut cost, - // Element::get( - // closest_merk.deref(), - // &follow_reference_result.first_key, - // true, - // &version - // ) - // ); - // // Updated backward references information: - // closest_element = cost_return_on_error_no_add!( - // cost, - // closest_element.referenced_from(ref_element.to_ref_path(), - // cascade_on_update) ); - // // And write it back: - // cost_return_on_error!( - // &mut cost, - // closest_merk.insert_internal( - // &follow_reference_result.first_key, - // closest_element, - // None, - // Some(value_hash), - // ) - // ); - - // todo!() - // } - - // /// Follows a reference returning the first link in the references chain and - // /// also the referenced item at the end of it. - // fn follow_reference( - // &mut self, - // self_path: SubtreePath<'b, B>, - // self_key: &[u8], - // reference_path: &ReferencePathType, - // ) -> CostResult, Error> { - // todo!() - // } - - /// Perform propagation of references' chains marked as changed. - fn propagate_updated_references(&mut self) -> CostResult<(), Error> { - todo!() - } - - /// Finalizes each Merk starting from the deepest subtrees, updating hashes - /// up to the root. - fn propagate_updated_merks(mut self) -> CostResult<(), Error> { + fn propagate_subtrees(&mut self) -> CostResult<(), Error> { let mut cost = Default::default(); - let version: &'db GroveVersion = self.version; + // This relies on [SubtreePath]'s ordering implementation to put the deepest + // path's first. + while let Some((path, flag_and_merk)) = self.merks.get_mut().pop_first() { + let merk = flag_and_merk.1; + if let Some((parent_path, parent_key)) = path.derive_parent() { + let mut parent_merk = cost_return_on_error!(&mut cost, self.get_merk(parent_path)); - // Picking Merks one by one as long as they have a parent - while let Some((parent_path, parent_key, subtree)) = - self.storage.merks.pop_first().and_then(|(path, subtree)| { - path.derive_parent() - .map(|(parent_path, parent_key)| (parent_path, parent_key, subtree)) - }) - { - // If a cached Merk wasn't changed, we don't need to propagate it: - if !subtree.to_propagate { - continue; + let (root_hash, root_key, sum) = cost_return_on_error!( + &mut cost, + merk.root_hash_key_and_sum().map_err(Error::MerkError) + ); + cost_return_on_error!( + &mut cost, + GroveDb::update_tree_item_preserve_flag( + &mut parent_merk, + parent_key, + root_key, + root_hash, + sum, + self.version, + ) + ); } - - let parent_subtree = cost_return_on_error!( - &mut cost, - self.storage.get_merk_mut_internal(parent_path, version) - ); - parent_subtree.to_propagate = true; - let (root_hash, root_key, root_sum) = cost_return_on_error!( - &mut cost, - subtree.merk.root_hash_key_and_sum().map_err(|e| e.into()) - ); - cost_return_on_error!( - &mut cost, - GroveDb::update_tree_item_preserve_flag( - &mut parent_subtree.merk, - parent_key, - root_key, - root_hash, - root_sum, - version - ) - ); } Ok(()).wrap_with_cost(cost) } } -struct FollowReferenceResult<'b, B> { - first_path: SubtreePath<'b, B>, - first_key: Vec, - last_element: Element, -} - -// /// A wrapper type to ensure `Element` is a reference wherever it is -// required. pub(crate) struct ReferenceElement(Element); - -// impl ReferenceElement { -// fn get_ref_path(&self) -> &ReferencePathType { -// match &self.0 { -// Element::Reference(ref_path, ..) | -// Element::BidirectionalReference(ref_path, ..) => { ref_path -// } -// _ => unreachable!(), -// } -// } - -// fn to_ref_path(self) -> ReferencePathType { -// match self.0 { -// Element::Reference(ref_path, ..) | -// Element::BidirectionalReference(ref_path, ..) => { ref_path -// } -// _ => unreachable!(), -// } -// } -// } - -// impl TryFrom for ReferenceElement { -// type Error = (); - -// fn try_from(value: Element) -> Result { -// match value { -// element @ Element::Reference(..) | element @ -// Element::BidirectionalReference(..) => { Ok(Self(element)) -// } -// _ => Err(()), -// } -// } -// } - -// /// A wrapper type to ensure `Element` is not a reference. -// pub(crate) struct NonReferenceElement(Element); - -// impl TryFrom for NonReferenceElement { -// type Error = (); - -// fn try_from(value: Element) -> Result { -// match value { -// Element::Reference(..) | Element::BidirectionalReference(..) => -// Err(()), element => Ok(Self(element)), -// } -// } -// } - -/// Handle to a cached Merk. +/// Wrapper over `Merk` tree to manage unqiue borrow dynamically. pub(crate) struct MerkHandle<'db, 'c> { - /// Reference to an opened Merk tree with transactional batch storage - /// context merk: &'c mut TxMerk<'db>, - version: &'db GroveVersion, - /// Mark this subtree as a subject to propagate - to_propagate: &'c mut bool, + taken_handle: &'c Cell, } -// /// Helper struct to signal `MerkCache` about updated references. -// struct UpdatedReferenceHandle<'c, 'b, B> { -// path: SubtreePath<'b, B>, -// updated_references: &'c UpdatedReferences<'b, B>, -// } - -// impl<'c, 'b, B: AsRef<[u8]>> UpdatedReferenceHandle<'c, 'b, B> { -// fn mark_updated_reference(&self, key: Key) { -// self.updated_references -// .borrow_mut() -// .insert((self.path.clone(), key)); -// } -// } - -/// It is allowed to dereference `MerkHandle` to regular Merks but in a -/// non-mutable way since we want to track what have been done to those Merks. impl<'db, 'c> Deref for MerkHandle<'db, 'c> { type Target = TxMerk<'db>; fn deref(&self) -> &Self::Target { - &self.merk + self.merk } } -/// This type represents changes made to a particular element, a difference -/// between its initial state and eventual state -enum Delta { - AddItem, - RemoveItem { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - }, - UpdateItemToReference { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - new_forward_reference: ReferencePathType, - }, - UpdateItemToItem { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - }, - AddReference { - new_forward_reference: ReferencePathType, - }, - RemoveReference { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - ex_forward_reference: ReferencePathType, - }, - UpdateReferenceToItem { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - ex_forward_reference: ReferencePathType, - }, - UpdateReferenceToReference { - backward_references: Vec<(ReferencePathType, CascadeOnUpdate)>, - new_forward_reference: ReferencePathType, - ex_forward_reference: ReferencePathType, - }, -} - -impl Delta { - fn new(old_element: Option, new_element: Element) -> Option { - match (old_element, new_element) { - ( - None | Some(Element::Tree(..) | Element::SumTree(..)), - Element::Item(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItem(..) - | Element::SumItemWithBackwardsReferences(..), - ) => Some(Self::AddItem), - ( - None | Some(Element::Tree(..) | Element::SumTree(..)), - Element::Reference(new_forward_reference, ..) - | Element::BidirectionalReference(new_forward_reference, ..), - ) => Some(Self::AddReference { - new_forward_reference, - }), - (_, Element::Tree(..) | Element::SumTree(..)) => None, - - ( - Some( - Element::ItemWithBackwardsReferences(_, backward_references, _) - | Element::SumItemWithBackwardsReferences(_, backward_references, _), - ), - Element::Item(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItem(..) - | Element::SumItemWithBackwardsReferences(..), - ) => Some(Self::UpdateItemToItem { - backward_references, - }), - ( - Some(Element::Item(..) | Element::SumItem(..)), - Element::Item(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItem(..) - | Element::SumItemWithBackwardsReferences(..), - ) => Some(Self::UpdateItemToItem { - backward_references: Vec::new(), - }), - ( - Some( - Element::ItemWithBackwardsReferences(_, backward_references, _) - | Element::SumItemWithBackwardsReferences(_, backward_references, _), - ), - Element::Reference(new_forward_reference, ..) - | Element::BidirectionalReference(new_forward_reference, ..), - ) => Some(Self::UpdateItemToReference { - backward_references, - new_forward_reference, - }), - ( - Some(Element::Item(..) | Element::SumItem(..)), - Element::Reference(new_forward_reference, ..) - | Element::BidirectionalReference(new_forward_reference, ..), - ) => Some(Self::UpdateItemToReference { - backward_references: Vec::new(), - new_forward_reference, - }), - ( - Some(Element::Reference(ex_forward_reference, ..)), - Element::Item(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItem(..) - | Element::SumItemWithBackwardsReferences(..), - ) => Some(Self::UpdateReferenceToItem { - backward_references: Vec::new(), - ex_forward_reference, - }), - ( - Some(Element::BidirectionalReference( - ex_forward_reference, - backward_references, - .., - )), - Element::Item(..) - | Element::ItemWithBackwardsReferences(..) - | Element::SumItem(..) - | Element::SumItemWithBackwardsReferences(..), - ) => Some(Self::UpdateReferenceToItem { - backward_references, - ex_forward_reference, - }), - ( - Some(Element::Reference(ex_forward_reference, ..)), - Element::Reference(new_forward_reference, ..) - | Element::BidirectionalReference(new_forward_reference, ..), - ) => Some(Self::UpdateReferenceToReference { - backward_references: Vec::new(), - new_forward_reference, - ex_forward_reference, - }), - ( - Some(Element::BidirectionalReference( - ex_forward_reference, - backward_references, - .., - )), - Element::Reference(new_forward_reference, ..) - | Element::BidirectionalReference(new_forward_reference, ..), - ) => Some(Self::UpdateReferenceToReference { - backward_references, - new_forward_reference, - ex_forward_reference, - }), - } - } - - fn new_from_delete(element: Element) -> Option { - match element { - Element::Item(..) | Element::SumItem(..) => Some(Self::RemoveItem { - backward_references: Vec::new(), - }), - Element::ItemWithBackwardsReferences(_, backward_references, ..) - | Element::SumItemWithBackwardsReferences(_, backward_references, ..) => { - Some(Self::RemoveItem { - backward_references, - }) - } - Element::Reference(ex_forward_reference, ..) => Some(Self::RemoveReference { - backward_references: Vec::new(), - ex_forward_reference, - }), - Element::BidirectionalReference(ex_forward_reference, backward_references, ..) => { - Some(Self::RemoveReference { - backward_references, - ex_forward_reference, - }) - } - Element::Tree(..) | Element::SumTree(..) => None, - } +impl<'db, 'c> DerefMut for MerkHandle<'db, 'c> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.merk } } -impl<'db, 'c> MerkHandle<'db, 'c> { - pub(crate) fn insert( - &mut self, - key: &[u8], - element: Element, - options: Option, - ) -> CostResult<(), Error> { - let mut costs = Default::default(); - - let old_value = cost_return_on_error!( - &mut costs, - match element { - Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => - if value.is_some() { - Err(Error::InvalidCodeExecution( - "a tree should be empty at the moment of insertion when not using \ - batches", - )) - .wrap_with_cost(Default::default()) - } else { - element - .insert_subtree(self.merk, key, NULL_HASH, options, self.version) - .map_ok(|_| None) - }, - _ => element.insert_if_changed_value(self.merk, key, options, self.version), - } - ); - - *self.to_propagate = true; - - Ok(()).wrap_with_cost(costs) - } - - // fn insert_internal( - // &mut self, - // key: &[u8], - // element: Element, - // options: Option, - // referenced_value_hash: Option, - // ) -> CostResult<(), Error> { - // let mut costs = Default::default(); - - // // In case the item that was changed has been referenced, we indicate - // that // references should be propagated after - // if cost_return_on_error!( - // &mut costs, - // match element { - // Element::Item(..) - // | Element::SumItem(..) - // | Element::ItemWithBackwardsReferences(..) - // | Element::SumItemWithBackwardsReferences(..) => element - // .insert_if_changed_value(self.merk, key, options, - // self.version) .map_ok(|r| r.0), - // Element::Reference(..) | Element::BidirectionalReference(..) => - // element .insert_reference( - // self.merk, - // key, - // referenced_value_hash.expect("todo"), - // options, - // self.version - // ), - // Element::Tree(ref value, ..) | Element::SumTree(ref value, ..) => - // if value.is_some() { - // Err(Error::InvalidCodeExecution( - // "a tree should be empty at the moment of insertion - // when not using \ batches", - // )) - // .wrap_with_cost(Default::default()) - // } else { - // element - // .insert_subtree(self.merk, key, NULL_HASH, options, - // self.version) .map_ok(|_| false) - // }, - // } - // ) { - // self.updated_reference_handle - // .mark_updated_reference(key.to_vec()); - // } - - // *self.to_propagate = true; - - // Ok(()).wrap_with_cost(costs) - // } - - fn new(cached_merk: &'c mut CachedMerk<'db>, version: &'db GroveVersion) -> Self { - Self { - merk: &mut cached_merk.merk, - version, - to_propagate: &mut cached_merk.to_propagate, - } +impl<'db, 'c> Drop for MerkHandle<'db, 'c> { + fn drop(&mut self) { + self.taken_handle.set(false); } } #[cfg(test)] mod tests { - use grovedb_costs::OperationCost; use grovedb_path::SubtreePath; - use grovedb_storage::Storage; + use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; use super::MerkCache; use crate::{ - reference_path::ReferencePathType, - tests::{make_deep_tree, ANOTHER_TEST_LEAF, TEST_LEAF}, + tests::{make_deep_tree, make_test_grovedb, TEST_LEAF}, Element, }; - #[test] - fn cached_subtrees_are_free() { - let version = GroveVersion::latest(); - let db = make_deep_tree(&version); - let tx = db.start_transaction(); - let mut cache = MerkCache::new(&db, &tx, version); - - let mut cost: OperationCost = Default::default(); - let [test1, test2] = cache - .get_multi_mut([ - SubtreePath::from(&[TEST_LEAF]), - SubtreePath::from(&[ANOTHER_TEST_LEAF]), - ]) - .unwrap_add_cost(&mut cost) - .expect("unable to get subtrees"); - - // Assert trees aren't empty - assert!(test1.root_hash().unwrap() != [0; 32]); - assert!(test2.root_hash().unwrap() != [0; 32]); - - // Assert some cost been paid - assert!(!cost.is_nothing()); - - let mut next_cost: OperationCost = Default::default(); - let [_test1, _test2] = cache - .get_multi_mut([ - SubtreePath::from(&[TEST_LEAF]), - SubtreePath::from(&[ANOTHER_TEST_LEAF]), - ]) - .unwrap_add_cost(&mut next_cost) - .expect("unable to get subtrees"); - - // Assert it was for free now - assert!(next_cost.is_nothing()); - } - #[test] #[should_panic] - fn overlapping_references_should_panic() { + fn cant_borrow_twice() { let version = GroveVersion::latest(); - let db = make_deep_tree(&version); + let db = make_test_grovedb(&version); let tx = db.start_transaction(); - let mut cache = MerkCache::new(&db, &tx, version); - let _ = cache.get_multi_mut([ - SubtreePath::from(&[TEST_LEAF]), - SubtreePath::from(&[TEST_LEAF]), - ]); + let cache = MerkCache::new(&db, &tx, version); + + cache.get_merk(SubtreePath::empty()).unwrap().unwrap(); + cache.get_merk(SubtreePath::empty()).unwrap().unwrap(); } #[test] - fn changes_to_merk_cache_provide_non_empty_batch() { + fn subtrees_are_propagated() { let version = GroveVersion::latest(); let db = make_deep_tree(&version); let tx = db.start_transaction(); - let mut cache = MerkCache::new(&db, &tx, version); - let [_subtree] = cache - .get_multi_mut([SubtreePath::from(&[TEST_LEAF])]) - .unwrap() - .unwrap(); + let path = SubtreePath::from(&[TEST_LEAF, b"innertree"]); + let item = Element::new_item(b"hello".to_vec()); - // Do nothing and finalize a batch: - assert!(cache.finalize().unwrap().unwrap().len() == 0); + let no_propagation_ops_count = { + let batch = StorageBatch::new(); - let mut cache = MerkCache::new(&db, &tx, version); - let [mut subtree] = cache - .get_multi_mut([SubtreePath::from(&[TEST_LEAF])]) - .unwrap() - .unwrap(); - subtree - .insert( - b"ayy", - Element::new_item(b"lmao".to_vec()).try_into().unwrap(), - None, - ) - .unwrap() - .unwrap(); + let mut merk = db + .open_transactional_merk_at_path(path.clone(), &tx, Some(&batch), &version) + .unwrap() + .unwrap(); - // Do something and finalize another batch: - assert!(cache.finalize().unwrap().unwrap().len() > 0); - } + item.insert(&mut merk, b"k1", None, &version) + .unwrap() + .unwrap(); - #[test] - fn changes_to_merk_are_propagated() { - let version = GroveVersion::latest(); - let db = make_deep_tree(&version); - let tx = db.start_transaction(); - - let pre_hash = db.root_hash(Some(&tx), &version).unwrap().unwrap(); + batch.len() + }; - let mut cache = MerkCache::new(&db, &tx, version); - let [mut subtree] = cache - .get_multi_mut([SubtreePath::from(&[TEST_LEAF, b"innertree"])]) - .unwrap() - .unwrap(); - subtree - .insert( - b"ayy", - Element::new_item(b"lmao".to_vec()).try_into().unwrap(), - None, - ) - .unwrap() - .unwrap(); + let cache = MerkCache::new(&db, &tx, version); - let batch = cache.finalize().unwrap().unwrap(); + let mut merk = cache.get_merk(path).unwrap().unwrap(); - db.db - .commit_multi_context_batch(*batch, Some(&tx)) + item.insert(&mut merk, b"k1", None, &version) .unwrap() .unwrap(); - assert_ne!( - pre_hash, - db.root_hash(Some(&tx), &version).unwrap().unwrap() - ) - } + drop(merk); - // #[test] - // fn changes_to_referenced_values_are_marked_uncommitted() { - // let version = GroveVersion::latest(); - // let db = make_deep_tree(&version); - // let tx = db.start_transaction(); - - // let mut cache = MerkCache::new(&db, &tx, version); - // cache - // .insert_reference( - // SubtreePath::from(&[TEST_LEAF, b"innertree"]), - // b"ayy", - // - // Element::new_reference(ReferencePathType::AbsolutePathReference(vec![ - // ANOTHER_TEST_LEAF.to_vec(), - // b"innertree2".to_vec(), - // b"k3".to_vec(), - // ])) - // .try_into() - // .unwrap(), - // false, - // None, - // ) - // .unwrap() - // .unwrap(); - - // assert!(cache.updated_references.borrow().is_empty()); - - // let [mut subtree] = cache - // .get_multi_mut([SubtreePath::from(&[ANOTHER_TEST_LEAF, - // b"innertree2"])]) .unwrap() - // .unwrap(); - - // subtree - // .insert( - // b"k3", - // Element::new_item(b"huh".to_vec()).try_into().unwrap(), - // None, - // ) - // .unwrap() - // .unwrap(); - - // assert!(!cache.updated_references.borrow().is_empty()); - // } + assert!(cache.into_batch().unwrap().unwrap().len() > no_propagation_ops_count); + } } diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index fb81532b..4accb506 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -707,11 +707,11 @@ impl GroveDb { merk_cache.insert(path.clone(), merk_to_delete_tree_from); cost_return_on_error!( &mut cost, - self.propagate_changes_with_batch_transaction( - batch, + self.propagate_changes_with_transaction( merk_cache, - &path, + path, transaction, + batch, grove_version, ) ); diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 6a07b41b..b6e00be0 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -221,12 +221,7 @@ impl GroveDb { } match element { - Element::Reference(ref reference_path, ..) - | Element::BidirectionalReference(ref reference_path, ..) => { - // TODO: 1. turn referenced value into a backreference kind - // 2. if referencing another reference first, that one shall become - // bidirectional, the rest shall be bidirectional already - + Element::Reference(ref reference_path, ..) => { let path = path.to_vec(); // TODO: need for support for references in path library let reference_path = cost_return_on_error!( &mut cost, @@ -288,16 +283,6 @@ impl GroveDb { } } _ => { - // TODO: Check if overwriting an backreference-flavored item: - // 1. Item/SumItem -- update reference chains - // 2. Replace bidirectional reference with item: Previous in chain -- - // update hashes Next in chain reference -- make one directional - // reference if the last one Next is an actual item -- downgrade from - // backreferenced-flavor if the last one - // 3. Replace bidirectional reference with another reference: Old target - // reference becomes one directional if the last one Old target item is - // downgraded if the last one Update previous in chain with a new - // target cost_return_on_error!( &mut cost, element.insert( @@ -850,8 +835,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 4, /* 1 to get tree, 1 to check prev value for ref propagation, 1 to - * insert, 1 to insert into root tree */ + seek_count: 3, // 1 to get tree, 1 to insert, 1 to insert into root tree storage_cost: StorageCost { added_bytes: 149, replaced_bytes: 0, @@ -918,7 +902,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, + seek_count: 5, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 84, // todo: verify @@ -1001,7 +985,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 8, + seek_count: 7, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 217, @@ -1080,7 +1064,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 8, + seek_count: 7, storage_cost: StorageCost { added_bytes: 170, replaced_bytes: 217, // todo: verify @@ -1141,7 +1125,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 4, // 1 to get tree, 1 to insert, 1 to insert into root tree + seek_count: 3, // 1 to get tree, 1 to insert, 1 to insert into root tree storage_cost: StorageCost { added_bytes: 153, replaced_bytes: 0, @@ -1406,7 +1390,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 5, // todo: verify this storage_cost: StorageCost { added_bytes: 150, replaced_bytes: 78, @@ -1471,7 +1455,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 4, // todo: verify this + seek_count: 3, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 0, @@ -1565,7 +1549,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 5, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 78, @@ -1661,7 +1645,7 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 6, // todo: verify this + seek_count: 5, // todo: verify this storage_cost: StorageCost { added_bytes: 156, replaced_bytes: 82, @@ -1730,13 +1714,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 4, // todo: verify this + seek_count: 3, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 112, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 124, + storage_loaded_bytes: 77, hash_node_calls: 2, } ); @@ -1784,13 +1768,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 190, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 277, // todo verify this + storage_loaded_bytes: 230, // todo verify this hash_node_calls: 8, } ); @@ -1838,13 +1822,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 248, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 411, // todo verify this + storage_loaded_bytes: 266, // todo verify this hash_node_calls: 9, } ); @@ -1904,13 +1888,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 10, // todo: verify this + seek_count: 9, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 409, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 632, // todo verify this + storage_loaded_bytes: 487, // todo verify this hash_node_calls: 11, } ); @@ -1958,13 +1942,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 0, replaced_bytes: 248, removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 421, // todo verify this + storage_loaded_bytes: 276, // todo verify this hash_node_calls: 9, } ); @@ -2012,13 +1996,13 @@ mod tests { assert_eq!( cost, OperationCost { - seek_count: 7, // todo: verify this + seek_count: 6, // todo: verify this storage_cost: StorageCost { added_bytes: 1, replaced_bytes: 191, // todo: verify this removed_bytes: NoStorageRemoval }, - storage_loaded_bytes: 279, + storage_loaded_bytes: 231, hash_node_calls: 8, } ); From 919ed0ea023adedad27718e90e5330beecca7156 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Fri, 6 Dec 2024 21:55:01 +0100 Subject: [PATCH 23/27] wip --- grovedb/src/merk_cache.rs | 24 ++++-- grovedb/src/operations/insert/mod.rs | 120 +++++++-------------------- grovedb/src/util.rs | 5 +- path/src/subtree_path.rs | 28 +++++++ path/src/subtree_path_builder.rs | 89 ++++++++++++++++---- path/src/util/compact_bytes.rs | 46 +++++++++- 6 files changed, 195 insertions(+), 117 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index ab0afc21..0e59ec88 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -8,7 +8,7 @@ use std::{ use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; use grovedb_merk::Merk; -use grovedb_path::SubtreePath; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, StorageBatch, @@ -25,7 +25,7 @@ pub(crate) struct MerkCache<'db, 'b, B: AsRef<[u8]>> { version: &'db GroveVersion, batch: Box, tx: &'db Transaction<'db>, - merks: UnsafeCell, Box<(Cell, TxMerk<'db>)>>>, + merks: UnsafeCell, Box<(Cell, TxMerk<'db>)>>>, } impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { @@ -51,7 +51,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// shall reach end of the scope or to be `drop`ped manually. pub(crate) fn get_merk<'c>( &'c self, - path: SubtreePath<'b, B>, + path: SubtreePathBuilder<'b, B>, ) -> CostResult, Error> { let mut cost = Default::default(); @@ -62,13 +62,13 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { .as_mut() .expect("`UnsafeCell` is never null") } - .entry(path.clone()) + .entry(path) { Entry::Vacant(e) => { let merk = cost_return_on_error!( &mut cost, self.db.open_transactional_merk_at_path( - e.key().clone(), + e.key().into(), self.tx, // SAFETY: batch is allocated on the heap and we use only shared // references, so as long as the `Box` allocation @@ -149,7 +149,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // path's first. while let Some((path, flag_and_merk)) = self.merks.get_mut().pop_first() { let merk = flag_and_merk.1; - if let Some((parent_path, parent_key)) = path.derive_parent() { + if let Some((parent_path, parent_key)) = path.derive_parent_owned() { let mut parent_merk = cost_return_on_error!(&mut cost, self.get_merk(parent_path)); let (root_hash, root_key, sum) = cost_return_on_error!( @@ -221,8 +221,14 @@ mod tests { let cache = MerkCache::new(&db, &tx, version); - cache.get_merk(SubtreePath::empty()).unwrap().unwrap(); - cache.get_merk(SubtreePath::empty()).unwrap().unwrap(); + cache + .get_merk(SubtreePath::empty().derive_owned()) + .unwrap() + .unwrap(); + cache + .get_merk(SubtreePath::empty().derive_owned()) + .unwrap() + .unwrap(); } #[test] @@ -251,7 +257,7 @@ mod tests { let cache = MerkCache::new(&db, &tx, version); - let mut merk = cache.get_merk(path).unwrap().unwrap(); + let mut merk = cache.get_merk(path.derive_owned()).unwrap().unwrap(); item.insert(&mut merk, b"k1", None, &version) .unwrap() diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index b6e00be0..929a9a81 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -13,8 +13,8 @@ use grovedb_version::{ }; use crate::{ - reference_path::path_from_reference_path_type, util::TxRef, Element, Error, GroveDb, - Transaction, TransactionArg, + merk_cache::MerkCache, reference_path::path_from_reference_path_type, util::TxRef, Element, + Error, GroveDb, Transaction, TransactionArg, }; #[derive(Clone)] /// Insert options @@ -70,79 +70,29 @@ impl GroveDb { grove_version.grovedb_versions.operations.insert.insert ); - let subtree_path: SubtreePath = path.into(); - let batch = StorageBatch::new(); - let tx = TxRef::new(&self.db, transaction); - - let collect_costs = self.insert_on_transaction( - subtree_path, - key, - element, - options.unwrap_or_default(), - tx.as_ref(), - &batch, - grove_version, - ); - - collect_costs.flat_map_ok(|_| { - self.db - .commit_multi_context_batch(batch, Some(tx.as_ref())) - .map_err(Into::into) - .map_ok(|_| tx.commit_local()) - .flatten() - }) - } - - fn insert_on_transaction<'db, 'b, B: AsRef<[u8]>>( - &self, - path: SubtreePath<'b, B>, - key: &[u8], - element: Element, - options: InsertOptions, - transaction: &'db Transaction, - batch: &StorageBatch, - grove_version: &GroveVersion, - ) -> CostResult<(), Error> { - check_grovedb_v0_with_cost!( - "insert_on_transaction", - grove_version - .grovedb_versions - .operations - .insert - .insert_on_transaction - ); - let mut cost = OperationCost::default(); + let merk_cache = MerkCache::new(&self, tx.as_ref(), grove_version); - let mut merk_cache: HashMap, Merk> = - HashMap::default(); - - let merk = cost_return_on_error!( + cost_return_on_error!( &mut cost, self.add_element_on_transaction( - path.clone(), + path.into(), key, element, - options, - transaction, - batch, - grove_version - ) - ); - merk_cache.insert(path.clone(), merk); - cost_return_on_error!( - &mut cost, - self.propagate_changes_with_transaction( - merk_cache, - path, - transaction, - batch, + options.unwrap_or_default(), + &merk_cache, grove_version ) ); - Ok(()).wrap_with_cost(cost) + let batch = cost_return_on_error!(&mut cost, merk_cache.into_batch()); + self.db + .commit_multi_context_batch(*batch, Some(tx.as_ref())) + .map_err(Into::into) + .map_ok(|_| tx.commit_local()) + .flatten() + .add_cost(cost) } /// Add subtree to another subtree. @@ -150,14 +100,13 @@ impl GroveDb { /// first make sure other merk exist /// if it exists, then create merk to be inserted, and get root hash /// we only care about root hash of merk to be inserted - fn add_element_on_transaction<'db, B: AsRef<[u8]>>( + fn add_element_on_transaction<'db, 'b, B: AsRef<[u8]>>( &'db self, - path: SubtreePath, + path: SubtreePath<'b, B>, key: &[u8], element: Element, options: InsertOptions, - transaction: &'db Transaction, - batch: &'db StorageBatch, + merk_cache: &MerkCache<'db, 'b, B>, grove_version: &GroveVersion, ) -> CostResult>, Error> { check_grovedb_v0_with_cost!( @@ -171,17 +120,10 @@ impl GroveDb { let mut cost = OperationCost::default(); - let mut subtree_to_insert_into = cost_return_on_error!( - &mut cost, - self.open_transactional_merk_at_path( - path.clone(), - transaction, - Some(batch), - grove_version - ) - ); - // if we don't allow a tree override then we should check + let mut subtree_to_insert_into = + cost_return_on_error!(&mut cost, merk_cache.get_merk(path.derive_owned())); + // if we don't allow a tree override then we should check if options.checks_for_override() { let maybe_element_bytes = cost_return_on_error!( &mut cost, @@ -229,15 +171,16 @@ impl GroveDb { .wrap_with_cost(OperationCost::default()) ); - let referenced_item = cost_return_on_error!( - &mut cost, - self.follow_reference( - reference_path.as_slice().into(), - false, - transaction, - grove_version - ) - ); + let referenced_item: Element = todo!(); + // cost_return_on_error!( + // &mut cost, + // self.follow_reference( + // reference_path.as_slice().into(), + // false, + // transaction, + // grove_version + // ) + // ); if matches!( referenced_item, @@ -295,7 +238,8 @@ impl GroveDb { } } - Ok(subtree_to_insert_into).wrap_with_cost(cost) + // Ok(subtree_to_insert_into).wrap_with_cost(cost) + todo!() } /// Inserts an element at the specified path and key if it does not already diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index f8761aa3..7a8e3081 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -8,7 +8,7 @@ use grovedb_storage::{ use grovedb_version::version::GroveVersion; use grovedb_visualize::DebugByteVectors; -use crate::{Element, Error, Transaction, TransactionArg}; +use crate::{merk_cache::MerkCache, Element, Error, Transaction, TransactionArg}; pub(crate) enum TxRef<'a, 'db: 'a> { Owned(Transaction<'db>), @@ -104,3 +104,6 @@ where .add_cost(cost) } } + +// pub(crate) fn follow_reference<'db, 'b, B>(merk_cache: &MerkCache<'db, 'b, +// B>, path: ) diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index 502a74c9..7864f84e 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -105,6 +105,25 @@ where } } +impl<'bl, 'br, BL, BR> PartialOrd> for SubtreePathBuilder<'bl, BL> +where + BL: AsRef<[u8]>, + BR: AsRef<[u8]>, +{ + fn partial_cmp(&self, other: &SubtreePathBuilder<'br, BR>) -> Option { + let iter_a = self.reverse_iter(); + let iter_b = other.reverse_iter(); + + Some( + iter_a + .len() + .cmp(&iter_b.len()) + .reverse() + .then_with(|| iter_a.cmp(iter_b)), + ) + } +} + impl<'bl, 'br, BL, BR> PartialOrd> for SubtreePath<'bl, BL> where BL: AsRef<[u8]>, @@ -124,6 +143,15 @@ where } } +impl<'bl, BL> Ord for SubtreePathBuilder<'bl, BL> +where + BL: AsRef<[u8]>, +{ + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.partial_cmp(other).expect("order is totally defined") + } +} + impl<'b, B: AsRef<[u8]>> Eq for SubtreePath<'b, B> {} impl<'b, B> From> for SubtreePath<'b, B> { diff --git a/path/src/subtree_path_builder.rs b/path/src/subtree_path_builder.rs index 4ef25f0a..4428e0c2 100644 --- a/path/src/subtree_path_builder.rs +++ b/path/src/subtree_path_builder.rs @@ -149,6 +149,28 @@ impl Default for SubtreePathBuilder<'static, [u8; 0]> { } } +impl SubtreePathBuilder<'static, B> { + /// Makes an owned `SubtreePathBuilder` out of iterator. + pub fn owned_from_iter>(iter: impl IntoIterator) -> Self { + let bytes = iter.into_iter().fold(CompactBytes::new(), |mut bytes, s| { + bytes.add_segment(s.as_ref()); + bytes + }); + + SubtreePathBuilder { + base: SubtreePath { + ref_variant: SubtreePathInner::Slice(&[]), + }, + relative: SubtreePathRelative::Multi(bytes), + } + } + + /// Create an owned version of `SubtreePathBuilder` from `SubtreePath`. + pub fn owned_from_path<'b, S: AsRef<[u8]>>(path: SubtreePath<'b, S>) -> Self { + Self::owned_from_iter(path.to_vec()) + } +} + impl SubtreePathBuilder<'_, B> { /// Returns the length of the subtree path. pub fn len(&self) -> usize { @@ -159,6 +181,24 @@ impl SubtreePathBuilder<'_, B> { pub fn is_empty(&self) -> bool { self.base.is_empty() && self.relative.is_empty() } + + /// Adds path segment in place. + pub fn push_segment(&mut self, segment: &[u8]) { + match &mut self.relative { + SubtreePathRelative::Empty => { + let mut bytes = CompactBytes::new(); + bytes.add_segment(segment); + self.relative = SubtreePathRelative::Multi(bytes); + } + SubtreePathRelative::Single(old_segment) => { + let mut bytes = CompactBytes::new(); + bytes.add_segment(old_segment); + bytes.add_segment(segment); + self.relative = SubtreePathRelative::Multi(bytes); + } + SubtreePathRelative::Multi(bytes) => bytes.add_segment(segment), + } + } } impl<'b, B: AsRef<[u8]>> SubtreePathBuilder<'b, B> { @@ -191,6 +231,37 @@ impl<'b, B: AsRef<[u8]>> SubtreePathBuilder<'b, B> { } } + /// Get a derived path for a parent and a chopped segment. Returned + /// [SubtreePath] will be linked to this [SubtreePath] because it might + /// contain owned data and it has to outlive [SubtreePath]. + pub fn derive_parent_owned(&self) -> Option<(SubtreePathBuilder<'b, B>, Vec)> { + match &self.relative { + SubtreePathRelative::Empty => self + .base + .derive_parent() + .map(|(path, key)| (path.derive_owned(), key.to_vec())), + SubtreePathRelative::Single(relative) => { + Some((self.base.derive_owned(), relative.to_vec())) + } + SubtreePathRelative::Multi(bytes) => { + let mut new_bytes = bytes.clone(); + if let Some(key) = new_bytes.pop_segment() { + Some(( + SubtreePathBuilder { + base: self.base.clone(), + relative: SubtreePathRelative::Multi(new_bytes), + }, + key, + )) + } else { + self.base + .derive_parent() + .map(|(path, key)| (path.derive_owned(), key.to_vec())) + } + } + } + } + /// Get a derived path with a child path segment added. pub fn derive_owned_with_child<'s, S>(&'b self, segment: S) -> SubtreePathBuilder<'b, B> where @@ -203,24 +274,6 @@ impl<'b, B: AsRef<[u8]>> SubtreePathBuilder<'b, B> { } } - /// Adds path segment in place. - pub fn push_segment(&mut self, segment: &[u8]) { - match &mut self.relative { - SubtreePathRelative::Empty => { - let mut bytes = CompactBytes::new(); - bytes.add_segment(segment); - self.relative = SubtreePathRelative::Multi(bytes); - } - SubtreePathRelative::Single(old_segment) => { - let mut bytes = CompactBytes::new(); - bytes.add_segment(old_segment); - bytes.add_segment(segment); - self.relative = SubtreePathRelative::Multi(bytes); - } - SubtreePathRelative::Multi(bytes) => bytes.add_segment(segment), - } - } - /// Returns an iterator for the subtree path by path segments. pub fn reverse_iter(&'b self) -> SubtreePathIter<'b, B> { match &self.relative { diff --git a/path/src/util/compact_bytes.rs b/path/src/util/compact_bytes.rs index 1e4362cb..d4cdba9d 100644 --- a/path/src/util/compact_bytes.rs +++ b/path/src/util/compact_bytes.rs @@ -31,7 +31,7 @@ use std::mem; /// Bytes vector wrapper to have multiple byte arrays allocated continuosuly. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub(crate) struct CompactBytes { n_segments: usize, data: Vec, @@ -64,6 +64,29 @@ impl CompactBytes { pub fn len(&self) -> usize { self.n_segments } + + pub fn pop_segment(&mut self) -> Option> { + if self.n_segments < 1 { + return None; + } + + let length_size = mem::size_of::(); + let last_segment_length = usize::from_ne_bytes( + self.data[self.data.len() - length_size..] + .try_into() + .expect("internal structure bug"), + ); + + let segment = self.data + [self.data.len() - last_segment_length - length_size..self.data.len() - length_size] + .to_vec(); + + self.data + .truncate(self.data.len() - last_segment_length - length_size); + self.n_segments -= 1; + + Some(segment) + } } #[derive(Debug, Clone)] @@ -160,4 +183,25 @@ mod tests { assert_eq!(iter.next(), None); assert_eq!(iter.next(), None); } + + #[test] + fn pop_segment() { + let mut bytes = CompactBytes::default(); + bytes.add_segment(b"ayya"); + bytes.add_segment(b"ayyb"); + bytes.add_segment(b"ayyc"); + bytes.add_segment(b"ayyd"); + + assert_eq!(bytes.pop_segment(), Some(b"ayyd".to_vec())); + assert_eq!(bytes.pop_segment(), Some(b"ayyc".to_vec())); + + let mut v: Vec<_> = bytes.reverse_iter().collect(); + v.reverse(); + assert_eq!(v, vec![b"ayya".to_vec(), b"ayyb".to_vec()]); + + assert_eq!(bytes.pop_segment(), Some(b"ayyb".to_vec())); + assert_eq!(bytes.pop_segment(), Some(b"ayya".to_vec())); + assert_eq!(bytes.pop_segment(), None); + assert_eq!(bytes.pop_segment(), None); + } } From 5ff62ec426ab1bff66b3a9fdec2e066333d69dc2 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Tue, 10 Dec 2024 21:51:01 +0100 Subject: [PATCH 24/27] wip --- grovedb-version/src/lib.rs | 10 +-- grovedb/src/merk_cache.rs | 64 ++++++-------- grovedb/src/operations/insert/mod.rs | 61 ++++++------- grovedb/src/reference_path.rs | 127 ++++++++++++++++++++++++++- grovedb/src/util.rs | 83 +++++++++++++++-- path/src/subtree_path.rs | 2 +- path/src/subtree_path_builder.rs | 15 +++- path/src/util/cow_like.rs | 2 +- 8 files changed, 274 insertions(+), 90 deletions(-) diff --git a/grovedb-version/src/lib.rs b/grovedb-version/src/lib.rs index 48b80a52..ca2bf9b9 100644 --- a/grovedb-version/src/lib.rs +++ b/grovedb-version/src/lib.rs @@ -1,4 +1,4 @@ -use crate::version::GroveVersion; +use version::GroveVersion; pub mod error; pub mod version; @@ -8,7 +8,7 @@ macro_rules! check_grovedb_v0_with_cost { ($method:expr, $version:expr) => {{ const EXPECTED_VERSION: u16 = 0; if $version != EXPECTED_VERSION { - return Err(GroveVersionError::UnknownVersionMismatch { + return Err($crate::error::GroveVersionError::UnknownVersionMismatch { method: $method.to_string(), known_versions: vec![EXPECTED_VERSION], received: $version, @@ -24,7 +24,7 @@ macro_rules! check_grovedb_v0 { ($method:expr, $version:expr) => {{ const EXPECTED_VERSION: u16 = 0; if $version != EXPECTED_VERSION { - return Err(GroveVersionError::UnknownVersionMismatch { + return Err($crate::error::GroveVersionError::UnknownVersionMismatch { method: $method.to_string(), known_versions: vec![EXPECTED_VERSION], received: $version, @@ -39,7 +39,7 @@ macro_rules! check_merk_v0_with_cost { ($method:expr, $version:expr) => {{ const EXPECTED_VERSION: u16 = 0; if $version != EXPECTED_VERSION { - return Err(GroveVersionError::UnknownVersionMismatch { + return Err($crate::error::GroveVersionError::UnknownVersionMismatch { method: $method.to_string(), known_versions: vec![EXPECTED_VERSION], received: $version, @@ -55,7 +55,7 @@ macro_rules! check_merk_v0 { ($method:expr, $version:expr) => {{ const EXPECTED_VERSION: u16 = 0; if $version != EXPECTED_VERSION { - return Err(GroveVersionError::UnknownVersionMismatch { + return Err($crate::error::GroveVersionError::UnknownVersionMismatch { method: $method.to_string(), known_versions: vec![EXPECTED_VERSION], received: $version, diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 0e59ec88..078c7f75 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -22,7 +22,7 @@ type TxMerk<'db> = Merk>; /// Structure to keep subtrees open in memory for repeated access. pub(crate) struct MerkCache<'db, 'b, B: AsRef<[u8]>> { db: &'db GroveDb, - version: &'db GroveVersion, + pub(crate) version: &'db GroveVersion, batch: Box, tx: &'db Transaction<'db>, merks: UnsafeCell, Box<(Cell, TxMerk<'db>)>>>, @@ -44,11 +44,7 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { } } - /// Returns cached Merk reference or opens one if needed. - /// - /// # Panics - /// Borrowing one Merk several times will cause a panic, previous borrow - /// shall reach end of the scope or to be `drop`ped manually. + /// Gets a smart pointer to a cached Merk or opens one if needed. pub(crate) fn get_merk<'c>( &'c self, path: SubtreePathBuilder<'b, B>, @@ -82,15 +78,9 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { self.version ) ); - e.insert(Box::new((true.into(), merk))) - } - Entry::Occupied(e) => { - if e.get().0.get() { - e.into_mut() - } else { - panic!("Double borrow of a cached Merk") - } + e.insert(Box::new((false.into(), merk))) } + Entry::Occupied(e) => e.into_mut(), }; let taken_handle_ref: *const Cell = &boxed_flag_merk.0 as *const _; @@ -158,14 +148,14 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { ); cost_return_on_error!( &mut cost, - GroveDb::update_tree_item_preserve_flag( - &mut parent_merk, + parent_merk.for_merk(|m| GroveDb::update_tree_item_preserve_flag( + m, parent_key, root_key, root_hash, sum, self.version, - ) + )) ); } } @@ -176,27 +166,19 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { /// Wrapper over `Merk` tree to manage unqiue borrow dynamically. pub(crate) struct MerkHandle<'db, 'c> { - merk: &'c mut TxMerk<'db>, + merk: *mut TxMerk<'db>, taken_handle: &'c Cell, } -impl<'db, 'c> Deref for MerkHandle<'db, 'c> { - type Target = TxMerk<'db>; - - fn deref(&self) -> &Self::Target { - self.merk - } -} - -impl<'db, 'c> DerefMut for MerkHandle<'db, 'c> { - fn deref_mut(&mut self) -> &mut Self::Target { - self.merk - } -} - -impl<'db, 'c> Drop for MerkHandle<'db, 'c> { - fn drop(&mut self) { +impl<'db, 'c> MerkHandle<'db, 'c> { + pub(crate) fn for_merk(&mut self, f: impl FnOnce(&mut TxMerk<'db>) -> T) -> T { + if self.taken_handle.get() { + panic!("Attempt to have double &mut borrow on Merk"); + } + self.taken_handle.set(true); + let result = f(unsafe { self.merk.as_mut().expect("pointer to Box cannot be null") }); self.taken_handle.set(false); + result } } @@ -221,14 +203,20 @@ mod tests { let cache = MerkCache::new(&db, &tx, version); - cache + let mut merk1 = cache .get_merk(SubtreePath::empty().derive_owned()) .unwrap() .unwrap(); - cache + let mut merk2 = cache .get_merk(SubtreePath::empty().derive_owned()) .unwrap() .unwrap(); + + merk1.for_merk(|_m1| { + merk2.for_merk(|_m2| { + // this shouldn't happen + }) + }); } #[test] @@ -259,9 +247,7 @@ mod tests { let mut merk = cache.get_merk(path.derive_owned()).unwrap().unwrap(); - item.insert(&mut merk, b"k1", None, &version) - .unwrap() - .unwrap(); + merk.for_merk(|m| item.insert(m, b"k1", None, &version).unwrap().unwrap()); drop(merk); diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index 929a9a81..f6baa932 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -6,15 +6,17 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; use grovedb_merk::{tree::NULL_HASH, Merk, MerkOptions}; -use grovedb_path::SubtreePath; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, Storage, StorageBatch}; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use crate::{ - merk_cache::MerkCache, reference_path::path_from_reference_path_type, util::TxRef, Element, - Error, GroveDb, Transaction, TransactionArg, + merk_cache::{MerkCache, MerkHandle}, + reference_path::path_from_reference_path_type, + util::{self, TxRef}, + Element, Error, GroveDb, Transaction, TransactionArg, }; #[derive(Clone)] /// Insert options @@ -100,15 +102,15 @@ impl GroveDb { /// first make sure other merk exist /// if it exists, then create merk to be inserted, and get root hash /// we only care about root hash of merk to be inserted - fn add_element_on_transaction<'db, 'b, B: AsRef<[u8]>>( + fn add_element_on_transaction<'db, 'b, 'c, B: AsRef<[u8]>>( &'db self, path: SubtreePath<'b, B>, key: &[u8], element: Element, options: InsertOptions, - merk_cache: &MerkCache<'db, 'b, B>, + merk_cache: &'c MerkCache<'db, 'b, B>, grove_version: &GroveVersion, - ) -> CostResult>, Error> { + ) -> CostResult, Error> { check_grovedb_v0_with_cost!( "add_element_on_transaction", grove_version @@ -128,12 +130,12 @@ impl GroveDb { let maybe_element_bytes = cost_return_on_error!( &mut cost, subtree_to_insert_into - .get( + .for_merk(|m| m.get( key, true, Some(&Element::value_defined_cost_for_serialized_value), grove_version, - ) + )) .map_err(|e| Error::CorruptedData(e.to_string())) ); if let Some(element_bytes) = maybe_element_bytes { @@ -164,24 +166,16 @@ impl GroveDb { match element { Element::Reference(ref reference_path, ..) => { - let path = path.to_vec(); // TODO: need for support for references in path library - let reference_path = cost_return_on_error!( + let referenced_item: Element = cost_return_on_error!( &mut cost, - path_from_reference_path_type(reference_path.clone(), &path, Some(key)) - .wrap_with_cost(OperationCost::default()) + util::follow_reference( + merk_cache, + path.derive_owned(), + key, + reference_path.clone() + ) ); - let referenced_item: Element = todo!(); - // cost_return_on_error!( - // &mut cost, - // self.follow_reference( - // reference_path.as_slice().into(), - // false, - // transaction, - // grove_version - // ) - // ); - if matches!( referenced_item, Element::Tree(_, _) | Element::SumTree(_, _, _) @@ -197,13 +191,13 @@ impl GroveDb { cost_return_on_error!( &mut cost, - element.insert_reference( - &mut subtree_to_insert_into, + subtree_to_insert_into.for_merk(|m| element.insert_reference( + m, key, referenced_element_value_hash, Some(options.as_merk_options()), grove_version, - ) + )) ); } Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { @@ -215,31 +209,30 @@ impl GroveDb { } else { cost_return_on_error!( &mut cost, - element.insert_subtree( - &mut subtree_to_insert_into, + subtree_to_insert_into.for_merk(|m| element.insert_subtree( + m, key, NULL_HASH, Some(options.as_merk_options()), grove_version - ) + )) ); } } _ => { cost_return_on_error!( &mut cost, - element.insert( - &mut subtree_to_insert_into, + subtree_to_insert_into.for_merk(|m| element.insert( + m, key, Some(options.as_merk_options()), grove_version - ) + )) ); } } - // Ok(subtree_to_insert_into).wrap_with_cost(cost) - todo!() + Ok(subtree_to_insert_into).wrap_with_cost(cost) } /// Inserts an element at the specified path and key if it does not already diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index 4a489aa9..ddc08a11 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -5,7 +5,7 @@ use std::fmt; use std::iter; use bincode::{Decode, Encode}; -use grovedb_path::SubtreePath; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; #[cfg(feature = "full")] use grovedb_visualize::visualize_to_vec; #[cfg(feature = "full")] @@ -204,6 +204,131 @@ impl ReferencePathType { ) -> Result>, Error> { path_from_reference_path_type(self, current_path, current_key) } + + pub fn absolute_qualified_path<'b, B: AsRef<[u8]>>( + self, + mut current_path: SubtreePathBuilder<'b, B>, + current_key: &[u8], + ) -> Result, Error> { + match self { + ReferencePathType::AbsolutePathReference(path) => { + Ok(SubtreePathBuilder::owned_from_iter(path)) + } + + ReferencePathType::UpstreamRootHeightReference(no_of_elements_to_keep, append_path) => { + let len = current_path.len(); + if no_of_elements_to_keep as usize > len { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + } + let n_to_remove = len - no_of_elements_to_keep as usize; + + let referenced_path = (0..n_to_remove).fold(current_path, |p, _| { + p.derive_parent_owned() + .expect("lenghts were checked above") + .0 + }); + let referenced_path = append_path.into_iter().fold(referenced_path, |mut p, s| { + p.push_segment(&s); + p + }); + + Ok(referenced_path) + } + + ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + no_of_elements_to_keep, + append_path, + ) => { + let len = current_path.len(); + if no_of_elements_to_keep as usize > len || len < 2 { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + } + + let parent_key = current_path + .reverse_iter() + .nth(1) + .expect("lengths were checked above") + .to_vec(); + + let n_to_remove = len - no_of_elements_to_keep as usize; + + let referenced_path = (0..n_to_remove).fold(current_path, |p, _| { + p.derive_parent_owned() + .expect("lenghts were checked above") + .0 + }); + let mut referenced_path = + append_path.into_iter().fold(referenced_path, |mut p, s| { + p.push_segment(&s); + p + }); + referenced_path.push_segment(&parent_key); + + Ok(referenced_path) + } + + // Discard the last n elements from current path, append new path to subpath + ReferencePathType::UpstreamFromElementHeightReference( + no_of_elements_to_discard_from_end, + append_path, + ) => { + let mut referenced_path = current_path; + for _ in 0..no_of_elements_to_discard_from_end { + if let Some((path, _)) = referenced_path.derive_parent_owned() { + referenced_path = path; + } else { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + } + } + + let referenced_path = append_path.into_iter().fold(referenced_path, |mut p, s| { + p.push_segment(&s); + p + }); + + Ok(referenced_path) + } + + ReferencePathType::CousinReference(cousin_key) => { + let Some((mut referred_path, _)) = current_path.derive_parent_owned() else { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + }; + + referred_path.push_segment(&cousin_key); + referred_path.push_segment(current_key); + + Ok(referred_path) + } + + ReferencePathType::RemovedCousinReference(cousin_path) => { + let Some((mut referred_path, _)) = current_path.derive_parent_owned() else { + return Err(Error::InvalidInput( + "reference stored path cannot satisfy reference constraints", + )); + }; + + cousin_path + .into_iter() + .for_each(|s| referred_path.push_segment(&s)); + referred_path.push_segment(current_key); + + Ok(referred_path) + } + + ReferencePathType::SiblingReference(sibling_key) => { + current_path.push_segment(&sibling_key); + Ok(current_path) + } + } + } } #[cfg(any(feature = "full", feature = "visualize"))] diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 7a8e3081..1ef9b605 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -1,14 +1,21 @@ -use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use std::collections::HashSet; + +use grovedb_costs::{ + cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, +}; use grovedb_merk::Merk; -use grovedb_path::SubtreePath; +use grovedb_path::{SubtreePath, SubtreePathBuilder}; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, Storage, StorageBatch, }; -use grovedb_version::version::GroveVersion; +use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; use grovedb_visualize::DebugByteVectors; -use crate::{merk_cache::MerkCache, Element, Error, Transaction, TransactionArg}; +use crate::{ + merk_cache::MerkCache, operations::MAX_REFERENCE_HOPS, reference_path::ReferencePathType, + Element, Error, Transaction, TransactionArg, +}; pub(crate) enum TxRef<'a, 'db: 'a> { Owned(Transaction<'db>), @@ -105,5 +112,69 @@ where } } -// pub(crate) fn follow_reference<'db, 'b, B>(merk_cache: &MerkCache<'db, 'b, -// B>, path: ) +pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( + merk_cache: &'c MerkCache<'db, 'b, B>, + path: SubtreePathBuilder<'b, B>, + key: &[u8], + ref_path: ReferencePathType, +) -> CostResult { + check_grovedb_v0_with_cost!( + "follow_reference", + merk_cache + .version + .grovedb_versions + .operations + .get + .follow_reference + ); + + let mut cost = OperationCost::default(); + + let mut hops_left = MAX_REFERENCE_HOPS; + let mut visited = HashSet::new(); + + let mut qualified_path = path.clone(); + qualified_path.push_segment(key); + + visited.insert(qualified_path); + + let mut current_path = path; + let mut current_key = key.to_vec(); + let mut current_ref = ref_path; + + while hops_left > 0 { + let referred_qualified_path = cost_return_on_error_no_add!( + cost, + current_ref.absolute_qualified_path(current_path, ¤t_key) + ); + + if !visited.insert(referred_qualified_path.clone()) { + return Err(Error::CyclicReference).wrap_with_cost(cost); + } + + let Some((referred_path, referred_key)) = referred_qualified_path.derive_parent_owned() + else { + return Err(Error::InvalidCodeExecution("empty reference")).wrap_with_cost(cost); + }; + + let mut referred_merk = + cost_return_on_error!(&mut cost, merk_cache.get_merk(referred_path.clone())); + let element = cost_return_on_error!( + &mut cost, + referred_merk + .for_merk(|m| { Element::get(m, &referred_key, true, merk_cache.version) }) + ); + + match element { + Element::Reference(ref_path, ..) | Element::BidirectionalReference(ref_path, ..) => { + current_path = referred_path; + current_key = referred_key; + current_ref = ref_path; + hops_left -= 1; + } + e => return Ok(e).wrap_with_cost(cost), + } + } + + Err(Error::ReferenceLimit).wrap_with_cost(cost) +} diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index 7864f84e..9fd50160 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -248,7 +248,7 @@ impl<'b, B: AsRef<[u8]>> SubtreePath<'b, B> { } /// Get a derived path with a child path segment added. - pub fn derive_owned_with_child<'s, S>(&'b self, segment: S) -> SubtreePathBuilder<'b, B> + pub fn derive_owned_with_child<'s, S>(&self, segment: S) -> SubtreePathBuilder<'b, B> where S: Into>, 's: 'b, diff --git a/path/src/subtree_path_builder.rs b/path/src/subtree_path_builder.rs index 4428e0c2..6ac77977 100644 --- a/path/src/subtree_path_builder.rs +++ b/path/src/subtree_path_builder.rs @@ -46,6 +46,15 @@ pub struct SubtreePathBuilder<'b, B> { pub(crate) relative: SubtreePathRelative<'b>, } +impl<'b, B> Clone for SubtreePathBuilder<'b, B> { + fn clone(&self) -> Self { + SubtreePathBuilder { + base: self.base.clone(), + relative: self.relative.clone(), + } + } +} + /// Hash order is the same as iteration order: from most deep path segment up to /// root. impl<'b, B: AsRef<[u8]>> Hash for SubtreePathBuilder<'b, B> { @@ -97,7 +106,7 @@ impl<'s, 'b, B> From<&'s SubtreePath<'b, B>> for SubtreePathBuilder<'b, B> { } /// Derived subtree path on top of base path. -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) enum SubtreePathRelative<'r> { /// Equivalent to the base path. Empty, @@ -149,7 +158,7 @@ impl Default for SubtreePathBuilder<'static, [u8; 0]> { } } -impl SubtreePathBuilder<'static, B> { +impl<'b, B> SubtreePathBuilder<'b, B> { /// Makes an owned `SubtreePathBuilder` out of iterator. pub fn owned_from_iter>(iter: impl IntoIterator) -> Self { let bytes = iter.into_iter().fold(CompactBytes::new(), |mut bytes, s| { @@ -166,7 +175,7 @@ impl SubtreePathBuilder<'static, B> { } /// Create an owned version of `SubtreePathBuilder` from `SubtreePath`. - pub fn owned_from_path<'b, S: AsRef<[u8]>>(path: SubtreePath<'b, S>) -> Self { + pub fn owned_from_path<'a, S: AsRef<[u8]>>(path: SubtreePath<'a, S>) -> Self { Self::owned_from_iter(path.to_vec()) } } diff --git a/path/src/util/cow_like.rs b/path/src/util/cow_like.rs index 78608ec8..02a53537 100644 --- a/path/src/util/cow_like.rs +++ b/path/src/util/cow_like.rs @@ -35,7 +35,7 @@ use std::{ /// A smart pointer that follows the semantics of [Cow](std::borrow::Cow) except /// provides no means for mutability and thus doesn't require [Clone]. -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum CowLike<'b> { Owned(Vec), Borrowed(&'b [u8]), From f4e25c021c2439fb0ad53f40d0aa2c3886e96481 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 11 Dec 2024 09:48:24 +0100 Subject: [PATCH 25/27] fix test --- grovedb/src/merk_cache.rs | 51 +++++++++++++++++++++++---------------- grovedb/src/tests/mod.rs | 2 -- grovedb/src/util.rs | 4 +++ 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/grovedb/src/merk_cache.rs b/grovedb/src/merk_cache.rs index 078c7f75..dfdba777 100644 --- a/grovedb/src/merk_cache.rs +++ b/grovedb/src/merk_cache.rs @@ -3,19 +3,15 @@ use std::{ cell::{Cell, UnsafeCell}, collections::{btree_map::Entry, BTreeMap}, - ops::{Deref, DerefMut}, }; -use grovedb_costs::{cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt}; +use grovedb_costs::{cost_return_on_error, CostResult, CostsExt}; use grovedb_merk::Merk; -use grovedb_path::{SubtreePath, SubtreePathBuilder}; -use grovedb_storage::{ - rocksdb_storage::{PrefixedRocksDbTransactionContext, RocksDbStorage}, - StorageBatch, -}; +use grovedb_path::SubtreePathBuilder; +use grovedb_storage::{rocksdb_storage::PrefixedRocksDbTransactionContext, StorageBatch}; use grovedb_version::version::GroveVersion; -use crate::{Element, Error, GroveDb, Transaction}; +use crate::{Error, GroveDb, Transaction}; type TxMerk<'db> = Merk>; @@ -52,6 +48,10 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { let mut cost = Default::default(); // SAFETY: there are no other references to `merks` memory at the same time. + // Note while it's possible to have direct references to actual Merk trees, + // outside of the scope of this function, this map (`merks`) has + // indirect connection to them through `Box`, thus there are no overlapping + // references, and that is requirement of `UnsafeCell` we have there. let boxed_flag_merk = match unsafe { self.merks .get() @@ -91,8 +91,8 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // references plus borrowing rules aren't violated (one `&mut` or many // `&` with no `&mut` at a time). // - // To make sure changes to the internal structure won't affect existing borrows - // we have an indirection in a form of `Box`, that allows us to move and update + // To make sure changes to the map won't affect existing borrows we have an + // indirection in a form of `Box`, that allows us to move and update // `MerkCache` with new subtrees and possible reallocations without breaking // `MerkHandle`'s references. We use `UnsafeCell` to connect lifetimes and check // in compile time that `MerkHandle`s won't outlive the cache, even though we @@ -100,18 +100,18 @@ impl<'db, 'b, B: AsRef<[u8]>> MerkCache<'db, 'b, B> { // exclusive for the whole time of `MerkHandle`, so it shall go intially through // a shared reference. // - // Borrowing rules are covered slightly more complicated way: - // 1. Of a pair behind heap allocation only Merk is uniquely borrowed by - // `MerkHandle` - // 2. Borrow flag is referenced by `MerkHandle` to be updated on `Drop` and is - // referenced while taking a new `MerkHandle` to check if it was already - // borrowed, that gives us two shared references to the same memory and - // that's allowed, note we're not referring to the Merk part of the pair - // 3. Borrow flag's reference points to a heap allocated memory and will remain - // valid just as the Merk reference to the memory right after the flag + // Borrowing rules are covered using a borrow flag of each Merk: + // 1. Borrow flag's reference points to a heap allocated memory and will remain + // valid. Since the reference is shared and no need to obtain a `&mut` + // reference this part of the memory is covered. + // 2. For the same reason the Merk's pointer can be converted to a reference, + // because the memory behind the `Box` is valid and `MerkHandle` can't + // outlive it since we use lifetime parameters. + // 3. We can get unique reference out of that pointer safely because of + // borrowing flag. Ok(unsafe { MerkHandle { - merk: merk_ptr.as_mut().expect("`Box` contents are never null"), + merk: merk_ptr, taken_handle: taken_handle_ref .as_ref() .expect("`Box` contents are never null"), @@ -175,9 +175,18 @@ impl<'db, 'c> MerkHandle<'db, 'c> { if self.taken_handle.get() { panic!("Attempt to have double &mut borrow on Merk"); } + self.taken_handle.set(true); - let result = f(unsafe { self.merk.as_mut().expect("pointer to Box cannot be null") }); + + // SAFETY: here we want to have `&mut` reference to Merk out of a pointer, there + // is a checklist for that: + // 1. Memory is valid, because `MerkHandle` can't outlive `MerkCache` and heap + // allocated Merks stay at their place for the whole `MerkCache` lifetime. + // 2. No other references exist because of `taken_handle` check above. + let result = f(unsafe { self.merk.as_mut().expect("`Box` contents are never null") }); + self.taken_handle.set(false); + result } } diff --git a/grovedb/src/tests/mod.rs b/grovedb/src/tests/mod.rs index 568bc6a8..d177566b 100644 --- a/grovedb/src/tests/mod.rs +++ b/grovedb/src/tests/mod.rs @@ -1190,8 +1190,6 @@ mod tests { ) .unwrap(); - dbg!(&result); - assert!(matches!( result, Err(Error::CorruptedReferencePathKeyNotFound(_)) diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 1ef9b605..b6e0756d 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -163,6 +163,10 @@ pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( &mut cost, referred_merk .for_merk(|m| { Element::get(m, &referred_key, true, merk_cache.version) }) + .map_err(|e| match e { + Error::PathKeyNotFound(s) => Error::CorruptedReferencePathKeyNotFound(s), + e => e, + }) ); match element { From f1214f3b78caf31deba064985223968eab395b55 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 11 Dec 2024 12:34:03 +0100 Subject: [PATCH 26/27] add ref resolver tests --- grovedb/src/reference_path.rs | 100 +++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index ddc08a11..01ee1a44 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -205,6 +205,7 @@ impl ReferencePathType { path_from_reference_path_type(self, current_path, current_key) } + /// TODO: deprecate the rest pub fn absolute_qualified_path<'b, B: AsRef<[u8]>>( self, mut current_path: SubtreePathBuilder<'b, B>, @@ -242,7 +243,7 @@ impl ReferencePathType { append_path, ) => { let len = current_path.len(); - if no_of_elements_to_keep as usize > len || len < 2 { + if no_of_elements_to_keep as usize > len || len < 1 { return Err(Error::InvalidInput( "reference stored path cannot satisfy reference constraints", )); @@ -250,7 +251,7 @@ impl ReferencePathType { let parent_key = current_path .reverse_iter() - .nth(1) + .next() .expect("lengths were checked above") .to_vec(); @@ -520,7 +521,7 @@ impl ReferencePathType { #[cfg(test)] mod tests { use grovedb_merk::proofs::Query; - use grovedb_path::SubtreePath; + use grovedb_path::{SubtreePath, SubtreePathBuilder}; use grovedb_version::version::GroveVersion; use crate::{ @@ -542,6 +543,20 @@ mod tests { ); } + #[test] + fn test_upstream_root_height_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]); + // selects the first 2 elements from the stored path and appends the new path. + let ref1 = + ReferencePathType::UpstreamRootHeightReference(2, vec![b"c".to_vec(), b"d".to_vec()]); + let final_path = ref1.absolute_qualified_path(stored_path, b"").unwrap(); + assert_eq!( + final_path.to_vec(), + vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec(), b"d".to_vec()] + ); + } + #[test] fn test_upstream_root_height_with_parent_addition_reference() { let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; @@ -563,6 +578,28 @@ mod tests { ); } + #[test] + fn test_upstream_root_height_with_parent_addition_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]); + // selects the first 2 elements from the stored path and appends the new path. + let ref1 = ReferencePathType::UpstreamRootHeightWithParentPathAdditionReference( + 2, + vec![b"c".to_vec(), b"d".to_vec()], + ); + let final_path = ref1.absolute_qualified_path(stored_path, b"").unwrap(); + assert_eq!( + final_path.to_vec(), + vec![ + b"a".to_vec(), + b"b".to_vec(), + b"c".to_vec(), + b"d".to_vec(), + b"m".to_vec() + ] + ); + } + #[test] fn test_upstream_from_element_height_reference() { let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; @@ -578,6 +615,22 @@ mod tests { ); } + #[test] + fn test_upstream_from_element_height_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]); + // discards the last element from the stored_path + let ref1 = ReferencePathType::UpstreamFromElementHeightReference( + 1, + vec![b"c".to_vec(), b"d".to_vec()], + ); + let final_path = ref1.absolute_qualified_path(stored_path, b"").unwrap(); + assert_eq!( + final_path.to_vec(), + vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec(), b"d".to_vec()] + ); + } + #[test] fn test_cousin_reference_no_key() { let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; @@ -600,6 +653,20 @@ mod tests { ); } + #[test] + fn test_cousin_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref()]); + let key = b"m".as_ref(); + // Replaces the immediate parent (in this case b) with the given key (c) + let ref1 = ReferencePathType::CousinReference(b"c".to_vec()); + let final_path = ref1.absolute_qualified_path(stored_path, key).unwrap(); + assert_eq!( + final_path.to_vec(), + vec![b"a".to_vec(), b"c".to_vec(), b"m".to_vec()] + ); + } + #[test] fn test_removed_cousin_reference_no_key() { let stored_path = vec![b"a".as_ref(), b"b".as_ref(), b"m".as_ref()]; @@ -622,6 +689,20 @@ mod tests { ); } + #[test] + fn test_removed_cousin_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref()]); + let key = b"m".as_ref(); + // Replaces the immediate parent (in this case b) with the given key (c) + let ref1 = ReferencePathType::RemovedCousinReference(vec![b"c".to_vec(), b"d".to_vec()]); + let final_path = ref1.absolute_qualified_path(stored_path, key).unwrap(); + assert_eq!( + final_path.to_vec(), + vec![b"a".to_vec(), b"c".to_vec(), b"d".to_vec(), b"m".to_vec()] + ); + } + #[test] fn test_sibling_reference() { let stored_path = vec![b"a".as_ref(), b"b".as_ref()]; @@ -634,6 +715,19 @@ mod tests { ); } + #[test] + fn test_sibling_reference_path_lib() { + let stored_path: SubtreePathBuilder<&[u8]> = + SubtreePathBuilder::owned_from_iter([b"a".as_ref(), b"b".as_ref()]); + let key = b"m".as_ref(); + let ref1 = ReferencePathType::SiblingReference(b"c".to_vec()); + let final_path = ref1.absolute_qualified_path(stored_path, key).unwrap(); + assert_eq!( + final_path.to_vec(), + vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec()] + ); + } + #[test] fn test_query_many_with_different_reference_types() { let grove_version = GroveVersion::latest(); From f38203801d23edd86b80972311ee812f7c1ba0b1 Mon Sep 17 00:00:00 2001 From: Evgeny Fomin Date: Wed, 11 Dec 2024 21:54:10 +0100 Subject: [PATCH 27/27] wip --- grovedb/src/element/mod.rs | 138 +++++++++++++++------------ grovedb/src/element/serialize.rs | 115 ++++++++++++---------- grovedb/src/error.rs | 4 + grovedb/src/operations/insert/mod.rs | 99 ++++++++++++++++++- grovedb/src/reference_path.rs | 2 +- grovedb/src/util.rs | 21 +++- path/Cargo.toml | 2 + path/src/subtree_path.rs | 29 ++++++ 8 files changed, 297 insertions(+), 113 deletions(-) diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index bbbc6b47..4ddad67c 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -64,10 +64,6 @@ pub const SUM_TREE_COST_SIZE: u32 = SUM_LAYER_COST_SIZE; // 12 /// int 64 sum value pub type SumValue = i64; -#[cfg(any(feature = "full", feature = "verify"))] -/// if the item is deleted or updated should we cascade -pub type CascadeOnUpdate = bool; - #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// @@ -92,23 +88,27 @@ pub enum Element { /// A reference to an object by its path BidirectionalReference( ReferencePathType, - Option<(ReferencePathType, CascadeOnUpdate)>, + Option, MaxReferenceHop, Option, ), /// An ordinary value that has a backwards reference - ItemWithBackwardsReferences( - Vec, - Vec<(ReferencePathType, CascadeOnUpdate)>, - Option, - ), + ItemWithBackwardsReferences(Vec, Vec, Option), /// Signed integer value that can be totaled in a sum tree that has a /// backwards reference - SumItemWithBackwardsReferences( - SumValue, - Vec<(ReferencePathType, CascadeOnUpdate)>, - Option, - ), + SumItemWithBackwardsReferences(SumValue, Vec, Option), +} + +/// A reference that points to an existing `Element::BidirectionalReference` +/// with a flag indicating whether to issue deletion of that reference chain if +/// the current `Element` no longer exists as a variant with backward reference +/// support or has been deleted completely. +#[derive(Clone, Encode, Decode, PartialEq, Eq, Hash)] +#[cfg_attr(not(any(feature = "full", feature = "visualize")), derive(Debug))] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct BackwardReference { + pub reference_path: ReferencePathType, + pub cascade_delete: bool, } impl fmt::Display for Element { @@ -183,34 +183,44 @@ impl fmt::Display for Element { let backwards_references_info = if backwards_references_count < 4 { backwards_references .iter() - .map(|(backwards_reference, cascade_on_update)| { - format!( - "{}:{}", - backwards_reference, - if *cascade_on_update { - "cascade" - } else { - "no cascade" - } - ) - }) + .map( + |BackwardReference { + reference_path, + cascade_delete, + }| { + format!( + "{}:{}", + reference_path, + if *cascade_delete { + "cascade" + } else { + "no cascade" + } + ) + }, + ) .collect::>() .join("|") } else { let first_backwards_references = backwards_references .iter() .take(3) - .map(|(backwards_reference, cascade_on_update)| { - format!( - "{}:{}", - backwards_reference, - if *cascade_on_update { - "cascade" - } else { - "no cascade" - } - ) - }) + .map( + |BackwardReference { + reference_path, + cascade_delete, + }| { + format!( + "{}:{}", + reference_path, + if *cascade_delete { + "cascade" + } else { + "no cascade" + } + ) + }, + ) .collect::>() .join("|"); format!( @@ -233,34 +243,44 @@ impl fmt::Display for Element { let backwards_references_info = if backwards_references_count < 4 { backwards_references .iter() - .map(|(backwards_reference, cascade_on_update)| { - format!( - "{}:{}", - backwards_reference, - if *cascade_on_update { - "cascade" - } else { - "no cascade" - } - ) - }) + .map( + |BackwardReference { + reference_path, + cascade_delete, + }| { + format!( + "{}:{}", + reference_path, + if *cascade_delete { + "cascade" + } else { + "no cascade" + } + ) + }, + ) .collect::>() .join("|") } else { let first_backwards_references = backwards_references .iter() .take(3) - .map(|(backwards_reference, cascade_on_update)| { - format!( - "{}:{}", - backwards_reference, - if *cascade_on_update { - "cascade" - } else { - "no cascade" - } - ) - }) + .map( + |BackwardReference { + reference_path, + cascade_delete, + }| { + format!( + "{}:{}", + reference_path, + if *cascade_delete { + "cascade" + } else { + "no cascade" + } + ) + }, + ) .collect::>() .join("|"); format!( diff --git a/grovedb/src/element/serialize.rs b/grovedb/src/element/serialize.rs index 24114ca2..41bd2239 100644 --- a/grovedb/src/element/serialize.rs +++ b/grovedb/src/element/serialize.rs @@ -2,7 +2,7 @@ //! Implements serialization functions in Element use bincode::config; -use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; +use grovedb_version::{check_grovedb_v0, version::GroveVersion}; #[cfg(any(feature = "full", feature = "verify"))] use crate::{Element, Error}; @@ -65,6 +65,71 @@ impl Element { } } +/// Wrapper type to alter [Element]'s bincode serialization for variants with +/// backward references, practically ignoring that part. +struct SerializeElementData<'a>(&'a Element); + +impl bincode::Encode for SerializeElementData<'_> { + fn encode( + &self, + encoder: &mut E, + ) -> Result<(), bincode::error::EncodeError> { + match self.0 { + Element::Item(field_0, field_1) => { + ::encode(&(0u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_1, encoder)?; + Ok(()) + } + Element::Reference(field_0, field_1, field_2) => { + ::encode(&(1u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_1, encoder)?; + ::bincode::Encode::encode(field_2, encoder)?; + Ok(()) + } + Element::Tree(field_0, field_1) => { + ::encode(&(2u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_1, encoder)?; + Ok(()) + } + Element::SumItem(field_0, field_1) => { + ::encode(&(3u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_1, encoder)?; + Ok(()) + } + Element::SumTree(field_0, field_1, field_2) => { + ::encode(&(4u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_1, encoder)?; + ::bincode::Encode::encode(field_2, encoder)?; + Ok(()) + } + Element::BidirectionalReference(field_0, _, field_2, field_3) => { + ::encode(&(5u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_2, encoder)?; + ::bincode::Encode::encode(field_3, encoder)?; + Ok(()) + } + Element::ItemWithBackwardsReferences(field_0, _, field_2) => { + ::encode(&(6u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_2, encoder)?; + Ok(()) + } + Element::SumItemWithBackwardsReferences(field_0, _, field_2) => { + ::encode(&(7u32), encoder)?; + ::bincode::Encode::encode(field_0, encoder)?; + ::bincode::Encode::encode(field_2, encoder)?; + Ok(()) + } + } + } +} + #[cfg(feature = "full")] #[cfg(test)] mod tests { @@ -192,51 +257,3 @@ mod tests { ); } } - -/// Wrapper type to alter [Element]'s bincode serialization for variants with -/// backward references, practically ignoring that part. -struct SerializeElementData<'a>(&'a Element); - -impl bincode::Encode for SerializeElementData<'_> { - fn encode( - &self, - encoder: &mut E, - ) -> Result<(), bincode::error::EncodeError> { - match self.0 { - Element::Item(data, flags) | Element::ItemWithBackwardsReferences(data, _, flags) => { - ::encode(&(0u32), encoder)?; - bincode::Encode::encode(data, encoder)?; - bincode::Encode::encode(flags, encoder)?; - Ok(()) - } - Element::Reference(ref_path, max_hops, flags) - | Element::BidirectionalReference(ref_path, _, max_hops, flags) => { - ::encode(&(1u32), encoder)?; - bincode::Encode::encode(ref_path, encoder)?; - bincode::Encode::encode(max_hops, encoder)?; - bincode::Encode::encode(flags, encoder)?; - Ok(()) - } - Element::Tree(root_key, flags) => { - ::encode(&(2u32), encoder)?; - bincode::Encode::encode(root_key, encoder)?; - bincode::Encode::encode(flags, encoder)?; - Ok(()) - } - Element::SumItem(sum_value, flags) - | Element::SumItemWithBackwardsReferences(sum_value, _, flags) => { - ::encode(&(3u32), encoder)?; - bincode::Encode::encode(sum_value, encoder)?; - bincode::Encode::encode(flags, encoder)?; - Ok(()) - } - Element::SumTree(root_key, sum_value, flags) => { - ::encode(&(4u32), encoder)?; - bincode::Encode::encode(root_key, encoder)?; - bincode::Encode::encode(sum_value, encoder)?; - bincode::Encode::encode(flags, encoder)?; - Ok(()) - } - } - } -} diff --git a/grovedb/src/error.rs b/grovedb/src/error.rs index 4446f4ab..6775b74e 100644 --- a/grovedb/src/error.rs +++ b/grovedb/src/error.rs @@ -53,6 +53,10 @@ pub enum Error { #[error("corrupted referenced path key not found: {0}")] CorruptedReferencePathParentLayerNotFound(String), + /// Bidirectional references rule was violated + #[error("bidirectional reference rule violation: {0}")] + BidirectionalReferenceRule(String), + /// The invalid parent layer path represents a logical error from the client /// library #[error("invalid parent layer path: {0}")] diff --git a/grovedb/src/operations/insert/mod.rs b/grovedb/src/operations/insert/mod.rs index f6baa932..589c0490 100644 --- a/grovedb/src/operations/insert/mod.rs +++ b/grovedb/src/operations/insert/mod.rs @@ -13,6 +13,7 @@ use grovedb_version::{ }; use crate::{ + element::BackwardReference, merk_cache::{MerkCache, MerkHandle}, reference_path::path_from_reference_path_type, util::{self, TxRef}, @@ -174,7 +175,8 @@ impl GroveDb { key, reference_path.clone() ) - ); + ) + .element; if matches!( referenced_item, @@ -200,6 +202,101 @@ impl GroveDb { )) ); } + + Element::BidirectionalReference(ref reference_path, ..) => { + let mut resolved_reference = cost_return_on_error!( + &mut cost, + util::follow_reference( + merk_cache, + path.derive_owned(), + key, + reference_path.clone() + ) + ); + let mut referenced_item: Element = resolved_reference.element; + + match &mut referenced_item { + Element::ItemWithBackwardsReferences(_, backward_references, _) + | Element::SumItemWithBackwardsReferences(_, backward_references, _) => { + backward_references.push(BackwardReference { + reference_path: cost_return_on_error_no_add!( + cost, + reference_path.invert(path, key).ok_or_else(|| { + Error::CorruptedReferencePathParentLayerNotFound( + "unable to get inverted reference".to_owned(), + ) + }) + ), + cascade_delete: true, + }); + } + Element::BidirectionalReference(_, r @ None, ..) => { + *r = Some(BackwardReference { + reference_path: cost_return_on_error_no_add!( + cost, + reference_path.invert(path, key).ok_or_else(|| { + Error::CorruptedReferencePathParentLayerNotFound( + "unable to get inverted reference".to_owned(), + ) + }) + ), + cascade_delete: true, + }); + } + Element::BidirectionalReference( + _, + Some(BackwardReference { reference_path, .. }), + .., + ) => { + let existing_reference_origin = reference_path + .clone() + .absolute_qualified_path(path.derive_owned(), key); + return Err(Error::BidirectionalReferenceRule(format!( + "only one bidirectional reference to a bidirectional reference at a \ + time is supported, existing reference is located at: {}", + existing_reference_origin + .map(|p| format!("{}", Into::>::into(&p))) + .unwrap_or_else(|_| "unreachable".to_owned()) + ))) + .wrap_with_cost(cost); + } + _ => { + return Err(Error::BidirectionalReferenceRule( + "Bidirectional references can only point variants with backward \ + references support" + .to_owned(), + )) + .wrap_with_cost(cost) + } + } + + let referenced_element_value_hash = + cost_return_on_error!(&mut cost, referenced_item.value_hash(grove_version)); + + // TODO: update pointed-to item with new backward reference + cost_return_on_error!( + &mut cost, + resolved_reference + .merk + .for_merk(|m| -> CostResult<(), Error> { + Ok(()).wrap_with_cost(Default::default()) + }) + ); + + // TODO: receive old item, if it was a bidi reference, then update old + // pointed-to item's list of backward references as well + cost_return_on_error!( + &mut cost, + subtree_to_insert_into.for_merk(|m| element.insert_reference( + m, + key, + referenced_element_value_hash, + Some(options.as_merk_options()), + grove_version, + )) + ); + } + Element::Tree(ref value, _) | Element::SumTree(ref value, ..) => { if value.is_some() { return Err(Error::InvalidCodeExecution( diff --git a/grovedb/src/reference_path.rs b/grovedb/src/reference_path.rs index 01ee1a44..e6fba7a2 100644 --- a/grovedb/src/reference_path.rs +++ b/grovedb/src/reference_path.rs @@ -63,7 +63,7 @@ pub enum ReferencePathType { impl ReferencePathType { /// Get an inverted reference - fn invert>(&self, path: SubtreePath, key: &[u8]) -> Option { + pub(crate) fn invert>(&self, path: SubtreePath, key: &[u8]) -> Option { Some(match self { // Absolute path shall point to a fully qualified path of the reference's origin ReferencePathType::AbsolutePathReference(_) => { diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index b6e0756d..81f37c9f 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -13,7 +13,9 @@ use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; use grovedb_visualize::DebugByteVectors; use crate::{ - merk_cache::MerkCache, operations::MAX_REFERENCE_HOPS, reference_path::ReferencePathType, + merk_cache::{MerkCache, MerkHandle}, + operations::MAX_REFERENCE_HOPS, + reference_path::ReferencePathType, Element, Error, Transaction, TransactionArg, }; @@ -112,12 +114,18 @@ where } } +pub(crate) struct ResolvedReference<'db, 'c> { + pub merk: MerkHandle<'db, 'c>, + pub key: Vec, + pub element: Element, +} + pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( merk_cache: &'c MerkCache<'db, 'b, B>, path: SubtreePathBuilder<'b, B>, key: &[u8], ref_path: ReferencePathType, -) -> CostResult { +) -> CostResult, Error> { check_grovedb_v0_with_cost!( "follow_reference", merk_cache @@ -176,7 +184,14 @@ pub(crate) fn follow_reference<'db, 'b, 'c, B: AsRef<[u8]>>( current_ref = ref_path; hops_left -= 1; } - e => return Ok(e).wrap_with_cost(cost), + e => { + return Ok(ResolvedReference { + merk: referred_merk, + key: referred_key, + element: e, + }) + .wrap_with_cost(cost) + } } } diff --git a/path/Cargo.toml b/path/Cargo.toml index 92417664..0d4ab183 100644 --- a/path/Cargo.toml +++ b/path/Cargo.toml @@ -9,3 +9,5 @@ documentation = "https://docs.rs/grovedb-path" repository = "https://github.com/dashpay/grovedb" [dependencies] +hex = "0.4.3" +itertools = "0.13.0" diff --git a/path/src/subtree_path.rs b/path/src/subtree_path.rs index 9fd50160..b754b91e 100644 --- a/path/src/subtree_path.rs +++ b/path/src/subtree_path.rs @@ -36,6 +36,7 @@ use std::{ cmp, + fmt::{self, Display}, hash::{Hash, Hasher}, }; @@ -51,6 +52,34 @@ pub struct SubtreePath<'b, B> { pub(crate) ref_variant: SubtreePathInner<'b, B>, } +impl<'b, B: AsRef<[u8]>> Display for SubtreePath<'b, B> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let path = self.to_vec(); + + fn fmt_segment(s: impl AsRef<[u8]>) -> String { + let bytes = s.as_ref(); + let hex_str = hex::encode(bytes); + let utf8_str = String::from_utf8(bytes.to_vec()); + let mut result = format!("h:{hex_str}"); + if let Ok(s) = utf8_str { + result.push_str("/s:"); + result.push_str(&s); + } + result + } + + f.write_str("[")?; + + for s in itertools::intersperse(path.into_iter().map(fmt_segment), ", ".to_owned()) { + f.write_str(&s)?; + } + + f.write_str("]")?; + + Ok(()) + } +} + /// Wrapped inner representation of subtree path ref. #[derive(Debug)] pub(crate) enum SubtreePathInner<'b, B> {