diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 0484cfa1..e4f3547e 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -89,7 +89,7 @@ pub fn util_path_to_string(path: &[Vec]) -> Vec { // Splits the given global chunk id into [SUBTREE_PREFIX:CHUNK_ID] pub fn util_split_global_chunk_id( global_chunk_id: &[u8], -) -> Result<(crate::SubtreePrefix, String), Error> { +) -> Result<(crate::SubtreePrefix, Vec), Error> { let chunk_prefix_length: usize = 32; if global_chunk_id.len() < chunk_prefix_length { return Err(Error::CorruptedData( @@ -101,13 +101,7 @@ pub fn util_split_global_chunk_id( let mut array = [0u8; 32]; array.copy_from_slice(chunk_prefix); let chunk_prefix_key: crate::SubtreePrefix = array; - let str_chunk_id = String::from_utf8(chunk_id.to_vec()); - match str_chunk_id { - Ok(s) => Ok((chunk_prefix_key, s)), - Err(_) => Err(Error::CorruptedData( - "unable to convert chunk id to string".to_string(), - )), - } + Ok((chunk_prefix_key, chunk_id.to_vec())) } #[cfg(feature = "full")] @@ -244,20 +238,15 @@ impl GroveDb { let chunk_producer_res = ChunkProducer::new(&merk); match chunk_producer_res { - Ok(mut chunk_producer) => match std::str::from_utf8(chunk_id) { - Ok(chunk_id_str) => { - let chunk_res = chunk_producer.chunk(chunk_id_str); - match chunk_res { - Ok((chunk, _)) => Ok(chunk), - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - } + Ok(mut chunk_producer) => { + let chunk_res = chunk_producer.chunk(chunk_id); + match chunk_res { + Ok((chunk, _)) => Ok(chunk), + Err(_) => Err(Error::CorruptedData( + "Unable to create to load chunk".to_string(), + )), } - Err(_) => Err(Error::CorruptedData( - "Unable to process chunk id".to_string(), - )), - }, + } Err(_) => Err(Error::CorruptedData( "Unable to create Chunk producer".to_string(), )), @@ -274,20 +263,15 @@ impl GroveDb { let chunk_producer_res = ChunkProducer::new(&merk); match chunk_producer_res { - Ok(mut chunk_producer) => match std::str::from_utf8(chunk_id) { - Ok(chunk_id_str) => { - let chunk_res = chunk_producer.chunk(chunk_id_str); - match chunk_res { - Ok((chunk, _)) => Ok(chunk), - Err(_) => Err(Error::CorruptedData( - "Unable to create to load chunk".to_string(), - )), - } + Ok(mut chunk_producer) => { + let chunk_res = chunk_producer.chunk(chunk_id); + match chunk_res { + Ok((chunk, _)) => Ok(chunk), + Err(_) => Err(Error::CorruptedData( + "Unable to create to load chunk".to_string(), + )), } - Err(_) => Err(Error::CorruptedData( - "Unable to process chunk id".to_string(), - )), - }, + } Err(_) => Err(Error::CorruptedData( "Unable to create Chunk producer".to_string(), )), @@ -380,12 +364,12 @@ impl GroveDb { } state_sync_info.pending_chunks.remove(global_chunk_id); if !chunk_data.is_empty() { - match restorer.process_chunk(chunk_id.to_string(), chunk_data) { + match restorer.process_chunk(&chunk_id, chunk_data) { Ok(next_chunk_ids) => { state_sync_info.num_processed_chunks += 1; for next_chunk_id in next_chunk_ids { let mut next_global_chunk_id = chunk_prefix.to_vec(); - next_global_chunk_id.extend(next_chunk_id.as_bytes().to_vec()); + next_global_chunk_id.extend(next_chunk_id.to_vec()); state_sync_info .pending_chunks .insert(next_global_chunk_id.clone()); diff --git a/merk/src/merk/chunks.rs b/merk/src/merk/chunks.rs index 8f840f91..f6b1b64c 100644 --- a/merk/src/merk/chunks.rs +++ b/merk/src/merk/chunks.rs @@ -38,10 +38,10 @@ use crate::{ chunk_op::ChunkOp, error::ChunkError, util::{ - chunk_height, chunk_id_from_traversal_instruction, - chunk_id_from_traversal_instruction_with_recovery, generate_traversal_instruction, - generate_traversal_instruction_as_string, number_of_chunks, - string_as_traversal_instruction, + chunk_height, chunk_index_from_traversal_instruction, + chunk_index_from_traversal_instruction_with_recovery, + generate_traversal_instruction, generate_traversal_instruction_as_vec_bytes, + number_of_chunks, vec_bytes_as_traversal_instruction, }, }, Node, Op, @@ -72,14 +72,14 @@ impl SubtreeChunk { #[derive(Debug)] pub struct MultiChunk { pub chunk: Vec, - pub next_index: Option, + pub next_index: Option>, pub remaining_limit: Option, } impl MultiChunk { pub fn new( chunk: Vec, - next_index: Option, + next_index: Option>, remaining_limit: Option, ) -> Self { Self { @@ -131,17 +131,17 @@ where } /// Returns the chunk at a given chunk id. - pub fn chunk(&mut self, chunk_id: &str) -> Result<(Vec, Option), Error> { - let traversal_instructions = string_as_traversal_instruction(chunk_id)?; - let chunk_index = chunk_id_from_traversal_instruction_with_recovery( + pub fn chunk(&mut self, chunk_id: &[u8]) -> Result<(Vec, Option>), Error> { + let traversal_instructions = vec_bytes_as_traversal_instruction(chunk_id)?; + let chunk_index = chunk_index_from_traversal_instruction_with_recovery( traversal_instructions.as_slice(), self.height, )?; let (chunk, next_index) = self.chunk_internal(chunk_index, traversal_instructions)?; - let index_string = next_index - .map(|index| generate_traversal_instruction_as_string(self.height, index)) + let next_chunk_id = next_index + .map(|index| generate_traversal_instruction_as_vec_bytes(self.height, index)) .transpose()?; - Ok((chunk, index_string)) + Ok((chunk, next_chunk_id)) } /// Returns the chunk at the given index @@ -186,12 +186,12 @@ where /// chunks or hit some optional limit pub fn multi_chunk_with_limit( &mut self, - chunk_id: &str, + chunk_id: &[u8], limit: Option, ) -> Result { // we want to convert the chunk id to the index - let chunk_index = string_as_traversal_instruction(chunk_id).and_then(|instruction| { - chunk_id_from_traversal_instruction(instruction.as_slice(), self.height) + let chunk_index = vec_bytes_as_traversal_instruction(chunk_id).and_then(|instruction| { + chunk_index_from_traversal_instruction(instruction.as_slice(), self.height) })?; self.multi_chunk_with_limit_and_index(chunk_index, limit) } @@ -267,11 +267,11 @@ where current_limit = subtree_multi_chunk.remaining_limit; } - let index_string = current_index - .map(|index| generate_traversal_instruction_as_string(self.height, index)) + let index_bytes = current_index + .map(|index| generate_traversal_instruction_as_vec_bytes(self.height, index)) .transpose()?; - Ok(MultiChunk::new(chunk, index_string, current_limit)) + Ok(MultiChunk::new(chunk, index_bytes, current_limit)) } /// Packs as many chunks as it can from a starting chunk index, into a @@ -371,7 +371,7 @@ where /// optimizing throughput compared to random access. // TODO: this is not better than random access, as we are not keeping state // that will make this more efficient, decide if this should be fixed or not - fn next_chunk(&mut self) -> Option, Option), Error>> { + fn next_chunk(&mut self) -> Option, Option>), Error>> { let max_index = number_of_chunks(self.height); if self.index > max_index { return None; @@ -383,7 +383,9 @@ where self.chunk_with_index(self.index) .and_then(|(chunk, chunk_index)| { chunk_index - .map(|index| generate_traversal_instruction_as_string(self.height, index)) + .map(|index| { + generate_traversal_instruction_as_vec_bytes(self.height, index) + }) .transpose() .map(|v| (chunk, v)) }), @@ -396,7 +398,7 @@ impl<'db, S> Iterator for ChunkProducer<'db, S> where S: StorageContext<'db>, { - type Item = Result<(Vec, Option), Error>; + type Item = Result<(Vec, Option>), Error>; fn next(&mut self) -> Option { self.next_chunk() @@ -424,7 +426,7 @@ mod test { tests::{traverse_get_kv_feature_type, traverse_get_node_hash}, LEFT, RIGHT, }, - util::traversal_instruction_as_string, + util::traversal_instruction_as_vec_bytes, }, tree::execute, Tree, @@ -1027,7 +1029,7 @@ mod test { // ensure that the remaining limit, next index and values given are correct // if limit is smaller than first chunk, we should get an error - let chunk_result = chunk_producer.multi_chunk_with_limit("", Some(5)); + let chunk_result = chunk_producer.multi_chunk_with_limit(vec![].as_slice(), Some(5)); assert!(matches!( chunk_result, Err(Error::ChunkingError(ChunkError::LimitTooSmall(..))) @@ -1052,7 +1054,7 @@ mod test { .expect("should generate chunk"); assert_eq!( chunk_result.next_index, - Some(traversal_instruction_as_string( + Some(traversal_instruction_as_vec_bytes( &generate_traversal_instruction(4, 4).unwrap() )) ); diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 94b99add..0a0b805e 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -63,7 +63,7 @@ use crate::{ proofs::{ chunk::{ chunk::{LEFT, RIGHT}, - util::traversal_instruction_as_string, + util::traversal_instruction_as_vec_bytes, }, query::query_item::QueryItem, Query, @@ -556,11 +556,11 @@ where pub fn verify( &self, skip_sum_checks: bool, - ) -> (BTreeMap, BTreeMap>) { + ) -> (BTreeMap, CryptoHash>, BTreeMap, Vec>) { let tree = self.tree.take(); - let mut bad_link_map: BTreeMap = BTreeMap::new(); - let mut parent_keys: BTreeMap> = BTreeMap::new(); + let mut bad_link_map: BTreeMap, CryptoHash> = BTreeMap::new(); + let mut parent_keys: BTreeMap, Vec> = BTreeMap::new(); let mut root_traversal_instruction = vec![]; // TODO: remove clone @@ -581,8 +581,8 @@ where &self, tree: &TreeNode, traversal_instruction: &mut Vec, - bad_link_map: &mut BTreeMap, - parent_keys: &mut BTreeMap>, + bad_link_map: &mut BTreeMap, CryptoHash>, + parent_keys: &mut BTreeMap, Vec>, skip_sum_checks: bool, ) { if let Some(link) = tree.link(LEFT) { @@ -617,8 +617,8 @@ where link: &Link, parent_key: &[u8], traversal_instruction: &mut Vec, - bad_link_map: &mut BTreeMap, - parent_keys: &mut BTreeMap>, + bad_link_map: &mut BTreeMap, CryptoHash>, + parent_keys: &mut BTreeMap, Vec>, skip_sum_checks: bool, ) { let (hash, key, sum) = match link { @@ -639,7 +639,7 @@ where _ => todo!(), }; - let instruction_id = traversal_instruction_as_string(traversal_instruction); + let instruction_id = traversal_instruction_as_vec_bytes(traversal_instruction); let node = TreeNode::get( &self.storage, key, @@ -648,29 +648,29 @@ where .unwrap(); if node.is_err() { - bad_link_map.insert(instruction_id.clone(), hash); - parent_keys.insert(instruction_id, parent_key.to_vec()); + bad_link_map.insert(instruction_id.to_vec(), hash); + parent_keys.insert(instruction_id.to_vec(), parent_key.to_vec()); return; } let node = node.unwrap(); if node.is_none() { - bad_link_map.insert(instruction_id.clone(), hash); - parent_keys.insert(instruction_id, parent_key.to_vec()); + bad_link_map.insert(instruction_id.to_vec(), hash); + parent_keys.insert(instruction_id.to_vec(), parent_key.to_vec()); return; } let node = node.unwrap(); if node.hash().unwrap() != hash { - bad_link_map.insert(instruction_id.clone(), hash); - parent_keys.insert(instruction_id, parent_key.to_vec()); + bad_link_map.insert(instruction_id.to_vec(), hash); + parent_keys.insert(instruction_id.to_vec(), parent_key.to_vec()); return; } // Need to skip this when restoring a sum tree if !skip_sum_checks && node.sum().unwrap() != sum { - bad_link_map.insert(instruction_id.clone(), hash); - parent_keys.insert(instruction_id, parent_key.to_vec()); + bad_link_map.insert(instruction_id.to_vec(), hash); + parent_keys.insert(instruction_id.to_vec(), parent_key.to_vec()); return; } diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index e2439f5c..9e26b1af 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -41,7 +41,7 @@ use crate::{ chunk::{LEFT, RIGHT}, chunk_op::ChunkOp, error::{ChunkError, ChunkError::InternalError}, - util::{string_as_traversal_instruction, traversal_instruction_as_string}, + util::{traversal_instruction_as_vec_bytes, vec_bytes_as_traversal_instruction}, }, tree::{execute, Child, Tree as ProofTree}, Node, Op, @@ -57,10 +57,10 @@ use crate::{ /// already. pub struct Restorer { merk: Merk, - chunk_id_to_root_hash: BTreeMap, + chunk_id_to_root_hash: BTreeMap, CryptoHash>, parent_key_value_hash: Option, // this is used to keep track of parents whose links need to be rewritten - parent_keys: BTreeMap>, + parent_keys: BTreeMap, Vec>, } impl<'db, S: StorageContext<'db>> Restorer { @@ -72,7 +72,7 @@ impl<'db, S: StorageContext<'db>> Restorer { parent_key_value_hash: Option, ) -> Self { let mut chunk_id_to_root_hash = BTreeMap::new(); - chunk_id_to_root_hash.insert(traversal_instruction_as_string(&[]), expected_root_hash); + chunk_id_to_root_hash.insert(traversal_instruction_as_vec_bytes(&[]), expected_root_hash); Self { merk, chunk_id_to_root_hash, @@ -81,17 +81,16 @@ impl<'db, S: StorageContext<'db>> Restorer { } } - // TODO: consider converting chunk id to a vec /// Processes a chunk at some chunk id, returns the chunks id's of chunks /// that can be requested pub fn process_chunk( &mut self, - chunk_id: String, + chunk_id: &[u8], chunk: Vec, - ) -> Result, Error> { + ) -> Result>, Error> { let expected_root_hash = self .chunk_id_to_root_hash - .get(&chunk_id) + .get(chunk_id) .ok_or(Error::ChunkRestoringError(ChunkError::UnexpectedChunk))?; let mut parent_key_value_hash: Option = None; @@ -100,14 +99,14 @@ impl<'db, S: StorageContext<'db>> Restorer { } let chunk_tree = Self::verify_chunk(chunk, expected_root_hash, &parent_key_value_hash)?; - let mut root_traversal_instruction = string_as_traversal_instruction(&chunk_id)?; + let mut root_traversal_instruction = vec_bytes_as_traversal_instruction(chunk_id)?; if root_traversal_instruction.is_empty() { let _ = self.merk.set_base_root_key(Some(chunk_tree.key().to_vec())); } else { // every non root chunk has some associated parent with an placeholder link // here we update the placeholder link to represent the true data - self.rewrite_parent_link(&chunk_id, &root_traversal_instruction, &chunk_tree)?; + self.rewrite_parent_link(chunk_id, &root_traversal_instruction, &chunk_tree)?; } // next up, we need to write the chunk and build the map again @@ -115,7 +114,7 @@ impl<'db, S: StorageContext<'db>> Restorer { if chunk_write_result.is_ok() { // if we were able to successfully write the chunk, we can remove // the chunk expected root hash from our chunk id map - self.chunk_id_to_root_hash.remove(&chunk_id); + self.chunk_id_to_root_hash.remove(chunk_id); } chunk_write_result @@ -123,10 +122,13 @@ impl<'db, S: StorageContext<'db>> Restorer { /// Process multi chunks (space optimized chunk proofs that can contain /// multiple singular chunks) - pub fn process_multi_chunk(&mut self, multi_chunk: Vec) -> Result, Error> { + pub fn process_multi_chunk( + &mut self, + multi_chunk: Vec, + ) -> Result>, Error> { let mut expect_chunk_id = true; let mut chunk_ids = vec![]; - let mut current_chunk_id: String = "".to_string(); + let mut current_chunk_id = vec![]; for chunk_op in multi_chunk { if (matches!(chunk_op, ChunkOp::ChunkId(..)) && !expect_chunk_id) @@ -138,11 +140,11 @@ impl<'db, S: StorageContext<'db>> Restorer { } match chunk_op { ChunkOp::ChunkId(instructions) => { - current_chunk_id = traversal_instruction_as_string(&instructions); + current_chunk_id = traversal_instruction_as_vec_bytes(&instructions); } ChunkOp::Chunk(chunk) => { // TODO: remove clone - let next_chunk_ids = self.process_chunk(current_chunk_id.clone(), chunk)?; + let next_chunk_ids = self.process_chunk(¤t_chunk_id, chunk)?; chunk_ids.extend(next_chunk_ids); } } @@ -210,7 +212,7 @@ impl<'db, S: StorageContext<'db>> Restorer { &mut self, chunk_tree: ProofTree, traversal_instruction: &mut Vec, - ) -> Result, Error> { + ) -> Result>, Error> { // this contains all the elements we want to write to storage let mut batch = self.merk.storage.new_batch(); let mut new_chunk_ids = Vec::new(); @@ -242,9 +244,10 @@ impl<'db, S: StorageContext<'db>> Restorer { Node::Hash(hash) => { // the node hash points to the root of another chunk // we get the chunk id and add the hash to restorer state - let chunk_id = traversal_instruction_as_string(node_traversal_instruction); - new_chunk_ids.push(chunk_id.clone()); - self.chunk_id_to_root_hash.insert(chunk_id.clone(), *hash); + let chunk_id = + traversal_instruction_as_vec_bytes(node_traversal_instruction); + new_chunk_ids.push(chunk_id.to_vec()); + self.chunk_id_to_root_hash.insert(chunk_id.to_vec(), *hash); // TODO: handle unwrap self.parent_keys .insert(chunk_id, parent_key.unwrap().to_owned()); @@ -276,7 +279,7 @@ impl<'db, S: StorageContext<'db>> Restorer { /// we need to update the parent link to reflect the correct data. fn rewrite_parent_link( &mut self, - chunk_id: &str, + chunk_id: &[u8], traversal_instruction: &[bool], chunk_tree: &ProofTree, ) -> Result<(), Error> { @@ -663,7 +666,7 @@ mod tests { // initial restorer state should contain just the root hash of the source merk assert_eq!(restorer.chunk_id_to_root_hash.len(), 1); assert_eq!( - restorer.chunk_id_to_root_hash.get(""), + restorer.chunk_id_to_root_hash.get(vec![].as_slice()), Some(merk.root_hash().unwrap()).as_ref() ); @@ -671,7 +674,10 @@ mod tests { let (chunk, _) = chunk_producer.chunk_with_index(1).unwrap(); // apply first chunk let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![]), chunk) + .process_chunk( + &traversal_instruction_as_vec_bytes(vec![].as_slice()), + chunk, + ) .expect("should process chunk successfully"); assert_eq!(new_chunk_ids.len(), 4); @@ -680,22 +686,22 @@ mod tests { assert_eq!(restorer.chunk_id_to_root_hash.len(), 4); // assert all the chunk hash values assert_eq!( - restorer.chunk_id_to_root_hash.get("11"), + restorer.chunk_id_to_root_hash.get(vec![1, 1].as_slice()), Some(get_node_hash(traverse_get_node_hash(&mut tree_walker, &[LEFT, LEFT])).unwrap()) .as_ref() ); assert_eq!( - restorer.chunk_id_to_root_hash.get("10"), + restorer.chunk_id_to_root_hash.get(vec![1, 0].as_slice()), Some(get_node_hash(traverse_get_node_hash(&mut tree_walker, &[LEFT, RIGHT])).unwrap()) .as_ref() ); assert_eq!( - restorer.chunk_id_to_root_hash.get("01"), + restorer.chunk_id_to_root_hash.get(vec![0, 1].as_slice()), Some(get_node_hash(traverse_get_node_hash(&mut tree_walker, &[RIGHT, LEFT])).unwrap()) .as_ref() ); assert_eq!( - restorer.chunk_id_to_root_hash.get("00"), + restorer.chunk_id_to_root_hash.get(vec![0, 0].as_slice()), Some(get_node_hash(traverse_get_node_hash(&mut tree_walker, &[RIGHT, RIGHT])).unwrap()) .as_ref() ); @@ -704,18 +710,26 @@ mod tests { let (chunk, _) = chunk_producer.chunk_with_index(2).unwrap(); // apply second chunk let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![LEFT, LEFT]), chunk) + .process_chunk( + &traversal_instruction_as_vec_bytes(&vec![LEFT, LEFT]), + chunk, + ) .unwrap(); assert_eq!(new_chunk_ids.len(), 0); // chunk_map should have 1 less element assert_eq!(restorer.chunk_id_to_root_hash.len(), 3); - assert_eq!(restorer.chunk_id_to_root_hash.get("11"), None); + assert_eq!( + restorer.chunk_id_to_root_hash.get(vec![1, 1].as_slice()), + None + ); // let's try to apply the second chunk again, should not work let (chunk, _) = chunk_producer.chunk_with_index(2).unwrap(); // apply second chunk - let chunk_process_result = - restorer.process_chunk(traversal_instruction_as_string(&vec![LEFT, LEFT]), chunk); + let chunk_process_result = restorer.process_chunk( + &traversal_instruction_as_vec_bytes(&vec![LEFT, LEFT]), + chunk, + ); assert!(chunk_process_result.is_err()); assert!(matches!( chunk_process_result, @@ -725,8 +739,10 @@ mod tests { // next let's get a random but expected chunk and work with that e.g. chunk 4 // but let's apply it to the wrong place let (chunk, _) = chunk_producer.chunk_with_index(4).unwrap(); - let chunk_process_result = - restorer.process_chunk(traversal_instruction_as_string(&vec![LEFT, RIGHT]), chunk); + let chunk_process_result = restorer.process_chunk( + &traversal_instruction_as_vec_bytes(&vec![LEFT, RIGHT]), + chunk, + ); assert!(chunk_process_result.is_err()); assert!(matches!( chunk_process_result, @@ -739,34 +755,52 @@ mod tests { let (chunk, _) = chunk_producer.chunk_with_index(5).unwrap(); // apply second chunk let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![RIGHT, RIGHT]), chunk) + .process_chunk( + &traversal_instruction_as_vec_bytes(&vec![RIGHT, RIGHT]), + chunk, + ) .unwrap(); assert_eq!(new_chunk_ids.len(), 0); // chunk_map should have 1 less element assert_eq!(restorer.chunk_id_to_root_hash.len(), 2); - assert_eq!(restorer.chunk_id_to_root_hash.get("00"), None); + assert_eq!( + restorer.chunk_id_to_root_hash.get(vec![0, 0].as_slice()), + None + ); // correctly apply chunk 3 let (chunk, _) = chunk_producer.chunk_with_index(3).unwrap(); // apply second chunk let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![LEFT, RIGHT]), chunk) + .process_chunk( + &traversal_instruction_as_vec_bytes(&vec![LEFT, RIGHT]), + chunk, + ) .unwrap(); assert_eq!(new_chunk_ids.len(), 0); // chunk_map should have 1 less element assert_eq!(restorer.chunk_id_to_root_hash.len(), 1); - assert_eq!(restorer.chunk_id_to_root_hash.get("10"), None); + assert_eq!( + restorer.chunk_id_to_root_hash.get(vec![1, 0].as_slice()), + None + ); // correctly apply chunk 4 let (chunk, _) = chunk_producer.chunk_with_index(4).unwrap(); // apply second chunk let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![RIGHT, LEFT]), chunk) + .process_chunk( + &traversal_instruction_as_vec_bytes(&vec![RIGHT, LEFT]), + chunk, + ) .unwrap(); assert_eq!(new_chunk_ids.len(), 0); // chunk_map should have 1 less element assert_eq!(restorer.chunk_id_to_root_hash.len(), 0); - assert_eq!(restorer.chunk_id_to_root_hash.get("01"), None); + assert_eq!( + restorer.chunk_id_to_root_hash.get(vec![0, 1].as_slice()), + None + ); // finalize merk let restored_merk = restorer.finalize().expect("should finalized successfully"); @@ -861,13 +895,11 @@ mod tests { let mut restorer = Restorer::new(restoration_merk, source_merk.root_hash().unwrap(), None); // perform chunk production and processing - let mut chunk_id_opt = Some("".to_string()); + let mut chunk_id_opt = Some(vec![]); while let Some(chunk_id) = chunk_id_opt { - let (chunk, next_chunk_id) = chunk_producer - .chunk(chunk_id.as_str()) - .expect("should get chunk"); + let (chunk, next_chunk_id) = chunk_producer.chunk(&chunk_id).expect("should get chunk"); restorer - .process_chunk(chunk_id.to_string(), chunk) + .process_chunk(&chunk_id, chunk) .expect("should process chunk successfully"); chunk_id_opt = next_chunk_id; } @@ -931,13 +963,13 @@ mod tests { assert_eq!(restorer.chunk_id_to_root_hash.len(), 1); assert_eq!( - restorer.chunk_id_to_root_hash.get(""), + restorer.chunk_id_to_root_hash.get(vec![].as_slice()), Some(merk.root_hash().unwrap()).as_ref() ); // generate multi chunk from root with no limit let chunk = chunk_producer - .multi_chunk_with_limit("", None) + .multi_chunk_with_limit(vec![].as_slice(), None) .expect("should generate multichunk"); assert_eq!(chunk.chunk.len(), 2); @@ -996,14 +1028,14 @@ mod tests { assert_eq!(restorer.chunk_id_to_root_hash.len(), 1); assert_eq!( - restorer.chunk_id_to_root_hash.get(""), + restorer.chunk_id_to_root_hash.get(vec![].as_slice()), Some(merk.root_hash().unwrap()).as_ref() ); // first restore the first chunk let (chunk, next_chunk_index) = chunk_producer.chunk_with_index(1).unwrap(); let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![]), chunk) + .process_chunk(&traversal_instruction_as_vec_bytes(&vec![]), chunk) .expect("should process chunk"); assert_eq!(new_chunk_ids.len(), 4); assert_eq!(next_chunk_index, Some(2)); @@ -1068,12 +1100,12 @@ mod tests { // build multi chunk with with limit of 325 let multi_chunk = chunk_producer - .multi_chunk_with_limit("", Some(600)) + .multi_chunk_with_limit(vec![].as_slice(), Some(600)) .unwrap(); // should only contain the first chunk assert_eq!(multi_chunk.chunk.len(), 2); // should point to chunk 2 - assert_eq!(multi_chunk.next_index, Some("11".to_string())); + assert_eq!(multi_chunk.next_index, Some(vec![1, 1])); let next_ids = restorer.process_multi_chunk(multi_chunk.chunk).unwrap(); assert_eq!(next_ids.len(), 4); assert_eq!(restorer.chunk_id_to_root_hash.len(), 4); @@ -1083,10 +1115,10 @@ mod tests { // with limit just above 642 should get 2 chunks (2 and 3) // disjoint, so multi chunk len should be 4 let multi_chunk = chunk_producer - .multi_chunk_with_limit(multi_chunk.next_index.unwrap().as_str(), Some(645)) + .multi_chunk_with_limit(multi_chunk.next_index.unwrap().as_slice(), Some(645)) .unwrap(); assert_eq!(multi_chunk.chunk.len(), 4); - assert_eq!(multi_chunk.next_index, Some("01".to_string())); + assert_eq!(multi_chunk.next_index, Some(vec![0u8, 1u8])); let next_ids = restorer.process_multi_chunk(multi_chunk.chunk).unwrap(); // chunks 2 and 3 are leaf chunks assert_eq!(next_ids.len(), 0); @@ -1095,7 +1127,7 @@ mod tests { // get the last 2 chunks let multi_chunk = chunk_producer - .multi_chunk_with_limit(multi_chunk.next_index.unwrap().as_str(), Some(645)) + .multi_chunk_with_limit(multi_chunk.next_index.unwrap().as_slice(), Some(645)) .unwrap(); assert_eq!(multi_chunk.chunk.len(), 4); assert_eq!(multi_chunk.next_index, None); @@ -1155,10 +1187,10 @@ mod tests { let mut restorer = Restorer::new(restoration_merk, source_merk.root_hash().unwrap(), None); // perform chunk production and processing - let mut chunk_id_opt = Some("".to_string()); + let mut chunk_id_opt = Some(vec![]); while let Some(chunk_id) = chunk_id_opt { let multi_chunk = chunk_producer - .multi_chunk_with_limit(chunk_id.as_str(), limit) + .multi_chunk_with_limit(&chunk_id, limit) .expect("should get chunk"); restorer .process_multi_chunk(multi_chunk.chunk) @@ -1234,14 +1266,14 @@ mod tests { assert_eq!(restorer.chunk_id_to_root_hash.len(), 1); assert_eq!( - restorer.chunk_id_to_root_hash.get(""), + restorer.chunk_id_to_root_hash.get(vec![].as_slice()), Some(merk.root_hash().unwrap()).as_ref() ); // first restore the first chunk let (chunk, next_chunk_index) = chunk_producer.chunk_with_index(1).unwrap(); let new_chunk_ids = restorer - .process_chunk(traversal_instruction_as_string(&vec![]), chunk) + .process_chunk(&traversal_instruction_as_vec_bytes(&vec![]), chunk) .expect("should process chunk"); assert_eq!(new_chunk_ids.len(), 4); assert_eq!(next_chunk_index, Some(2)); diff --git a/merk/src/proofs/chunk/util.rs b/merk/src/proofs/chunk/util.rs index 2f64ba8d..39c513b7 100644 --- a/merk/src/proofs/chunk/util.rs +++ b/merk/src/proofs/chunk/util.rs @@ -170,14 +170,17 @@ fn exit_node_count(height: usize) -> usize { 2_usize.pow(height as u32) } -/// Generate instruction for traversing to a given chunk in a binary tree -pub fn generate_traversal_instruction(height: usize, chunk_id: usize) -> Result, Error> { +/// Generate instruction for traversing to a given chunk index in a binary tree +pub fn generate_traversal_instruction( + height: usize, + chunk_index: usize, +) -> Result, Error> { let mut instructions = vec![]; let total_chunk_count = number_of_chunks(height); // out of bounds - if chunk_id < 1 || chunk_id > total_chunk_count { + if chunk_index < 1 || chunk_index > total_chunk_count { return Err(Error::ChunkingError(ChunkError::OutOfBounds( "chunk id out of bounds", ))); @@ -202,7 +205,7 @@ pub fn generate_traversal_instruction(height: usize, chunk_id: usize) -> Result< // checks if we last decision we made got us to the desired chunk id let advance_result = chunk_range.advance_range_start().unwrap(); chunk_range = advance_result.0; - if advance_result.1 == chunk_id { + if advance_result.1 == chunk_index { return Ok(instructions); } } else { @@ -211,7 +214,7 @@ pub fn generate_traversal_instruction(height: usize, chunk_id: usize) -> Result< // we first check which half the desired chunk is // then follow that path let chunk_id_half = chunk_range - .which_half(chunk_id) + .which_half(chunk_index) .expect("chunk id must exist in range"); instructions.push(chunk_id_half); chunk_range = chunk_range @@ -226,9 +229,9 @@ pub fn generate_traversal_instruction(height: usize, chunk_id: usize) -> Result< Ok(instructions) } -/// Determine the chunk id given the traversal instruction and the max height of -/// the tree -pub fn chunk_id_from_traversal_instruction( +/// Determine the chunk index given the traversal instruction and the max height +/// of the tree +pub fn chunk_index_from_traversal_instruction( traversal_instruction: &[bool], height: usize, ) -> Result { @@ -238,7 +241,7 @@ pub fn chunk_id_from_traversal_instruction( } let mut chunk_count = number_of_chunks(height); - let mut current_chunk_id = 1; + let mut current_chunk_index = 1; let mut layer_heights = chunk_height_per_layer(height); let last_layer_height = layer_heights.pop().expect("confirmed not empty"); @@ -301,62 +304,64 @@ pub fn chunk_id_from_traversal_instruction( chunk_count /= exit_node_count(layer_height); - current_chunk_id = current_chunk_id + offset_multiplier * chunk_count + 1; + current_chunk_index = current_chunk_index + offset_multiplier * chunk_count + 1; start_index = end_index; } - Ok(current_chunk_id) + Ok(current_chunk_index) } -/// Determine the chunk id given the traversal instruction and the max height of -/// the tree. This can recover from traversal instructions not pointing to a +/// Determine the chunk index given the traversal instruction and the max height +/// of the tree. This can recover from traversal instructions not pointing to a /// chunk boundary, in such a case, it backtracks until it hits a chunk /// boundary. -pub fn chunk_id_from_traversal_instruction_with_recovery( +pub fn chunk_index_from_traversal_instruction_with_recovery( traversal_instruction: &[bool], height: usize, ) -> Result { - let chunk_id_result = chunk_id_from_traversal_instruction(traversal_instruction, height); - if chunk_id_result.is_err() { - return chunk_id_from_traversal_instruction_with_recovery( + let chunk_index_result = chunk_index_from_traversal_instruction(traversal_instruction, height); + if chunk_index_result.is_err() { + return chunk_index_from_traversal_instruction_with_recovery( &traversal_instruction[0..traversal_instruction.len() - 1], height, ); } - chunk_id_result + chunk_index_result } -/// Generate instruction for traversing to a given chunk in a binary tree, -/// returns string representation -pub fn generate_traversal_instruction_as_string( +/// Generate instruction for traversing to a given chunk index in a binary tree, +/// returns vec bytes representation +pub fn generate_traversal_instruction_as_vec_bytes( height: usize, - chunk_id: usize, -) -> Result { - let instruction = generate_traversal_instruction(height, chunk_id)?; - Ok(traversal_instruction_as_string(&instruction)) + chunk_index: usize, +) -> Result, Error> { + let instruction = generate_traversal_instruction(height, chunk_index)?; + Ok(traversal_instruction_as_vec_bytes(&instruction)) } -/// Convert traversal instruction to byte string +/// Convert traversal instruction to bytes vec /// 1 represents left (true) /// 0 represents right (false) -pub fn traversal_instruction_as_string(instruction: &[bool]) -> String { +pub fn traversal_instruction_as_vec_bytes(instruction: &[bool]) -> Vec { instruction .iter() - .map(|v| if *v { "1" } else { "0" }) + .map(|v| if *v { 1u8 } else { 0u8 }) .collect() } -/// Converts a string that represents a traversal instruction +/// Converts a vec bytes that represents a traversal instruction /// to a vec of bool, true = left and false = right -pub fn string_as_traversal_instruction(instruction_string: &str) -> Result, Error> { - instruction_string - .chars() - .map(|char| match char { - '1' => Ok(LEFT), - '0' => Ok(RIGHT), +pub fn vec_bytes_as_traversal_instruction( + instruction_vec_bytes: &[u8], +) -> Result, Error> { + instruction_vec_bytes + .iter() + .map(|byte| match byte { + 1u8 => Ok(LEFT), + 0u8 => Ok(RIGHT), _ => Err(Error::ChunkingError(ChunkError::BadTraversalInstruction( - "failed to parse instruction string", + "failed to parse instruction vec bytes", ))), }) .collect() @@ -568,26 +573,32 @@ mod test { #[test] fn test_traversal_instruction_as_string() { - assert_eq!(traversal_instruction_as_string(&vec![]), ""); - assert_eq!(traversal_instruction_as_string(&vec![LEFT]), "1"); - assert_eq!(traversal_instruction_as_string(&vec![RIGHT]), "0"); + assert_eq!(traversal_instruction_as_vec_bytes(&vec![]), vec![]); + assert_eq!(traversal_instruction_as_vec_bytes(&vec![LEFT]), vec![1u8]); + assert_eq!(traversal_instruction_as_vec_bytes(&vec![RIGHT]), vec![0u8]); assert_eq!( - traversal_instruction_as_string(&vec![RIGHT, LEFT, LEFT, RIGHT]), - "0110" + traversal_instruction_as_vec_bytes(&vec![RIGHT, LEFT, LEFT, RIGHT]), + vec![0u8, 1u8, 1u8, 0u8] ); } #[test] fn test_instruction_string_to_traversal_instruction() { - assert_eq!(string_as_traversal_instruction("1").unwrap(), vec![LEFT]); - assert_eq!(string_as_traversal_instruction("0").unwrap(), vec![RIGHT]); assert_eq!( - string_as_traversal_instruction("001").unwrap(), + vec_bytes_as_traversal_instruction(&vec![1u8]).unwrap(), + vec![LEFT] + ); + assert_eq!( + vec_bytes_as_traversal_instruction(&vec![0u8]).unwrap(), + vec![RIGHT] + ); + assert_eq!( + vec_bytes_as_traversal_instruction(&vec![0u8, 0u8, 1u8]).unwrap(), vec![RIGHT, RIGHT, LEFT] ); - assert!(string_as_traversal_instruction("002").is_err()); + assert!(vec_bytes_as_traversal_instruction(&vec![0u8, 0u8, 2u8]).is_err()); assert_eq!( - string_as_traversal_instruction("").unwrap(), + vec_bytes_as_traversal_instruction(&vec![]).unwrap(), Vec::::new() ); } @@ -597,69 +608,69 @@ mod test { // tree of height 4 let traversal_instruction = generate_traversal_instruction(4, 1).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), 1 ); let traversal_instruction = generate_traversal_instruction(4, 2).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), 2 ); let traversal_instruction = generate_traversal_instruction(4, 3).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), 3 ); let traversal_instruction = generate_traversal_instruction(4, 4).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 4).unwrap(), 4 ); // tree of height 6 let traversal_instruction = generate_traversal_instruction(6, 1).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 1 ); let traversal_instruction = generate_traversal_instruction(6, 2).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 2 ); let traversal_instruction = generate_traversal_instruction(6, 3).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 3 ); let traversal_instruction = generate_traversal_instruction(6, 4).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 4 ); let traversal_instruction = generate_traversal_instruction(6, 5).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 5 ); let traversal_instruction = generate_traversal_instruction(6, 6).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 6 ); let traversal_instruction = generate_traversal_instruction(6, 7).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 7 ); let traversal_instruction = generate_traversal_instruction(6, 8).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 8 ); let traversal_instruction = generate_traversal_instruction(6, 9).unwrap(); assert_eq!( - chunk_id_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), + chunk_index_from_traversal_instruction(traversal_instruction.as_slice(), 6).unwrap(), 9 ); } @@ -674,26 +685,26 @@ mod test { // function with recovery we expect this to backtrack to the last chunk // boundary e.g. [left] should backtrack to [] // [left, left, right, left] should backtrack to [left, left, right] - assert!(chunk_id_from_traversal_instruction(&[LEFT], 5).is_err()); + assert!(chunk_index_from_traversal_instruction(&[LEFT], 5).is_err()); assert_eq!( - chunk_id_from_traversal_instruction_with_recovery(&[LEFT], 5).unwrap(), + chunk_index_from_traversal_instruction_with_recovery(&[LEFT], 5).unwrap(), 1 ); assert_eq!( - chunk_id_from_traversal_instruction_with_recovery(&[LEFT, LEFT], 5).unwrap(), + chunk_index_from_traversal_instruction_with_recovery(&[LEFT, LEFT], 5).unwrap(), 1 ); assert_eq!( - chunk_id_from_traversal_instruction_with_recovery(&[LEFT, LEFT, RIGHT], 5).unwrap(), + chunk_index_from_traversal_instruction_with_recovery(&[LEFT, LEFT, RIGHT], 5).unwrap(), 3 ); assert_eq!( - chunk_id_from_traversal_instruction_with_recovery(&[LEFT, LEFT, RIGHT, LEFT], 5) + chunk_index_from_traversal_instruction_with_recovery(&[LEFT, LEFT, RIGHT, LEFT], 5) .unwrap(), 3 ); assert_eq!( - chunk_id_from_traversal_instruction_with_recovery(&[LEFT; 50], 5).unwrap(), + chunk_index_from_traversal_instruction_with_recovery(&[LEFT; 50], 5).unwrap(), 2 ); } diff --git a/tutorials/src/bin/proofs.rs b/tutorials/src/bin/proofs.rs index e62fb17b..173b700d 100644 --- a/tutorials/src/bin/proofs.rs +++ b/tutorials/src/bin/proofs.rs @@ -28,7 +28,7 @@ fn main() { let path_query = PathQuery::new_unsized(path, query.clone()); // Execute the query and collect the result items in "elements". let (_elements, _) = db - .query_item_value(&path_query, true, None) + .query_item_value(&path_query, true, false, true, None) .unwrap() .expect("expected successful get_path_query"); diff --git a/tutorials/src/bin/query-complex.rs b/tutorials/src/bin/query-complex.rs index a101bb37..131faa92 100644 --- a/tutorials/src/bin/query-complex.rs +++ b/tutorials/src/bin/query-complex.rs @@ -66,7 +66,7 @@ fn main() { // Execute the path query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, None) + .query_item_value(&path_query, true, false, true, None) .unwrap() .expect("expected successful get_path_query"); diff --git a/tutorials/src/bin/query-simple.rs b/tutorials/src/bin/query-simple.rs index 05ac6264..6bc7a2fd 100644 --- a/tutorials/src/bin/query-simple.rs +++ b/tutorials/src/bin/query-simple.rs @@ -36,7 +36,7 @@ fn main() { // Execute the query and collect the result items in "elements". let (elements, _) = db - .query_item_value(&path_query, true, None) + .query_item_value(&path_query, true, false, true,None) .unwrap() .expect("expected successful get_path_query"); diff --git a/tutorials/src/bin/replication.rs b/tutorials/src/bin/replication.rs index fc9c058c..e285861a 100644 --- a/tutorials/src/bin/replication.rs +++ b/tutorials/src/bin/replication.rs @@ -203,7 +203,7 @@ fn query_db(db: &GroveDb, path: &[&[u8]], key: Vec) { let path_query = PathQuery::new_unsized(path_vec, query.clone()); let (elements, _) = db - .query_item_value(&path_query, true, None) + .query_item_value(&path_query, true, false,true, None) .unwrap() .expect("expected successful get_path_query"); for e in elements.into_iter() {