diff --git a/wnfs-common/src/metadata.rs b/wnfs-common/src/metadata.rs index b20d9291..02632cdf 100644 --- a/wnfs-common/src/metadata.rs +++ b/wnfs-common/src/metadata.rs @@ -7,7 +7,7 @@ use serde::{ de::{DeserializeOwned, Error as DeError}, Deserialize, Deserializer, Serialize, Serializer, }; -use std::{collections::BTreeMap, convert::TryInto, fmt::Display}; +use std::{collections::BTreeMap, fmt::Display}; //-------------------------------------------------------------------------------------------------- // Type Definitions diff --git a/wnfs-common/src/storable.rs b/wnfs-common/src/storable.rs index 53673936..f48de854 100644 --- a/wnfs-common/src/storable.rs +++ b/wnfs-common/src/storable.rs @@ -35,8 +35,6 @@ macro_rules! impl_storable_from_serde { }; } -pub use impl_storable_from_serde; - //-------------------------------------------------------------------------------------------------- // Type Definitions //-------------------------------------------------------------------------------------------------- diff --git a/wnfs-hamt/src/diff.rs b/wnfs-hamt/src/diff.rs index 42fa77f4..2343b43d 100644 --- a/wnfs-hamt/src/diff.rs +++ b/wnfs-hamt/src/diff.rs @@ -283,7 +283,7 @@ where } for Pair { key, value } in &other_values { - if main_map.get(key).is_none() { + if !main_map.contains_key(key) { changes.push(KeyValueChange { r#type: ChangeType::Remove, key: key.clone(), diff --git a/wnfs-hamt/src/merge.rs b/wnfs-hamt/src/merge.rs index 748a0af9..b0f9d8b0 100644 --- a/wnfs-hamt/src/merge.rs +++ b/wnfs-hamt/src/merge.rs @@ -13,16 +13,15 @@ use wnfs_common::{ //-------------------------------------------------------------------------------------------------- /// Merges a node with another with the help of a resolver function. -pub async fn merge( +pub async fn merge( main_link: Link>>, other_link: Link>>, - f: F, - store: &B, + f: impl Fn(&V, &V) -> Result, + store: &impl BlockStore, ) -> Result>> where - F: Fn(&V, &V) -> Result, - K: Storable + Eq + Clone + Hash + AsRef<[u8]>, - V: Storable + Eq + Clone, + K: Storable + Eq + Clone + CondSync + Hash + AsRef<[u8]>, + V: Storable + Eq + Clone + CondSync, K::Serializable: Serialize + DeserializeOwned, V::Serializable: Serialize + DeserializeOwned, H: Hasher + CondSync, diff --git a/wnfs-hamt/src/strategies/kv.rs b/wnfs-hamt/src/strategies/kv.rs index fa97449b..f56c3252 100644 --- a/wnfs-hamt/src/strategies/kv.rs +++ b/wnfs-hamt/src/strategies/kv.rs @@ -12,13 +12,14 @@ use wnfs_common::{ // Functions //-------------------------------------------------------------------------------------------------- -pub fn generate_kvs( +pub fn generate_kvs( key: impl Strategy, value: impl Strategy, size: impl Into, ) -> impl Strategy> where - K: Eq + Hash, + K: Debug + Clone + Eq + Hash, + V: Debug + Clone, { vec((key, value), size).prop_map(|vec| { vec.into_iter() diff --git a/wnfs-hamt/src/strategies/operations.rs b/wnfs-hamt/src/strategies/operations.rs index cf6bf422..ed3da7fc 100644 --- a/wnfs-hamt/src/strategies/operations.rs +++ b/wnfs-hamt/src/strategies/operations.rs @@ -183,13 +183,13 @@ where /// println!("{:?}", node); /// } /// ``` -pub async fn node_from_operations( +pub async fn node_from_operations( operations: &Operations, store: &impl BlockStore, ) -> Result>> where - K: Storable + Clone + Debug + AsRef<[u8]>, - V: Storable + Clone + Debug, + K: Storable + Clone + Debug + CondSync + AsRef<[u8]>, + V: Storable + Clone + Debug + CondSync, K::Serializable: Serialize + DeserializeOwned, V::Serializable: Serialize + DeserializeOwned, { diff --git a/wnfs-wasm/src/fs/private/file.rs b/wnfs-wasm/src/fs/private/file.rs index 1f6926dd..16502047 100644 --- a/wnfs-wasm/src/fs/private/file.rs +++ b/wnfs-wasm/src/fs/private/file.rs @@ -89,6 +89,23 @@ impl PrivateFile { self.read_at(value!(0).into(), None, forest, store) } + /// Gets the exact content size without fetching all content blocks. + #[wasm_bindgen(js_name = "getSize")] + pub fn get_size(&self, forest: &PrivateForest, store: BlockStore) -> JsResult { + let file = Rc::clone(&self.0); + let store = ForeignBlockStore(store); + let forest = Rc::clone(&forest.0); + + Ok(future_to_promise(async move { + let size = file + .size(&forest, &store) + .await + .map_err(error("Cannot determine file size"))?; + + Ok(value!(size as usize)) + })) + } + /// Gets the metadata of this file. pub fn metadata(&self) -> JsResult { JsMetadata(self.0.get_metadata()).try_into() diff --git a/wnfs-wasm/src/fs/public/file.rs b/wnfs-wasm/src/fs/public/file.rs index e675d9fd..0d4d28a1 100644 --- a/wnfs-wasm/src/fs/public/file.rs +++ b/wnfs-wasm/src/fs/public/file.rs @@ -95,6 +95,22 @@ impl PublicFile { self.read_at(value!(0).into(), None, store) } + /// Gets the exact content size without fetching all content blocks. + #[wasm_bindgen(js_name = "getSize")] + pub fn get_size(&self, store: BlockStore) -> JsResult { + let file = Rc::clone(&self.0); + let store = ForeignBlockStore(store); + + Ok(future_to_promise(async move { + let size = file + .size(&store) + .await + .map_err(error("Cannot determine file size"))?; + + Ok(value!(size as usize)) + })) + } + /// Gets the metadata of this file. pub fn metadata(&self) -> JsResult { JsMetadata(self.0.get_metadata()).try_into() diff --git a/wnfs-wasm/tests/private.spec.ts b/wnfs-wasm/tests/private.spec.ts index 5261e478..4fcf0fa1 100644 --- a/wnfs-wasm/tests/private.spec.ts +++ b/wnfs-wasm/tests/private.spec.ts @@ -453,6 +453,31 @@ test.describe("PrivateFile", () => { expect(new Uint8Array(Object.values(content))).toEqual(new Uint8Array([2, 3, 4])); }); + test("getSize returns the exact content size", async ({ page }) => { + const size = await page.evaluate(async () => { + const { + wnfs: { PrivateFile, PrivateForest }, + mock: { MemoryBlockStore, Rng }, + } = await window.setup(); + + const rng = new Rng(); + const initialForest = new PrivateForest(rng); + const store = new MemoryBlockStore(); + var [file, forest] = await PrivateFile.withContent( + initialForest.emptyName(), + new Date(), + new Uint8Array(2 * 1024 * 1024), + initialForest, + store, + rng, + ); + + return await file.getSize(forest, store); + }); + + expect(size).toEqual(2 * 1024 * 1024); + }); + test("A PrivateDirectory has the correct metadata", async ({ page }) => { const result = await page.evaluate(async () => { const { diff --git a/wnfs-wasm/tests/public.spec.ts b/wnfs-wasm/tests/public.spec.ts index 02e4df67..77941019 100644 --- a/wnfs-wasm/tests/public.spec.ts +++ b/wnfs-wasm/tests/public.spec.ts @@ -29,9 +29,7 @@ test.describe("PublicDirectory", () => { expect(result).toBeDefined(); }); - test("lookupNode cannot fetch file not added to directory", async ({ - page, - }) => { + test("lookupNode cannot fetch file not added to directory", async ({ page }) => { const result = await page.evaluate(async () => { const { wnfs: { PublicDirectory }, @@ -65,18 +63,12 @@ test.describe("PublicDirectory", () => { ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); - let result0 = await rootDir.getNode( - ["pictures", "cats", "tabby.png"], - store - ); + let result0 = await rootDir.getNode(["pictures", "cats", "tabby.png"], store); - let result1 = await rootDir.getNode( - ["pictures", "dogs", "bingo.png"], - store - ); + let result1 = await rootDir.getNode(["pictures", "dogs", "bingo.png"], store); return [result0, result1]; }); @@ -102,7 +94,7 @@ test.describe("PublicDirectory", () => { ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); await rootDir.getNode(["pictures", "cats", "tabby.png"], store); @@ -130,7 +122,7 @@ test.describe("PublicDirectory", () => { ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); const result = await rootDir.ls(["pictures"], store); @@ -158,14 +150,14 @@ test.describe("PublicDirectory", () => { ["pictures", "dogs", "billie.jpeg"], sampleCID, time, - store + store, ); var { rootDir } = await rootDir.write( ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); var { rootDir } = await rootDir.rm(["pictures", "cats"], store); @@ -190,18 +182,13 @@ test.describe("PublicDirectory", () => { const store = new MemoryBlockStore(); const root = new PublicDirectory(time); - var { rootDir } = await root.write( - ["pictures", "cats", "luna.jpeg"], - sampleCID, - time, - store - ); + var { rootDir } = await root.write(["pictures", "cats", "luna.jpeg"], sampleCID, time, store); var { rootDir } = await rootDir.write( ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); var { rootDir } = await rootDir.mkdir(["images"], time, store); @@ -210,7 +197,7 @@ test.describe("PublicDirectory", () => { ["pictures", "cats"], ["images", "cats"], time, - store + store, ); const imagesContent = await rootDir.ls(["images"], store); @@ -236,28 +223,18 @@ test.describe("PublicDirectory", () => { const store = new MemoryBlockStore(); const root = new PublicDirectory(time); - var { rootDir } = await root.write( - ["pictures", "cats", "luna.jpeg"], - sampleCID, - time, - store - ); + var { rootDir } = await root.write(["pictures", "cats", "luna.jpeg"], sampleCID, time, store); var { rootDir } = await rootDir.write( ["pictures", "cats", "tabby.png"], sampleCID, time, - store + store, ); var { rootDir } = await rootDir.mkdir(["images"], time, store); - var { rootDir } = await rootDir.cp( - ["pictures", "cats"], - ["images", "cats"], - time, - store - ); + var { rootDir } = await rootDir.cp(["pictures", "cats"], ["images", "cats"], time, store); const imagesContent = await rootDir.ls(["images"], store); @@ -314,10 +291,7 @@ test.describe("PublicDirectory", () => { const readBack = await file2.getContent(store); const partialRead = await file2.readAt(7, 5, store); - return [ - new TextDecoder().decode(readBack), - new TextDecoder().decode(partialRead) - ]; + return [new TextDecoder().decode(readBack), new TextDecoder().decode(partialRead)]; }); expect(result[0]).toEqual("Hello, World!"); @@ -370,4 +344,25 @@ test.describe("PublicDirectory", () => { expect(result).not.toBeUndefined(); expect(result).toEqual("bafkr4ihkr4ld3m4gqkjf4reryxsy2s5tkbxprqkow6fin2iiyvreuzzab4"); }); + + test("A PublicFile has a content size", async ({ page }) => { + const result = await page.evaluate(async () => { + const { + wnfs: { PublicFile }, + mock: { MemoryBlockStore }, + } = await window.setup(); + + const store = new MemoryBlockStore(); + const time = new Date(); + const file = new PublicFile(time); + + const longString = "x".repeat(5 * 1024 * 1024); + const content = new TextEncoder().encode(longString); + const file2 = await file.setContent(time, content, store); + + return await file2.getSize(store); + }); + + expect(result).toEqual(5 * 1024 * 1024); + }); }); diff --git a/wnfs/src/private/file.rs b/wnfs/src/private/file.rs index 6920e5c8..7b589384 100644 --- a/wnfs/src/private/file.rs +++ b/wnfs/src/private/file.rs @@ -104,8 +104,8 @@ pub(crate) enum FileContent { pub struct PrivateForestContent { pub(crate) key: SnapshotKey, pub(crate) base_name: NameAccumulator, - pub(crate) block_count: usize, - pub(crate) block_content_size: usize, + pub(crate) block_count: u64, + pub(crate) block_content_size: u64, } #[derive(Serialize, Deserialize)] @@ -586,6 +586,51 @@ impl PrivateFile { Ok(self.prepare_next_revision()?.get_metadata_mut()) } + /// Gets the exact content size without fetching all content blocks. + /// + /// # Examples + /// + /// ``` + /// use anyhow::Result; + /// use chrono::Utc; + /// use rand_chacha::ChaCha12Rng; + /// use rand_core::SeedableRng; + /// use wnfs::{ + /// private::{PrivateFile, forest::{hamt::HamtForest, traits::PrivateForest}}, + /// common::{MemoryBlockStore, utils::get_random_bytes}, + /// }; + /// + /// #[async_std::main] + /// async fn main() -> Result<()> { + /// let store = &MemoryBlockStore::new(); + /// let rng = &mut ChaCha12Rng::from_entropy(); + /// let forest = &mut HamtForest::new_rsa_2048_rc(rng); + /// + /// let content = get_random_bytes::<324_568>(rng).to_vec(); + /// let file = PrivateFile::with_content( + /// &forest.empty_name(), + /// Utc::now(), + /// content.clone(), + /// forest, + /// store, + /// rng, + /// ) + /// .await?; + /// + /// let mut size = file.size(forest, store).await?; + /// + /// assert_eq!(content.len() as u64, size); + /// + /// Ok(()) + /// } + /// ``` + pub async fn size(&self, forest: &impl PrivateForest, store: &impl BlockStore) -> Result { + match &self.content.content { + FileContent::Inline { data } => Ok(data.len() as u64), + FileContent::External(forest_content) => forest_content.size(forest, store).await, + } + } + /// Gets the entire content of a file. /// /// # Examples @@ -851,10 +896,9 @@ impl PrivateForestContent { rng: &mut impl CryptoRngCore, ) -> Result { let (key, base_name) = Self::prepare_key_and_base_name(file_name, rng); - let block_count = (content.len() as f64 / MAX_BLOCK_CONTENT_SIZE as f64).ceil() as usize; + let block_count = (content.len() as f64 / MAX_BLOCK_CONTENT_SIZE as f64).ceil() as u64; - for (index, name) in - Self::generate_shard_labels(&key, 0, block_count, &base_name).enumerate() + for (name, index) in Self::generate_shard_labels(&key, 0, block_count, &base_name).zip(0..) { let start = index * MAX_BLOCK_CONTENT_SIZE; let end = content.len().min((index + 1) * MAX_BLOCK_CONTENT_SIZE); @@ -872,7 +916,7 @@ impl PrivateForestContent { key, base_name: forest.get_accumulated_name(&base_name), block_count, - block_content_size: MAX_BLOCK_CONTENT_SIZE, + block_content_size: MAX_BLOCK_CONTENT_SIZE as u64, }) } @@ -925,8 +969,8 @@ impl PrivateForestContent { Ok(PrivateForestContent { key, base_name: forest.get_accumulated_name(&base_name), - block_count: block_index as usize, - block_content_size: MAX_BLOCK_CONTENT_SIZE, + block_count: block_index, + block_content_size: MAX_BLOCK_CONTENT_SIZE as u64, }) } @@ -977,7 +1021,7 @@ impl PrivateForestContent { store: &'a impl BlockStore, ) -> Result> { let block_content_size = MAX_BLOCK_CONTENT_SIZE as u64; - let mut chunk_size_upper_bound = self.get_size_upper_bound() - byte_offset as usize; + let mut chunk_size_upper_bound = (self.get_size_upper_bound() - byte_offset) as usize; if let Some(len_limit) = len_limit { chunk_size_upper_bound = chunk_size_upper_bound.min(len_limit); @@ -1024,7 +1068,7 @@ impl PrivateForestContent { forest: &impl PrivateForest, store: &impl BlockStore, ) -> Result> { - let mut content = Vec::with_capacity(Self::get_size_upper_bound(self)); + let mut content = Vec::with_capacity(Self::get_size_upper_bound(self) as usize); self.stream(0, forest, store) .try_for_each(|chunk| { content.extend_from_slice(&chunk); @@ -1035,19 +1079,32 @@ impl PrivateForestContent { } /// Gets an upper bound estimate of the content size. - pub fn get_size_upper_bound(&self) -> usize { + pub fn get_size_upper_bound(&self) -> u64 { self.block_count * self.block_content_size } + /// Gets the exact size of the content. + pub async fn size(&self, forest: &impl PrivateForest, store: &impl BlockStore) -> Result { + let size_without_last_block = + std::cmp::max(0, self.block_count - 1) * self.block_content_size; + + let size_last_block = self + .read_at(size_without_last_block, None, forest, store) + .await? + .len() as u64; + + Ok(size_without_last_block + size_last_block) + } + /// Generates the labels for all of the content shard blocks. pub(crate) fn generate_shard_labels<'a>( key: &'a SnapshotKey, mut block_index: u64, - block_count: usize, + block_count: u64, base_name: &'a Name, ) -> impl Iterator + 'a { iter::from_fn(move || { - if block_index >= block_count as u64 { + if block_index >= block_count { return None; } diff --git a/wnfs/src/public/file.rs b/wnfs/src/public/file.rs index b5f4108d..97bf841e 100644 --- a/wnfs/src/public/file.rs +++ b/wnfs/src/public/file.rs @@ -309,6 +309,44 @@ impl PublicFile { } } + /// Gets the exact content size without fetching all content blocks. + /// + /// # Examples + /// + /// ``` + /// use anyhow::Result; + /// use rand_chacha::ChaCha12Rng; + /// use rand_core::SeedableRng; + /// use chrono::Utc; + /// use wnfs::{ + /// public::PublicFile, + /// common::{MemoryBlockStore, utils::get_random_bytes}, + /// }; + /// + /// #[async_std::main] + /// async fn main() -> Result<()> { + /// let store = &MemoryBlockStore::new(); + /// let rng = &mut ChaCha12Rng::from_entropy(); + /// let content = get_random_bytes::<324_568>(rng).to_vec(); + /// let file = PublicFile::with_content( + /// Utc::now(), + /// content.clone(), + /// store, + /// ) + /// .await?; + /// + /// let mut size = file.size(store).await?; + /// + /// assert_eq!(content.len() as u64, size); + /// + /// Ok(()) + /// } + /// ``` + pub async fn size(&self, store: &impl BlockStore) -> Result { + let value = self.userland.resolve_value(store).await?; + Ok(value.filesize().unwrap_or(0)) + } + /// Gets the entire content of a file. /// /// # Examples