diff --git a/wnfs-common/Cargo.toml b/wnfs-common/Cargo.toml index be7cea25..d4e8904d 100644 --- a/wnfs-common/Cargo.toml +++ b/wnfs-common/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-common" homepage = "https://fission.codes" authors = ["The Fission Authors"] @@ -19,11 +20,11 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" async-once-cell = "0.5" -async-trait = "0.1" base64 = { version = "0.21", optional = true } base64-serde = { version = "0.7", optional = true } bytes = { version = "1.4", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +cid = "0.10" dashmap = "5.5.3" futures = "0.3" libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] } diff --git a/wnfs-common/src/blockstore.rs b/wnfs-common/src/blockstore.rs index 8f139d28..f5dd08e1 100644 --- a/wnfs-common/src/blockstore.rs +++ b/wnfs-common/src/blockstore.rs @@ -3,17 +3,16 @@ use crate::{ utils::{Arc, CondSend, CondSync}, BlockStoreError, MAX_BLOCK_SIZE, }; -use anyhow::{bail, Result}; -use async_trait::async_trait; use bytes::Bytes; +use futures::Future; use libipld::{ cbor::DagCborCodec, cid::Version, multihash::{Code, MultihashDigest}, - serde as ipld_serde, Cid, + Cid, }; use parking_lot::Mutex; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; //-------------------------------------------------------------------------------------------------- @@ -45,38 +44,74 @@ pub const CODEC_DAG_PB: u64 = 0x70; pub const CODEC_RAW: u64 = 0x55; //-------------------------------------------------------------------------------------------------- -// Type Definitions +// Traits //-------------------------------------------------------------------------------------------------- /// For types that implement block store operations like adding, getting content from the store. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait BlockStore: Sized + CondSync { - async fn get_block(&self, cid: &Cid) -> Result; - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result; - - async fn get_deserializable(&self, cid: &Cid) -> Result - where - V: DeserializeOwned, - { - let bytes = self.get_block(cid).await?; - let ipld = decode(bytes.as_ref(), DagCborCodec)?; - Ok(ipld_serde::from_ipld::(ipld)?) +pub trait BlockStore: CondSync { + /// Retrieve a block from this store via its hash (`Cid`). + /// + /// If this store can't find the block, it may raise an error like `BlockNotFound`. + fn get_block( + &self, + cid: &Cid, + ) -> impl Future> + CondSend; + + /// Put some bytes into the blockstore. These bytes should be encoded with the given codec. + /// + /// E.g. `CODEC_RAW` for raw bytes blocks, `CODEC_DAG_CBOR` for dag-cbor, etc. + /// + /// This codec will determine the codec encoded in the final `Cid` that's returned. + /// + /// If the codec is incorrect, this function won't fail, but any tools that depend on the + /// correctness of the codec may fail. (E.g. tools that follow the links of blocks). + /// + /// This funciton allows the blockstore to choose the hashing function itself. + /// The hashing function that was chosen will be readable from the `Cid` metadata. + /// + /// If you need control over the concrete hashing function that's used, see `put_block_keyed`. + fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> impl Future> + CondSend { + let bytes = bytes.into(); + async move { + let cid = self.create_cid(&bytes, codec)?; + self.put_block_keyed(cid, bytes).await?; + Ok(cid) + } } - async fn put_serializable(&self, value: &V) -> Result - where - V: Serialize + CondSync, - { - let bytes = encode(&ipld_serde::to_ipld(value)?, DagCborCodec)?; - self.put_block(bytes, CODEC_DAG_CBOR).await - } + /// Put a block of data into this blockstore. The block's CID needs to match the CID given. + /// + /// It's up to the blockstore whether to check this fact or assume it when this function is called. + /// + /// The default implementation of `put_block` will use this function under the hood and use + /// the correct CID provided by the `create_cid` function. + /// + /// This is useful to be able to add blocks that were generated from other + /// clients with differently configured hashing functions to this blockstore. + fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> impl Future> + CondSend; + + /// Find out whether a call to `get_block` would return with a result or not. + /// + /// This is useful for data exchange protocols to find out what needs to be fetched + /// externally and what doesn't. + fn has_block( + &self, + cid: &Cid, + ) -> impl Future> + CondSend; // This should be the same in all implementations of BlockStore - fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { // If there are too many bytes, abandon this task if bytes.len() > MAX_BLOCK_SIZE { - bail!(BlockStoreError::MaximumBlockSizeExceeded(bytes.len())) + return Err(BlockStoreError::MaximumBlockSizeExceeded(bytes.len())); } // Compute the Blake3 hash of the bytes @@ -93,6 +128,66 @@ pub trait BlockStore: Sized + CondSync { // Implementations //-------------------------------------------------------------------------------------------------- +impl BlockStore for &B { + async fn get_block(&self, cid: &Cid) -> Result { + (**self).get_block(cid).await + } + + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { + (**self).put_block(bytes, codec).await + } + + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { + (**self).put_block_keyed(cid, bytes).await + } + + async fn has_block(&self, cid: &Cid) -> Result { + (**self).has_block(cid).await + } + + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + (**self).create_cid(bytes, codec) + } +} + +impl BlockStore for Box { + async fn get_block(&self, cid: &Cid) -> Result { + (**self).get_block(cid).await + } + + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { + (**self).put_block(bytes, codec).await + } + + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { + (**self).put_block_keyed(cid, bytes).await + } + + async fn has_block(&self, cid: &Cid) -> Result { + (**self).has_block(cid).await + } + + fn create_cid(&self, bytes: &[u8], codec: u64) -> Result { + (**self).create_cid(bytes, codec) + } +} + /// An in-memory block store to simulate IPFS. /// /// IPFS is basically a glorified HashMap. @@ -111,11 +206,8 @@ impl MemoryBlockStore { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for MemoryBlockStore { - /// Retrieves an array of bytes from the block store with given CID. - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { let bytes = self .0 .lock() @@ -126,18 +218,18 @@ impl BlockStore for MemoryBlockStore { Ok(bytes) } - /// Stores an array of bytes in the block store. - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { - // Convert the bytes into a Bytes object - let bytes: Bytes = bytes.into(); + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { + self.0.lock().insert(cid, bytes.into()); - // Try to build the CID from the bytes and codec - let cid = self.create_cid(&bytes, codec)?; - - // Insert the bytes into the HashMap using the CID as the key - self.0.lock().insert(cid, bytes); + Ok(()) + } - Ok(cid) + async fn has_block(&self, cid: &Cid) -> Result { + Ok(self.0.lock().contains_key(cid)) } } @@ -146,21 +238,18 @@ impl BlockStore for MemoryBlockStore { //-------------------------------------------------------------------------------------------------- /// Tests the retrieval property of a BlockStore-conforming type. -pub async fn bs_retrieval_test(store: &T) -> Result<()> -where - T: BlockStore + 'static, -{ +pub async fn bs_retrieval_test(store: impl BlockStore) -> Result<(), BlockStoreError> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = b"hello world".to_vec(); // Insert the objects into the blockstore - let first_cid = store.put_serializable(&first_bytes).await?; - let second_cid = store.put_serializable(&second_bytes).await?; + let first_cid = store.put_block(first_bytes.clone(), CODEC_RAW).await?; + let second_cid = store.put_block(second_bytes.clone(), CODEC_RAW).await?; // Retrieve the objects from the blockstore - let first_loaded: Vec = store.get_deserializable(&first_cid).await?; - let second_loaded: Vec = store.get_deserializable(&second_cid).await?; + let first_loaded = store.get_block(&first_cid).await?; + let second_loaded = store.get_block(&second_cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(first_loaded, first_bytes); @@ -170,24 +259,21 @@ where } /// Tests the duplication of a BlockStore-conforming type. -pub async fn bs_duplication_test(store: &T) -> Result<()> -where - T: BlockStore + 'static, -{ +pub async fn bs_duplication_test(store: impl BlockStore) -> Result<(), BlockStoreError> { // Example objects to insert and remove from the blockstore let first_bytes = vec![1, 2, 3, 4, 5]; let second_bytes = first_bytes.clone(); // Insert the objects into the blockstore - let first_cid = store.put_serializable(&first_bytes).await?; - let second_cid = store.put_serializable(&second_bytes).await?; + let first_cid = store.put_block(first_bytes.clone(), CODEC_RAW).await?; + let second_cid = store.put_block(second_bytes.clone(), CODEC_RAW).await?; // Assert that the two vecs produced the same CID assert_eq!(first_cid, second_cid); // Retrieve the objects from the blockstore - let first_loaded: Vec = store.get_deserializable(&first_cid).await?; - let second_loaded: Vec = store.get_deserializable(&second_cid).await?; + let first_loaded = store.get_block(&first_cid).await?; + let second_loaded = store.get_block(&second_cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(first_loaded, first_bytes); @@ -200,22 +286,22 @@ where } /// Tests the serialization of a BlockStore-conforming type. -pub async fn bs_serialization_test(store: &T) -> Result<()> +pub async fn bs_serialization_test(store: &T) -> Result<(), BlockStoreError> where - T: BlockStore + Serialize + 'static + for<'de> Deserialize<'de>, + T: BlockStore + Serialize + for<'de> Deserialize<'de>, { // Example objects to insert and remove from the blockstore let bytes = vec![1, 2, 3, 4, 5]; // Insert the object into the blockstore - let cid = store.put_serializable(&bytes).await?; + let cid = store.put_block(bytes.clone(), CODEC_RAW).await?; // Serialize the BlockStore let serial_store: Vec = encode(&store, DagCborCodec)?; // Construct a new BlockStore from the Serialized object let deserial_store: T = decode(&serial_store, DagCborCodec)?; // Retrieve the object from the blockstore - let loaded: Vec = deserial_store.get_deserializable(&cid).await?; + let loaded = deserial_store.get_block(&cid).await?; // Assert that the objects are the same as the ones we inserted assert_eq!(loaded, bytes); @@ -231,9 +317,9 @@ mod tests { #[async_std::test] async fn memory_blockstore() -> Result<()> { let store = &MemoryBlockStore::new(); - bs_retrieval_test(store).await?; - bs_duplication_test(store).await?; - bs_serialization_test(store).await?; + bs_retrieval_test::(store).await?; + bs_duplication_test::(store).await?; + bs_serialization_test::(store).await?; Ok(()) } } diff --git a/wnfs-common/src/error.rs b/wnfs-common/src/error.rs index c75cf329..6db9b2e3 100644 --- a/wnfs-common/src/error.rs +++ b/wnfs-common/src/error.rs @@ -16,9 +16,9 @@ pub enum BlockStoreError { #[error("Cannot find specified CID in block store: {0}")] CIDNotFound(Cid), - #[error("Cannot find handler for block with CID: {0}")] - BlockHandlerNotFound(Cid), + #[error("CID error during blockstore operation: {0}")] + CIDError(#[from] cid::Error), - #[error("Lock poisoned")] - LockPoisoned, + #[error(transparent)] + Custom(#[from] anyhow::Error), } diff --git a/wnfs-common/src/lib.rs b/wnfs-common/src/lib.rs index b3ac7b9a..74302b54 100644 --- a/wnfs-common/src/lib.rs +++ b/wnfs-common/src/lib.rs @@ -6,7 +6,6 @@ mod link; mod metadata; mod pathnodes; mod storable; -mod traits; pub mod utils; pub use blockstore::*; diff --git a/wnfs-common/src/link.rs b/wnfs-common/src/link.rs index d6fb8cec..1f3bb8b1 100644 --- a/wnfs-common/src/link.rs +++ b/wnfs-common/src/link.rs @@ -1,7 +1,6 @@ -use crate::{traits::IpldEq, utils::CondSync, BlockStore, Storable}; +use crate::{utils::CondSync, BlockStore, Storable}; use anyhow::Result; use async_once_cell::OnceCell; -use async_trait::async_trait; use libipld::Cid; use std::fmt::{self, Debug, Formatter}; @@ -137,18 +136,6 @@ impl Link { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -impl IpldEq for Link { - async fn eq(&self, other: &Link, store: &impl BlockStore) -> Result { - if self == other { - return Ok(true); - } - - Ok(self.resolve_cid(store).await? == other.resolve_cid(store).await?) - } -} - impl From for Link { fn from(value: T) -> Self { Self::Decoded { value } @@ -227,7 +214,6 @@ mod tests { use crate::{BlockStore, Link, MemoryBlockStore, Storable}; use anyhow::Result; use async_once_cell::OnceCell; - use async_trait::async_trait; use libipld::Cid; use serde::{Deserialize, Serialize}; @@ -238,7 +224,6 @@ mod tests { persisted_as: OnceCell, } - #[async_trait] impl Storable for Example { type Serializable = Example; @@ -292,7 +277,7 @@ mod tests { async fn link_value_can_be_resolved() { let store = &MemoryBlockStore::default(); let example = Example::new(256); - let cid = store.put_serializable(&example).await.unwrap(); + let cid = example.store(store).await.unwrap(); let link = Link::::from_cid(cid); let value = link.resolve_value(store).await.unwrap(); diff --git a/wnfs-common/src/metadata.rs b/wnfs-common/src/metadata.rs index 8325821b..b20d9291 100644 --- a/wnfs-common/src/metadata.rs +++ b/wnfs-common/src/metadata.rs @@ -7,7 +7,7 @@ use serde::{ de::{DeserializeOwned, Error as DeError}, Deserialize, Deserializer, Serialize, Serializer, }; -use std::{collections::BTreeMap, convert::TryInto}; +use std::{collections::BTreeMap, convert::TryInto, fmt::Display}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -24,17 +24,16 @@ pub enum NodeType { SnapshotSharePointer, } -impl ToString for NodeType { - fn to_string(&self) -> String { - match self { +impl Display for NodeType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { NodeType::PublicFile => "wnfs/pub/file", NodeType::PublicDirectory => "wnfs/pub/dir", NodeType::PrivateFile => "wnfs/priv/file", NodeType::PrivateDirectory => "wnfs/priv/dir", NodeType::TemporalSharePointer => "wnfs/share/temporal", NodeType::SnapshotSharePointer => "wnfs/share/snapshot", - } - .to_string() + }) } } diff --git a/wnfs-common/src/storable.rs b/wnfs-common/src/storable.rs index 79854cae..53673936 100644 --- a/wnfs-common/src/storable.rs +++ b/wnfs-common/src/storable.rs @@ -2,13 +2,13 @@ //! that are implemented for most WNFS structures, such as `PublicFile`, `PublicDirectory`, //! `PublicNode`, `HamtForest` etc. use crate::{ - utils::{Arc, CondSync}, + utils::{Arc, CondSend, CondSync}, BlockStore, }; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use bytes::Bytes; +use futures::Future; use libipld::{cbor::DagCborCodec, Cid}; use serde::{de::DeserializeOwned, Serialize}; @@ -20,8 +20,6 @@ use serde::{de::DeserializeOwned, Serialize}; macro_rules! impl_storable_from_serde { ( $( $ty:ty $( : < $( $generics:ident ),+ > )? ),+ ) => { $( - #[cfg_attr(not(target_arch = "wasm32"), ::async_trait::async_trait)] - #[cfg_attr(target_arch = "wasm32", ::async_trait::async_trait(?Send))] impl $( < $( $generics ),+ > )? $crate::Storable for $ty $( where $( $generics: ::serde::Serialize + ::serde::de::DeserializeOwned + Clone + $crate::utils::CondSync ),+ )?{ type Serializable = $ty; @@ -62,19 +60,22 @@ pub use impl_storable_from_serde; /// If you do so, remember to initialize the `OnceCell` if a `Cid` is passed in the /// `from_serializable` call, such that a `store` call right after a `load` call is practically /// free. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait Storable: Sized { /// The at-rest representation of this storable type. type Serializable: StoreIpld + LoadIpld + CondSync; /// Turn the current type into the at-rest representation of this type. - async fn to_serializable(&self, store: &impl BlockStore) -> Result; + fn to_serializable( + &self, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Take an at-rest representation of this type and turn it into the in-memory representation. /// You can use the `cid` parameter to populate a cache. - async fn from_serializable(cid: Option<&Cid>, serializable: Self::Serializable) - -> Result; + fn from_serializable( + cid: Option<&Cid>, + serializable: Self::Serializable, + ) -> impl Future> + CondSend; /// Return a serialization cache, if it exists. /// By default, this always returns `None`. @@ -85,16 +86,21 @@ pub trait Storable: Sized { /// Store this data type in a given `BlockStore`. /// /// This will short-circuit by using the `persisted_as` once-cell, if available. - async fn store(&self, store: &impl BlockStore) -> Result { + fn store(&self, store: &impl BlockStore) -> impl Future> + CondSend + where + Self: CondSync, + { let store_future = async { let (bytes, codec) = self.to_serializable(store).await?.encode_ipld()?; - store.put_block(bytes, codec).await + Ok(store.put_block(bytes, codec).await?) }; - if let Some(persisted_as) = self.persisted_as() { - persisted_as.get_or_try_init(store_future).await.cloned() - } else { - store_future.await + async { + if let Some(persisted_as) = self.persisted_as() { + persisted_as.get_or_try_init(store_future).await.cloned() + } else { + store_future.await + } } } @@ -102,10 +108,12 @@ pub trait Storable: Sized { /// /// This will pass on the CID to the `from_serializable` function so it can /// populate a cache in some cases. - async fn load(cid: &Cid, store: &impl BlockStore) -> Result { - let bytes = store.get_block(cid).await?; - let serializable = Self::Serializable::decode_ipld(cid, bytes)?; - Self::from_serializable(Some(cid), serializable).await + fn load(cid: &Cid, store: &impl BlockStore) -> impl Future> + CondSend { + async { + let bytes = store.get_block(cid).await?; + let serializable = Self::Serializable::decode_ipld(cid, bytes)?; + Self::from_serializable(Some(cid), serializable).await + } } } @@ -158,8 +166,6 @@ impl LoadIpld for T { // } // } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Arc { type Serializable = T::Serializable; diff --git a/wnfs-common/src/traits.rs b/wnfs-common/src/traits.rs deleted file mode 100644 index 5cd3afff..00000000 --- a/wnfs-common/src/traits.rs +++ /dev/null @@ -1,15 +0,0 @@ -use crate::BlockStore; -use anyhow::Result; -use async_trait::async_trait; - -//-------------------------------------------------------------------------------------------------- -// Traits -//-------------------------------------------------------------------------------------------------- - -/// Implements deep equality check for two types. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] -pub trait IpldEq { - /// Checks if the two items are deeply equal. - async fn eq(&self, other: &Self, store: &impl BlockStore) -> Result; -} diff --git a/wnfs-common/src/utils/test.rs b/wnfs-common/src/utils/test.rs index f0f429f8..3bbc44e3 100644 --- a/wnfs-common/src/utils/test.rs +++ b/wnfs-common/src/utils/test.rs @@ -1,7 +1,6 @@ use super::{Arc, CondSend, CondSync}; -use crate::{BlockStore, MemoryBlockStore, CODEC_DAG_CBOR, CODEC_RAW}; +use crate::{BlockStore, BlockStoreError, MemoryBlockStore, CODEC_DAG_CBOR, CODEC_RAW}; use anyhow::Result; -use async_trait::async_trait; use base64_serde::base64_serde_type; use bytes::Bytes; use libipld::{ @@ -112,18 +111,34 @@ impl SnapshotBlockStore { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for SnapshotBlockStore { #[inline] - async fn get_block(&self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { self.inner.get_block(cid).await } #[inline] - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { self.inner.put_block(bytes, codec).await } + + #[inline] + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { + self.inner.put_block_keyed(cid, bytes).await + } + + #[inline] + async fn has_block(&self, cid: &Cid) -> Result { + self.inner.has_block(cid).await + } } impl Sampleable for S diff --git a/wnfs-hamt/Cargo.toml b/wnfs-hamt/Cargo.toml index b61c35fa..431949a5 100644 --- a/wnfs-hamt/Cargo.toml +++ b/wnfs-hamt/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-hamt" homepage = "https://fission.codes" authors = ["The Fission Authors"] @@ -20,7 +21,6 @@ authors = ["The Fission Authors"] anyhow = "1.0" async-once-cell = "0.5" async-recursion = "1.0" -async-trait = "0.1" bitvec = { version = "1.0", features = ["serde"] } blake3 = { version = "1.4", features = ["traits-preview"] } chrono = { version = "0.4.23", default-features = false, features = ["clock", "std"] } diff --git a/wnfs-hamt/src/hamt.rs b/wnfs-hamt/src/hamt.rs index 19980295..2211c96f 100644 --- a/wnfs-hamt/src/hamt.rs +++ b/wnfs-hamt/src/hamt.rs @@ -1,7 +1,6 @@ use super::{KeyValueChange, Node, HAMT_VERSION}; use crate::{serializable::HamtSerializable, Hasher}; use anyhow::Result; -use async_trait::async_trait; use libipld::Cid; use semver::Version; use serde::{de::DeserializeOwned, Serialize}; @@ -129,8 +128,6 @@ impl Hamt { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Hamt where K: Storable + CondSync, diff --git a/wnfs-hamt/src/node.rs b/wnfs-hamt/src/node.rs index 7624be6e..453e828e 100644 --- a/wnfs-hamt/src/node.rs +++ b/wnfs-hamt/src/node.rs @@ -7,7 +7,6 @@ use crate::{serializable::NodeSerializable, HAMT_VALUES_BUCKET_SIZE}; use anyhow::{bail, Result}; use async_once_cell::OnceCell; use async_recursion::async_recursion; -use async_trait::async_trait; use bitvec::array::BitArray; use either::{Either, Either::*}; use libipld::Cid; @@ -22,7 +21,7 @@ use std::{ marker::PhantomData, }; use wnfs_common::{ - utils::{Arc, BoxFuture, CondSend, CondSync}, + utils::{boxed_fut, Arc, BoxFuture, CondSend, CondSync}, BlockStore, HashOutput, Link, Storable, }; @@ -838,8 +837,6 @@ where } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Node where K: Storable + CondSync, @@ -855,7 +852,8 @@ where let mut pointers = Vec::with_capacity(self.pointers.len()); for pointer in self.pointers.iter() { - pointers.push(pointer.to_serializable(store).await?); + // Boxing the future due to recursion + pointers.push(boxed_fut(pointer.to_serializable(store)).await?); } Ok(NodeSerializable(bitmask, pointers)) diff --git a/wnfs-hamt/src/pointer.rs b/wnfs-hamt/src/pointer.rs index e115b155..042589d9 100644 --- a/wnfs-hamt/src/pointer.rs +++ b/wnfs-hamt/src/pointer.rs @@ -1,7 +1,6 @@ use super::{error::HamtError, hash::Hasher, Node, HAMT_VALUES_BUCKET_SIZE}; use crate::serializable::PointerSerializable; use anyhow::Result; -use async_trait::async_trait; use libipld::Cid; use serde::{de::DeserializeOwned, Serialize}; use std::fmt::Debug; @@ -110,8 +109,6 @@ impl Pointer { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Pointer where K: Storable + CondSync, @@ -155,8 +152,6 @@ where } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for Pair where K: Storable + CondSync, diff --git a/wnfs-nameaccumulator/Cargo.toml b/wnfs-nameaccumulator/Cargo.toml index 6732aec7..726f93aa 100644 --- a/wnfs-nameaccumulator/Cargo.toml +++ b/wnfs-nameaccumulator/Cargo.toml @@ -18,7 +18,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" -async-trait = "0.1" blake3 = { version = "1.4", features = ["traits-preview"] } libipld = { version = "0.16", features = ["dag-cbor", "derive", "serde-codec"] } num-bigint-dig = { version = "0.8.2", features = ["prime", "zeroize"], optional = true } @@ -26,7 +25,7 @@ num-integer = "0.1.45" num-traits = "0.2.15" once_cell = "1.0" rand_core = "0.6" -rug = { version = "1.22", optional = true, default-features = false, features = ["rand", "integer", "num-traits"] } +rug = { version = "1.24", optional = true, default-features = false, features = ["rand", "integer", "num-traits"] } serde = { version = "1.0", features = ["rc"] } serde_bytes = "0.11.9" thiserror = "1.0" diff --git a/wnfs-nameaccumulator/src/name.rs b/wnfs-nameaccumulator/src/name.rs index 81a8eef7..1d73bfa1 100644 --- a/wnfs-nameaccumulator/src/name.rs +++ b/wnfs-nameaccumulator/src/name.rs @@ -443,8 +443,6 @@ impl<'a, B: Big> BatchedProofVerification<'a, B> { macro_rules! impl_storable { ( $ty:ty ) => { - #[cfg_attr(not(target_arch = "wasm32"), ::async_trait::async_trait)] - #[cfg_attr(target_arch = "wasm32", ::async_trait::async_trait(?Send))] impl Storable for $ty { type Serializable = $ty; @@ -823,7 +821,7 @@ mod snapshot_tests { use crate::{BigNumDig, NameSegment}; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; - use wnfs_common::{utils::SnapshotBlockStore, BlockStore}; + use wnfs_common::utils::SnapshotBlockStore; #[async_std::test] async fn test_name_accumulator() { @@ -841,7 +839,7 @@ mod snapshot_tests { setup, ); - let cid = store.put_serializable(&acc).await.unwrap(); + let cid = acc.store(store).await.unwrap(); let name = store.get_block_snapshot(&cid).await.unwrap(); insta::assert_json_snapshot!(name); diff --git a/wnfs-nameaccumulator/src/traits.rs b/wnfs-nameaccumulator/src/traits.rs index 3ff2d68d..8d80c984 100644 --- a/wnfs-nameaccumulator/src/traits.rs +++ b/wnfs-nameaccumulator/src/traits.rs @@ -238,10 +238,8 @@ impl Big for BigNumRug { } fn to_bytes_be(n: &Self::Num) -> [u8; N] { - let vec = n.to_digits(Order::MsfLe); let mut bytes = [0u8; N]; - let zero_bytes = N - vec.len(); - bytes[zero_bytes..].copy_from_slice(&vec); + n.write_digits(&mut bytes, Order::MsfLe); bytes } @@ -293,6 +291,8 @@ mod rug_tests { use crate::{Big, BigNumRug}; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; + use rug::Integer; + use std::str::FromStr; /// We need this property for snapshot testing #[test] @@ -314,4 +314,18 @@ mod rug_tests { let run_three = BigNumRug::rand_below(&ceiling, &mut ChaCha12Rng::seed_from_u64(1)); assert_ne!(run_one, run_three); } + + #[test] + fn test_to_bytes_be_snapshot() { + let modulus = Integer::from_str( + "25195908475657893494027183240048398571429282126204032027777137836043662020707595556264018525880784406918290641249515082189298559149176184502808489120072844992687392807287776735971418347270261896375014971824691165077613379859095700097330459748808428401797429100642458691817195118746121515172654632282216869987549182422433637259085141865462043576798423387184774447920739934236584823824281198163815010674810451660377306056201619676256133844143603833904414952634432190114657544454178424020924616515723350778707749817125772467962926386356373289912154831438167899885040445364023527381951378636564391212010397122822120720357", + ).expect("Can parse integer"); + + let modulus_hex = "c7970ceedcc3b0754490201a7aa613cd73911081c790f5f1a8726f463550bb5b7ff0db8e1ea1189ec72f93d1650011bd721aeeacc2acde32a04107f0648c2813a31f5b0b7765ff8b44b4b6ffc93384b646eb09c7cf5e8592d40ea33c80039f35b4f14a04b51f7bfd781be4d1673164ba8eb991c2c4d730bbbe35f592bdef524af7e8daefd26c66fc02c479af89d64d373f442709439de66ceb955f3ea37d5159f6135809f85334b5cb1813addc80cd05609f10ac6a95ad65872c909525bdad32bc729592642920f24c61dc5b3c3b7923e56b16a4d9d373d8721f24a3fc0f1b3131f55615172866bccc30f95054c824e733a5eb6817f7bc16399d48c6361cc7e5"; + + let bytes = BigNumRug::to_bytes_be::<256>(&modulus); + let hex_encoded = hex::encode(bytes); + + assert_eq!(hex_encoded, modulus_hex); + } } diff --git a/wnfs-unixfs-file/Cargo.toml b/wnfs-unixfs-file/Cargo.toml index 83c521a3..7bc0b70a 100644 --- a/wnfs-unixfs-file/Cargo.toml +++ b/wnfs-unixfs-file/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-unixfs-file" homepage = "https://fission.codes" authors = ["The Fission Authors"] @@ -19,7 +20,6 @@ authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" async-stream = "0.3" -async-trait = "0.1" bytes = "1.5" futures = "0.3" libipld = { version = "0.16", features = [] } diff --git a/wnfs-unixfs-file/src/types.rs b/wnfs-unixfs-file/src/types.rs index 79c89700..a669a0c5 100644 --- a/wnfs-unixfs-file/src/types.rs +++ b/wnfs-unixfs-file/src/types.rs @@ -38,7 +38,9 @@ impl Block { } pub async fn store(&self, store: &impl BlockStore) -> Result { - store.put_block(self.data.clone(), self.codec.into()).await + Ok(store + .put_block(self.data.clone(), self.codec.into()) + .await?) } /// Validate the block. Will return an error if the links are wrong. diff --git a/wnfs-unixfs-file/src/unixfs.rs b/wnfs-unixfs-file/src/unixfs.rs index 74b02658..2c3f9db2 100644 --- a/wnfs-unixfs-file/src/unixfs.rs +++ b/wnfs-unixfs-file/src/unixfs.rs @@ -5,7 +5,6 @@ use crate::{ types::{Block, Link, LinkRef, Links, PbLinks}, }; use anyhow::{anyhow, bail, ensure, Result}; -use async_trait::async_trait; use bytes::Bytes; use futures::FutureExt; use libipld::Cid; @@ -223,8 +222,6 @@ impl UnixFsFile { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for UnixFsFile { type Serializable = UnixFsFile; diff --git a/wnfs-wasm/Cargo.toml b/wnfs-wasm/Cargo.toml index 24014f78..a850c4f7 100644 --- a/wnfs-wasm/Cargo.toml +++ b/wnfs-wasm/Cargo.toml @@ -12,13 +12,13 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs-wasm" homepage = "https://fission.codes" authors = ["The Fission Authors"] [dependencies] anyhow = "1.0" -async-trait = "0.1" bytes = "1.4.0" cfg-if = "1.0" chrono = { version = "0.4", default-features = false, features = ["clock", "std", "wasmbind"] } diff --git a/wnfs-wasm/src/fs/blockstore.rs b/wnfs-wasm/src/fs/blockstore.rs index b01d9f84..e68d2de1 100644 --- a/wnfs-wasm/src/fs/blockstore.rs +++ b/wnfs-wasm/src/fs/blockstore.rs @@ -2,28 +2,39 @@ use super::utils::anyhow_error; use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use js_sys::{Promise, Uint8Array}; use libipld_core::cid::Cid; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen_futures::JsFuture; -use wnfs::common::BlockStore as WnfsBlockStore; +use wnfs::common::{BlockStore as WnfsBlockStore, BlockStoreError}; //-------------------------------------------------------------------------------------------------- // Externs //-------------------------------------------------------------------------------------------------- +#[wasm_bindgen(typescript_custom_section)] +const TS_BLOCKSTORE: &'static str = r#" +export interface BlockStore { + putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise; + getBlock(cid: Uint8Array): Promise; + hasBlock(cid: Uint8Array): Promise; +} +"#; + #[wasm_bindgen] extern "C" { #[wasm_bindgen(typescript_type = "BlockStore")] pub type BlockStore; - #[wasm_bindgen(method, js_name = "putBlock")] - pub(crate) fn put_block(store: &BlockStore, bytes: Vec, codec: u32) -> Promise; + #[wasm_bindgen(method, js_name = "putBlockKeyed")] + pub(crate) fn put_block_keyed(store: &BlockStore, cid: Vec, bytes: Vec) -> Promise; #[wasm_bindgen(method, js_name = "getBlock")] pub(crate) fn get_block(store: &BlockStore, cid: Vec) -> Promise; + + #[wasm_bindgen(method, js_name = "hasBlock")] + pub(crate) fn has_block(store: &BlockStore, cid: Vec) -> Promise; } //-------------------------------------------------------------------------------------------------- @@ -38,31 +49,40 @@ pub struct ForeignBlockStore(pub(crate) BlockStore); // Implementations //-------------------------------------------------------------------------------------------------- -#[async_trait(?Send)] impl WnfsBlockStore for ForeignBlockStore { - /// Stores an array of bytes in the block store. - async fn put_block(&self, bytes: impl Into, codec: u64) -> Result { + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into, + ) -> Result<(), BlockStoreError> { let bytes: Bytes = bytes.into(); - let value = JsFuture::from(self.0.put_block(bytes.into(), codec.try_into()?)) + JsFuture::from(self.0.put_block_keyed(cid.to_bytes(), bytes.into())) .await - .map_err(anyhow_error("Cannot get block: {:?}"))?; - - // Convert the value to a vector of bytes. - let bytes = Uint8Array::new(&value).to_vec(); + .map_err(anyhow_error("Cannot put block: {:?}"))?; - // Construct CID from the bytes. - Ok(Cid::try_from(&bytes[..])?) + Ok(()) } - /// Retrieves an array of bytes from the block store with given CID. - async fn get_block<'a>(&'a self, cid: &Cid) -> Result { + async fn get_block(&self, cid: &Cid) -> Result { let value = JsFuture::from(self.0.get_block(cid.to_bytes())) .await .map_err(anyhow_error("Cannot get block: {:?}"))?; + if value.is_undefined() { + return Err(BlockStoreError::CIDNotFound(*cid)); + } + // Convert the value to a vector of bytes. let bytes = Uint8Array::new(&value).to_vec(); Ok(Bytes::from(bytes)) } + + async fn has_block(&self, cid: &Cid) -> Result { + let value = JsFuture::from(self.0.has_block(cid.to_bytes())) + .await + .map_err(anyhow_error("Cannot run has_block: {:?}"))?; + + Ok(js_sys::Boolean::from(value).value_of()) + } } diff --git a/wnfs-wasm/src/fs/mod.rs b/wnfs-wasm/src/fs/mod.rs index 6199a17e..74501ab2 100644 --- a/wnfs-wasm/src/fs/mod.rs +++ b/wnfs-wasm/src/fs/mod.rs @@ -2,7 +2,6 @@ mod blockstore; mod metadata; mod private; mod public; -mod types; mod utils; pub use blockstore::*; diff --git a/wnfs-wasm/src/fs/private/exchange_key.rs b/wnfs-wasm/src/fs/private/exchange_key.rs index 03b92ed4..71a088ca 100644 --- a/wnfs-wasm/src/fs/private/exchange_key.rs +++ b/wnfs-wasm/src/fs/private/exchange_key.rs @@ -1,6 +1,5 @@ use crate::fs::utils::anyhow_error; use anyhow::Result; -use async_trait::async_trait; use js_sys::{Promise, Uint8Array}; use wasm_bindgen::prelude::wasm_bindgen; use wasm_bindgen_futures::JsFuture; @@ -45,7 +44,6 @@ pub struct ForeignPrivateKey(pub(crate) PrivateKey); // Implementations //-------------------------------------------------------------------------------------------------- -#[async_trait(?Send)] impl WnfsExchangeKey for ForeignExchangeKey { async fn from_modulus(modulus: &[u8]) -> Result where @@ -67,7 +65,6 @@ impl WnfsExchangeKey for ForeignExchangeKey { } } -#[async_trait(?Send)] impl WnfsPrivateKey for ForeignPrivateKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let v = JsFuture::from(self.0.decrypt(ciphertext)) diff --git a/wnfs-wasm/src/fs/public/file.rs b/wnfs-wasm/src/fs/public/file.rs index 0312cc78..e675d9fd 100644 --- a/wnfs-wasm/src/fs/public/file.rs +++ b/wnfs-wasm/src/fs/public/file.rs @@ -103,7 +103,7 @@ impl PublicFile { /// Gets the content cid of the file. #[wasm_bindgen(js_name = "getRawContentCid")] pub fn get_raw_content_cid(&self, store: BlockStore) -> JsResult { - let mut file = Rc::clone(&self.0); + let file = Rc::clone(&self.0); let store = ForeignBlockStore(store); Ok(future_to_promise(async move { diff --git a/wnfs-wasm/src/fs/types.rs b/wnfs-wasm/src/fs/types.rs deleted file mode 100644 index 3b52cbd1..00000000 --- a/wnfs-wasm/src/fs/types.rs +++ /dev/null @@ -1,9 +0,0 @@ -use wasm_bindgen::prelude::wasm_bindgen; - -#[wasm_bindgen(typescript_custom_section)] -const TS_BLOCKSTORE: &'static str = r#" -export interface BlockStore { - putBlock(bytes: Uint8Array, code: number): Promise; - getBlock(cid: Uint8Array): Promise; -} -"#; diff --git a/wnfs-wasm/tests/mock.ts b/wnfs-wasm/tests/mock.ts index fb5fa0a6..695a79fd 100644 --- a/wnfs-wasm/tests/mock.ts +++ b/wnfs-wasm/tests/mock.ts @@ -30,16 +30,20 @@ class MemoryBlockStore { /** Stores an array of bytes in the block store. */ async getBlock(cid: Uint8Array): Promise { - const decoded_cid = CID.decode(cid); - return this.store.get(decoded_cid.toString()); + const decodedCid = CID.decode(cid); + return this.store.get(decodedCid.toString()); } /** Retrieves an array of bytes from the block store with given CID. */ - async putBlock(bytes: Uint8Array, codec: number): Promise { - const hash = await sha256.digest(bytes); - const cid = CID.create(1, codec, hash); - this.store.set(cid.toString(), bytes); - return cid.bytes; + async putBlockKeyed(cid: Uint8Array, bytes: Uint8Array): Promise { + const decodedCid = CID.decode(cid); + this.store.set(decodedCid.toString(), bytes); + } + + /** Finds out whether a block is retrievable from this blockstore */ + async hasBlock(cid: Uint8Array): Promise { + const decodedCid = CID.decode(cid); + return this.store.has(decodedCid.toString()); } } diff --git a/wnfs-wasm/tests/public.spec.ts b/wnfs-wasm/tests/public.spec.ts index 3cc34a18..02e4df67 100644 --- a/wnfs-wasm/tests/public.spec.ts +++ b/wnfs-wasm/tests/public.spec.ts @@ -368,6 +368,6 @@ test.describe("PublicDirectory", () => { }); expect(result).not.toBeUndefined(); - expect(result).toEqual("bafkreibm6jg3ux5qumhcn2b3flc3tyu6dmlb4xa7u5bf44yegnrjhc4yeq"); + expect(result).toEqual("bafkr4ihkr4ld3m4gqkjf4reryxsy2s5tkbxprqkow6fin2iiyvreuzzab4"); }); }); diff --git a/wnfs-wasm/yarn.lock b/wnfs-wasm/yarn.lock index 73cbeda5..3ee07444 100644 --- a/wnfs-wasm/yarn.lock +++ b/wnfs-wasm/yarn.lock @@ -2009,11 +2009,6 @@ multicast-dns@^7.2.5: dns-packet "^5.2.2" thunky "^1.0.2" -multiformats@^12.0.1: - version "12.1.3" - resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-12.1.3.tgz#cbf7a9861e11e74f8228b21376088cb43ba8754e" - integrity sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw== - multiformats@^13.0.0: version "13.0.0" resolved "https://registry.yarnpkg.com/multiformats/-/multiformats-13.0.0.tgz#97f3341b16c34716a14518d178ea0c190e987c32" diff --git a/wnfs/Cargo.toml b/wnfs/Cargo.toml index c8386907..0fd9bd0f 100644 --- a/wnfs/Cargo.toml +++ b/wnfs/Cargo.toml @@ -12,6 +12,7 @@ categories = [ license = "Apache-2.0" readme = "README.md" edition = "2021" +rust-version = "1.75" repository = "https://github.com/wnfs-wg/rs-wnfs/tree/main/wnfs" homepage = "https://fission.codes" authors = ["The Fission Authors"] @@ -22,7 +23,6 @@ anyhow = "1.0" async-once-cell = "0.5" async-recursion = "1.0" async-stream = "0.3" -async-trait = "0.1" blake3 = { version = "1.4", features = ["traits-preview"] } bytes = "1.4.0" chacha20poly1305 = "0.10" diff --git a/wnfs/examples/mnemonic_based.rs b/wnfs/examples/mnemonic_based.rs index 00ab711a..cd2088ba 100644 --- a/wnfs/examples/mnemonic_based.rs +++ b/wnfs/examples/mnemonic_based.rs @@ -1,5 +1,4 @@ use anyhow::{anyhow, Result}; -use async_trait::async_trait; use bip39::{Language, Mnemonic, MnemonicType, Seed}; use chrono::Utc; use rand_chacha::ChaCha12Rng; @@ -187,8 +186,6 @@ impl SeededExchangeKey { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateKey for SeededExchangeKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let padding = Oaep::new::(); @@ -196,8 +193,6 @@ impl PrivateKey for SeededExchangeKey { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl ExchangeKey for PublicExchangeKey { async fn encrypt(&self, data: &[u8]) -> Result> { let padding = Oaep::new::(); diff --git a/wnfs/examples/tiered_blockstores.rs b/wnfs/examples/tiered_blockstores.rs index 3e36bb0a..4cfab8d4 100644 --- a/wnfs/examples/tiered_blockstores.rs +++ b/wnfs/examples/tiered_blockstores.rs @@ -4,7 +4,6 @@ //! work with high latency. use anyhow::Result; -use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; use libipld_core::cid::Cid; @@ -17,7 +16,7 @@ use wnfs::{ PrivateDirectory, PrivateNode, }, }; -use wnfs_common::{utils::CondSend, Storable}; +use wnfs_common::{utils::CondSend, BlockStoreError, Storable}; #[async_std::main] async fn main() -> Result<()> { @@ -96,19 +95,36 @@ struct TieredBlockStore { cold: C, } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl BlockStore for TieredBlockStore { - async fn get_block(&self, cid: &Cid) -> Result { - match self.hot.get_block(cid).await { - Ok(block) => Ok(block), - // We could technically get better about this - // and only match "NotFound" errors. - Err(_) => self.cold.get_block(cid).await, + async fn get_block(&self, cid: &Cid) -> Result { + if self.hot.has_block(cid).await? { + self.hot.get_block(cid).await + } else { + self.cold.get_block(cid).await } } - async fn put_block(&self, bytes: impl Into + CondSend, codec: u64) -> Result { - self.hot.put_block(bytes.into(), codec).await + async fn put_block( + &self, + bytes: impl Into + CondSend, + codec: u64, + ) -> Result { + self.hot.put_block(bytes, codec).await + } + + async fn put_block_keyed( + &self, + cid: Cid, + bytes: impl Into + CondSend, + ) -> Result<(), BlockStoreError> { + self.hot.put_block_keyed(cid, bytes).await + } + + async fn has_block(&self, cid: &Cid) -> Result { + if self.hot.has_block(cid).await? { + return Ok(true); + } + + self.cold.has_block(cid).await } } diff --git a/wnfs/src/lib.rs b/wnfs/src/lib.rs index bbb7c8c8..bef16225 100644 --- a/wnfs/src/lib.rs +++ b/wnfs/src/lib.rs @@ -161,7 +161,7 @@ pub const WNFS_VERSION: semver::Version = semver::Version::new(1, 0, 0); /// The result of an basic get operation. pub(crate) enum SearchResult { Missing(T, usize), - NotADir(T, usize), + NotADir(T, #[allow(unused)] usize), Found(T), } diff --git a/wnfs/src/private/directory.rs b/wnfs/src/private/directory.rs index 88c62d0c..f15a6b5a 100644 --- a/wnfs/src/private/directory.rs +++ b/wnfs/src/private/directory.rs @@ -1374,7 +1374,7 @@ impl PrivateDirectoryContent { let block = snapshot_key.encrypt(&bytes, rng)?; // Store content section in blockstore and get Cid. - store.put_block(block, CODEC_RAW).await + Ok(store.put_block(block, CODEC_RAW).await?) }) .await?) } diff --git a/wnfs/src/private/file.rs b/wnfs/src/private/file.rs index dfae4cb0..6920e5c8 100644 --- a/wnfs/src/private/file.rs +++ b/wnfs/src/private/file.rs @@ -826,7 +826,7 @@ impl PrivateFileContent { let block = snapshot_key.encrypt(&bytes, rng)?; // Store content section in blockstore and get Cid. - store.put_block(block, CODEC_RAW).await + Ok(store.put_block(block, CODEC_RAW).await?) }) .await?) } diff --git a/wnfs/src/private/forest/hamt.rs b/wnfs/src/private/forest/hamt.rs index a93796fb..cb1fed72 100644 --- a/wnfs/src/private/forest/hamt.rs +++ b/wnfs/src/private/forest/hamt.rs @@ -1,7 +1,6 @@ use super::traits::PrivateForest; use crate::error::FsError; use anyhow::Result; -use async_trait::async_trait; use libipld_core::cid::Cid; use quick_cache::sync::Cache; use rand_core::CryptoRngCore; @@ -231,8 +230,6 @@ impl HamtForest { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for HamtForest { fn empty_name(&self) -> Name { Name::empty(&self.accumulator) @@ -336,8 +333,6 @@ impl PrivateForest for HamtForest { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for Arc { fn empty_name(&self) -> Name { (**self).empty_name() @@ -397,8 +392,6 @@ impl PrivateForest for Arc { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for HamtForest { type Serializable = HamtForestSerializable; diff --git a/wnfs/src/private/forest/proofs.rs b/wnfs/src/private/forest/proofs.rs index 475a4b15..3a3d7990 100644 --- a/wnfs/src/private/forest/proofs.rs +++ b/wnfs/src/private/forest/proofs.rs @@ -1,7 +1,6 @@ use super::{hamt::HamtForest, traits::PrivateForest}; use crate::error::{FsError, VerificationError}; use anyhow::{bail, Result}; -use async_trait::async_trait; use libipld_core::cid::Cid; use std::collections::{BTreeSet, HashMap}; use wnfs_common::{ @@ -150,8 +149,6 @@ impl Default for ForestProofs { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateForest for ProvingHamtForest { fn empty_name(&self) -> Name { self.forest.empty_name() diff --git a/wnfs/src/private/forest/traits.rs b/wnfs/src/private/forest/traits.rs index 85c243cf..cc9833eb 100644 --- a/wnfs/src/private/forest/traits.rs +++ b/wnfs/src/private/forest/traits.rs @@ -4,7 +4,7 @@ use crate::{ }; use anyhow::Result; use async_stream::stream; -use async_trait::async_trait; +use futures::Future; use libipld_core::cid::Cid; use std::collections::BTreeSet; use wnfs_common::{ @@ -20,8 +20,6 @@ use wnfs_nameaccumulator::{AccumulatorSetup, ElementsProof, Name, NameAccumulato /// It also stores the accumulator setup information for running /// name accumulator operations. Upon put or remove, it'll run /// these operations for the caller. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait PrivateForest: CondSync { /// Construct what represents the empty name in this forest. /// @@ -86,42 +84,50 @@ pub trait PrivateForest: CondSync { /// assert!(forest.has_by_hash(access_key.get_label(), store).await.unwrap()); /// } /// ``` - async fn has_by_hash(&self, name_hash: &HashOutput, store: &impl BlockStore) -> Result; + fn has_by_hash( + &self, + name_hash: &HashOutput, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Check whether a certain name has any values. - async fn has(&self, name: &Name, store: &impl BlockStore) -> Result; + fn has( + &self, + name: &Name, + store: &impl BlockStore, + ) -> impl Future> + CondSend; /// Adds new encrypted values at the given key. - async fn put_encrypted( + fn put_encrypted( &mut self, name: &Name, values: I, store: &impl BlockStore, - ) -> Result + ) -> impl Future> + CondSend where I: IntoIterator + CondSend, I::IntoIter: CondSend; /// Gets the CIDs to blocks of ciphertext by hash of name. - async fn get_encrypted_by_hash<'b>( + fn get_encrypted_by_hash<'b>( &'b self, name_hash: &HashOutput, store: &impl BlockStore, - ) -> Result>>; + ) -> impl Future>>> + CondSend; /// Gets the CIDs to blocks of ciphertext by name. - async fn get_encrypted( + fn get_encrypted( &self, name: &Name, store: &impl BlockStore, - ) -> Result>>; + ) -> impl Future>>> + CondSend; /// Removes the CIDs to blocks of ciphertext by name. - async fn remove_encrypted( + fn remove_encrypted( &mut self, name: &Name, store: &impl BlockStore, - ) -> Result>>>; + ) -> impl Future>>>> + CondSend; /// Returns a stream of all private nodes that could be decrypted at given revision. /// diff --git a/wnfs/src/private/keys/access.rs b/wnfs/src/private/keys/access.rs index 3c5b1b2e..120490c6 100644 --- a/wnfs/src/private/keys/access.rs +++ b/wnfs/src/private/keys/access.rs @@ -127,12 +127,12 @@ mod snapshot_tests { use rand::Rng; use rand_chacha::ChaCha12Rng; use rand_core::SeedableRng; - use wnfs_common::{utils::SnapshotBlockStore, BlockStore}; + use testresult::TestResult; + use wnfs_common::{encode, libipld::json::DagJsonCodec}; #[async_std::test] - async fn test_access_key() { + async fn test_access_key() -> TestResult { let rng = &mut ChaCha12Rng::seed_from_u64(0); - let store = &SnapshotBlockStore::default(); let private_ref = PrivateRef::with_temporal_key(rng.gen(), TemporalKey(rng.gen()), Cid::default()); @@ -140,13 +140,18 @@ mod snapshot_tests { let temporal_access_key = AccessKey::Temporal(TemporalAccessKey::from(&private_ref)); let snapshot_access_key = AccessKey::Snapshot(SnapshotAccessKey::from(&private_ref)); - let temp_cid = store.put_serializable(&temporal_access_key).await.unwrap(); - let snap_cid = store.put_serializable(&snapshot_access_key).await.unwrap(); - - let temp_key = store.get_block_snapshot(&temp_cid).await.unwrap(); - let snap_key = store.get_block_snapshot(&snap_cid).await.unwrap(); + let temp_key = as_dag_json_value(&temporal_access_key)?; + let snap_key = as_dag_json_value(&snapshot_access_key)?; insta::assert_json_snapshot!(temp_key); insta::assert_json_snapshot!(snap_key); + + Ok(()) + } + + fn as_dag_json_value(s: impl Serialize) -> Result { + let dag_json = encode(&libipld_core::serde::to_ipld(s)?, DagJsonCodec)?; + let value = serde_json::from_slice(&dag_json)?; + Ok(value) } } diff --git a/wnfs/src/private/keys/exchange.rs b/wnfs/src/private/keys/exchange.rs index 4c589707..f8390bf0 100644 --- a/wnfs/src/private/keys/exchange.rs +++ b/wnfs/src/private/keys/exchange.rs @@ -3,11 +3,12 @@ use crate::error::RsaError; #[cfg(test)] use anyhow::anyhow; use anyhow::Result; -use async_trait::async_trait; +use futures::Future; #[cfg(test)] use rsa::{traits::PublicKeyParts, BigUint, Oaep}; #[cfg(test)] use sha2::Sha256; +use wnfs_common::utils::CondSend; //-------------------------------------------------------------------------------------------------- // Constants @@ -28,26 +29,22 @@ pub const PUBLIC_KEY_EXPONENT: u64 = 65537; /// More on exchange keys [here][key]. /// /// [key]: https://github.com/wnfs-wg/spec/blob/main/spec/shared-private-data.md#2-exchange-keys-partition -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait ExchangeKey { /// Creates an RSA public key from the public key modulus. /// /// The exponent is expected to be of the value [`PUBLIC_KEY_EXPONENT`](constant.PUBLIC_KEY_EXPONENT.html) constant. - async fn from_modulus(modulus: &[u8]) -> Result + fn from_modulus(modulus: &[u8]) -> impl Future> + CondSend where Self: Sized; /// Encrypts data with the public key. - async fn encrypt(&self, data: &[u8]) -> Result>; + fn encrypt(&self, data: &[u8]) -> impl Future>> + CondSend; } /// The `PrivateKey` trait represents a RSA private key type that can be used to decrypt data encrypted with corresponding public key. -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] pub trait PrivateKey { /// Decrypts ciphertext with the private key. - async fn decrypt(&self, ciphertext: &[u8]) -> Result>; + fn decrypt(&self, ciphertext: &[u8]) -> impl Future>> + CondSend; } pub type PublicKeyModulus = Vec; @@ -89,8 +86,6 @@ impl RsaPrivateKey { } #[cfg(test)] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl ExchangeKey for RsaPublicKey { async fn encrypt(&self, data: &[u8]) -> Result> { let padding = Oaep::new::(); @@ -110,8 +105,6 @@ impl ExchangeKey for RsaPublicKey { } #[cfg(test)] -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl PrivateKey for RsaPrivateKey { async fn decrypt(&self, ciphertext: &[u8]) -> Result> { let padding = Oaep::new::(); diff --git a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap index 2976296c..361e1745 100644 --- a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap +++ b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key-2.snap @@ -3,23 +3,19 @@ source: wnfs/src/private/keys/access.rs expression: snap_key --- { - "cid": "bafyr4ics27f5xqe7ryshkpoln5w7nkvvsdnoaw55r5gfb5earluczlil4a", - "value": { - "wnfs/share/snapshot": { - "contentCid": { - "/": "baeaaaaa" - }, - "label": { - "/": { - "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" - } - }, - "snapshotKey": { - "/": { - "bytes": "7SoxAmhQW3XHxn5UpettI89xokpcEjFXapTSkFRQleU" - } + "wnfs/share/snapshot": { + "contentCid": { + "/": "baeaaaaa" + }, + "label": { + "/": { + "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" + } + }, + "snapshotKey": { + "/": { + "bytes": "7SoxAmhQW3XHxn5UpettI89xokpcEjFXapTSkFRQleU" } } - }, - "bytes": "oXN3bmZzL3NoYXJlL3NuYXBzaG90o2VsYWJlbFggf7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooRqY29udGVudENpZNgqRQABAAAAa3NuYXBzaG90S2V5WCDtKjECaFBbdcfGflSl620jz3GiSlwSMVdqlNKQVFCV5Q==" + } } diff --git a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap index 79f2b6fe..fe57ba35 100644 --- a/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap +++ b/wnfs/src/private/keys/snapshots/wnfs__private__keys__access__snapshot_tests__access_key.snap @@ -3,23 +3,19 @@ source: wnfs/src/private/keys/access.rs expression: temp_key --- { - "cid": "bafyr4ih2hblsecub5jvxxlbkhjgmabg2mnaxzq7jtzkchd4s4sacqxwbge", - "value": { - "wnfs/share/temporal": { - "contentCid": { - "/": "baeaaaaa" - }, - "label": { - "/": { - "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" - } - }, - "temporalKey": { - "/": { - "bytes": "mmMoPLrw/bzrH2R5sZfzqI3Q2Akv5yp8VigVOHOLB+I" - } + "wnfs/share/temporal": { + "contentCid": { + "/": "baeaaaaa" + }, + "label": { + "/": { + "bytes": "f7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooQ" + } + }, + "temporalKey": { + "/": { + "bytes": "mmMoPLrw/bzrH2R5sZfzqI3Q2Akv5yp8VigVOHOLB+I" } } - }, - "bytes": "oXN3bmZzL3NoYXJlL3RlbXBvcmFso2VsYWJlbFggf7J7lBYC0B0RVCIRE0/HGqyuVON+fQB7u3tV7/BiooRqY29udGVudENpZNgqRQABAAAAa3RlbXBvcmFsS2V5WCCaYyg8uvD9vOsfZHmxl/OojdDYCS/nKnxWKBU4c4sH4g==" + } } diff --git a/wnfs/src/private/node/header.rs b/wnfs/src/private/node/header.rs index f9dafe7a..fa850b02 100644 --- a/wnfs/src/private/node/header.rs +++ b/wnfs/src/private/node/header.rs @@ -165,7 +165,7 @@ impl PrivateNodeHeader { let temporal_key = self.derive_temporal_key(); let cbor_bytes = serde_ipld_dagcbor::to_vec(&self.to_serializable(forest))?; let ciphertext = temporal_key.key_wrap_encrypt(&cbor_bytes)?; - store.put_block(ciphertext, CODEC_RAW).await + Ok(store.put_block(ciphertext, CODEC_RAW).await?) } pub(crate) fn to_serializable( diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 8c534e8f..927d6a02 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -8,12 +8,11 @@ use crate::{ }; use anyhow::{bail, ensure, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use libipld_core::cid::Cid; use std::collections::{BTreeMap, BTreeSet}; use wnfs_common::{ - utils::{error, Arc}, + utils::{boxed_fut, error, Arc}, BlockStore, Metadata, NodeType, Storable, }; @@ -827,8 +826,6 @@ impl Clone for PublicDirectory { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicDirectory { type Serializable = PublicNodeSerializable; @@ -836,7 +833,8 @@ impl Storable for PublicDirectory { let userland = { let mut map = BTreeMap::new(); for (name, link) in self.userland.iter() { - map.insert(name.clone(), link.resolve_cid(store).await?); + // Boxing the future due to recursion + map.insert(name.clone(), boxed_fut(link.resolve_cid(store)).await?); } map }; @@ -894,7 +892,7 @@ mod tests { use chrono::Utc; use libipld_core::ipld::Ipld; use testresult::TestResult; - use wnfs_common::MemoryBlockStore; + use wnfs_common::{decode, libipld::cbor::DagCborCodec, MemoryBlockStore}; #[async_std::test] async fn look_up_can_fetch_file_added_to_directory() -> TestResult { @@ -1245,10 +1243,14 @@ mod tests { root_dir.mkdir(&["test".into()], time, store).await.unwrap(); - let ipld = store - .get_deserializable::(&root_dir.store(store).await.unwrap()) - .await - .unwrap(); + let ipld: Ipld = decode( + &store + .get_block(&root_dir.store(store).await.unwrap()) + .await + .unwrap(), + DagCborCodec, + ) + .unwrap(); match ipld { Ipld::Map(map) => match map.get("wnfs/pub/dir") { Some(Ipld::Map(content)) => match content.get("previous") { diff --git a/wnfs/src/public/file.rs b/wnfs/src/public/file.rs index 8720a6da..b5f4108d 100644 --- a/wnfs/src/public/file.rs +++ b/wnfs/src/public/file.rs @@ -4,7 +4,6 @@ use super::{PublicFileSerializable, PublicNodeSerializable}; use crate::{error::FsError, is_readable_wnfs_version, traits::Id, WNFS_VERSION}; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use futures::{AsyncRead, AsyncReadExt}; use libipld_core::cid::Cid; @@ -422,8 +421,6 @@ impl PublicFile { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicFile { type Serializable = PublicNodeSerializable; diff --git a/wnfs/src/public/link.rs b/wnfs/src/public/link.rs index 34048e65..234f094e 100644 --- a/wnfs/src/public/link.rs +++ b/wnfs/src/public/link.rs @@ -52,31 +52,25 @@ impl PublicLink { /// Gets the Cid stored in type. It attempts to get it from the store if it is not present in type. #[inline] - pub async fn resolve_cid(&self, store: &(impl BlockStore + ?Sized)) -> Result { + pub async fn resolve_cid(&self, store: &impl BlockStore) -> Result { self.0.resolve_cid(store).await } /// Gets the value stored in link. It attempts to get it from the store if it is not present in link. #[inline] - pub async fn resolve_value(&self, store: &(impl BlockStore + ?Sized)) -> Result<&PublicNode> { + pub async fn resolve_value(&self, store: &impl BlockStore) -> Result<&PublicNode> { self.0.resolve_value(store).await } /// Gets mut value stored in link. It attempts to get it from the store if it is not present in link. #[inline] - pub async fn resolve_value_mut( - &mut self, - store: &(impl BlockStore + ?Sized), - ) -> Result<&mut PublicNode> { + pub async fn resolve_value_mut(&mut self, store: &impl BlockStore) -> Result<&mut PublicNode> { self.0.resolve_value_mut(store).await } /// Gets an owned value from type. It attempts to it get from the store if it is not present in type. #[inline] - pub async fn resolve_owned_value( - self, - store: &(impl BlockStore + ?Sized), - ) -> Result { + pub async fn resolve_owned_value(self, store: &impl BlockStore) -> Result { self.0.resolve_owned_value(store).await } diff --git a/wnfs/src/public/node/node.rs b/wnfs/src/public/node/node.rs index 4eb9ea81..fb0bc4e1 100644 --- a/wnfs/src/public/node/node.rs +++ b/wnfs/src/public/node/node.rs @@ -8,7 +8,6 @@ use crate::{ }; use anyhow::{bail, Result}; use async_once_cell::OnceCell; -use async_trait::async_trait; use chrono::{DateTime, Utc}; use libipld_core::cid::Cid; use std::collections::BTreeSet; @@ -267,8 +266,6 @@ impl From for PublicNode { } } -#[cfg_attr(not(target_arch = "wasm32"), async_trait)] -#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] impl Storable for PublicNode { type Serializable = PublicNodeSerializable; diff --git a/wnfs/src/root_tree.rs b/wnfs/src/root_tree.rs index e11b7e3b..4d959a6f 100644 --- a/wnfs/src/root_tree.rs +++ b/wnfs/src/root_tree.rs @@ -25,6 +25,8 @@ use std::collections::HashMap; #[cfg(test)] use wnfs_common::MemoryBlockStore; use wnfs_common::{ + decode, encode, + libipld::cbor::DagCborCodec, utils::{Arc, CondSend}, BlockStore, Metadata, Storable, }; @@ -309,7 +311,11 @@ where version: WNFS_VERSION, }; - store.put_serializable(&serializable).await + let cid = store + .put_block(encode(&serializable, DagCborCodec)?, DagCborCodec.into()) + .await?; + + Ok(cid) } pub async fn load( @@ -318,7 +324,8 @@ where rng: R, private_map: HashMap, Arc>, ) -> Result> { - let deserialized: RootTreeSerializable = store.get_deserializable(cid).await?; + let deserialized: RootTreeSerializable = + decode(&store.get_block(cid).await?, DagCborCodec)?; let forest = Arc::new(HamtForest::load(&deserialized.forest, store).await?); let public_root = Arc::new(PublicDirectory::load(&deserialized.public, store).await?); let exchange_root = Arc::new(PublicDirectory::load(&deserialized.exchange, store).await?);