diff --git a/Cargo.lock b/Cargo.lock index 61d1ea0c..d465b04e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -239,7 +239,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.0", "constant_time_eq", ] @@ -306,6 +306,12 @@ version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.0" @@ -355,7 +361,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen", ] @@ -389,7 +395,7 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "lazy_static", ] @@ -413,6 +419,41 @@ dependencies = [ "syn", ] +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.3.2" @@ -470,6 +511,18 @@ dependencies = [ "instant", ] +[[package]] +name = "field_names" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca4fdab1b9b7e274e7de51202e37f9cfa542b28c77f8d09b817d77a726b4807" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fixedbitset" version = "0.4.1" @@ -534,7 +587,7 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi", ] @@ -584,6 +637,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" version = "1.8.1" @@ -600,7 +659,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -762,7 +821,7 @@ version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "value-bag", ] @@ -772,6 +831,12 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memory_units" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8452105ba047068f40ff7093dd1d9da90898e63dd61736462e9cdda6a90ad3c3" + [[package]] name = "multibase" version = "0.9.1" @@ -877,7 +942,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "redox_syscall", "smallvec", @@ -912,7 +977,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "log", "wepoll-ffi", @@ -1120,7 +1185,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "cpufeatures", "digest", ] @@ -1157,6 +1222,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.90" @@ -1186,7 +1257,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "fastrand", "libc", "redox_syscall", @@ -1292,7 +1363,7 @@ version = "0.2.80" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "wasm-bindgen-macro", ] @@ -1317,7 +1388,7 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "js-sys", "wasm-bindgen", "web-sys", @@ -1382,12 +1453,15 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "cfg-if 1.0.0", "chrono", + "console_error_panic_hook", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "wasm-bindgen-test", "web-sys", + "wee_alloc", "wnfs", ] @@ -1401,6 +1475,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wee_alloc" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb3b5a6b2bb17cb6ad44a2e68a43e8d2722c997da10e928665c72ec6c0a0b8e" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "memory_units", + "winapi", +] + [[package]] name = "wepoll-ffi" version = "0.1.2" @@ -1495,6 +1581,7 @@ dependencies = [ "async-std", "async-trait", "chrono", + "field_names", "hashbrown 0.12.0", "libipld", "multihash", diff --git a/crates/fs/Cargo.toml b/crates/fs/Cargo.toml index 0d50ddf8..22e5b2fa 100644 --- a/crates/fs/Cargo.toml +++ b/crates/fs/Cargo.toml @@ -25,6 +25,7 @@ hashbrown = "0.12.0" async-trait = "0.1.53" async-std = { version = "1.11.0", features = ["attributes"] } async-recursion = "1.0.0" +field_names = "0.2.0" [lib] path = "lib.rs" diff --git a/crates/fs/README.md b/crates/fs/README.md index c5b53dd8..22bb3999 100644 --- a/crates/fs/README.md +++ b/crates/fs/README.md @@ -1 +1,17 @@ -## The FileSystem +## The FileSystem + +### Building the Project + +- Build project + +```bash +cargo build --release +``` + +### Testing the Project + +- Build project + +```bash +cargo test --release +``` diff --git a/crates/fs/common/blockstore.rs b/crates/fs/common/blockstore.rs index 4e3f6c16..348c0498 100644 --- a/crates/fs/common/blockstore.rs +++ b/crates/fs/common/blockstore.rs @@ -19,25 +19,27 @@ use super::FsError; //-------------------------------------------------------------------------------------------------- /// For types that implement getting a block from a CID. -#[async_trait] +#[async_trait(?Send)] pub trait BlockStoreLookup { - async fn get_block<'a>(&'a self, cid: &Cid) -> Result>; + async fn get_block<'a>(&'a self, cid: &Cid) -> Result>>; } -/// For types that implement loading a cbor model from a blockstore using a CID. -#[async_trait] +/// For types that implement loading decodable object from a blockstore using a CID. +#[async_trait(?Send)] pub trait BlockStoreCidLoad { - /// Loads a cbor model from the store with provided CID. + /// Loads a decodable object from the store with provided CID. async fn load, C: Codec>(&self, cid: &Cid, decoder: C) -> Result; } -/// For types that implement block store operations. -#[async_trait] +/// For types that implement block store operations like adding, getting content from the store. +#[async_trait(?Send)] pub trait BlockStore: BlockStoreLookup + BlockStoreCidLoad { async fn put_block(&mut self, bytes: Vec, codec: IpldCodec) -> Result; } -/// An in-memory block store to simulate IPFS. IPFS is basically an glorified HashMap. +/// An in-memory block store to simulate IPFS. +/// +/// IPFS is basically an glorified HashMap. #[derive(Debug, Default)] pub struct MemoryBlockStore(HashMap>); @@ -52,7 +54,7 @@ impl MemoryBlockStore { } } -#[async_trait] +#[async_trait(?Send)] impl BlockStore for MemoryBlockStore { /// Stores an array of bytes in the block store. async fn put_block(&mut self, bytes: Vec, codec: IpldCodec) -> Result { @@ -65,10 +67,10 @@ impl BlockStore for MemoryBlockStore { } } -#[async_trait] +#[async_trait(?Send)] impl BlockStoreLookup for MemoryBlockStore { /// Retrieves an array of bytes from the block store with given CID. - async fn get_block<'a>(&'a self, cid: &Cid) -> Result> { + async fn get_block<'a>(&'a self, cid: &Cid) -> Result>> { let bytes = self .0 .get(&cid.to_string()) @@ -78,9 +80,9 @@ impl BlockStoreLookup for MemoryBlockStore { } } -#[async_trait] +#[async_trait(?Send)] impl BlockStoreCidLoad for MemoryBlockStore { - /// Loads a cbor-encoded data from the store with provided CID. + /// Loads a CBOR-encoded data from the store with provided CID. async fn load, C: Codec>(&self, cid: &Cid, decoder: C) -> Result { let bytes = self.get_block(cid).await?; let decoded = decoder.decode(bytes.as_ref())?; diff --git a/crates/fs/common/error.rs b/crates/fs/common/error.rs index dedc8cf9..a5a6b9cf 100644 --- a/crates/fs/common/error.rs +++ b/crates/fs/common/error.rs @@ -7,11 +7,16 @@ use std::{ use anyhow::Result; +/// File system errors. #[derive(Debug, Clone, PartialEq, Eq)] pub enum FsError { CIDNotFoundInBlockstore, InvalidPath, - NodeNotFound, + NotAFile, + NotADirectory, + NotFound, + FileAlreadyExists, + UndecodableCborData(String), } impl std::error::Error for FsError {} diff --git a/crates/fs/common/metadata.rs b/crates/fs/common/metadata.rs index 069b0158..dc21e580 100644 --- a/crates/fs/common/metadata.rs +++ b/crates/fs/common/metadata.rs @@ -1,24 +1,30 @@ //! File system metadata. use std::{ + cmp::Ordering, io::{Read, Seek, Write}, str::FromStr, }; use anyhow::Result; use chrono::{DateTime, Utc}; +use field_names::FieldNames; use libipld::{ - cbor::DagCborCodec, + cbor::{cbor::MajorKind, decode, encode, DagCborCodec}, codec::{Decode, Encode}, DagCbor, }; use semver::Version; +use crate::FsError; + +use super::error; + //-------------------------------------------------------------------------------------------------- // Type Definitions //-------------------------------------------------------------------------------------------------- -/// Represents the type of node in the UnixFS file system. +/// The different types a UnixFS can be. /// /// See https://docs.ipfs.io/concepts/file-systems/#unix-file-system-unixfs #[derive(Debug, Clone, PartialEq, Eq, Copy, DagCbor)] @@ -65,7 +71,7 @@ pub struct UnixFsMetadata { } /// The metadata of a node on the WNFS file system. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, FieldNames)] pub struct Metadata { pub(crate) unix_fs: UnixFsMetadata, pub(crate) version: Version, @@ -101,18 +107,77 @@ impl Metadata { impl Decode for Metadata { fn decode(c: DagCborCodec, r: &mut R) -> Result { - let unix_fs = UnixFsMetadata::decode(c, r)?; - let version_str = String::decode(c, r)?; - let version = Version::from_str(&version_str)?; + // Ensure the major kind is a map. + let major = decode::read_major(r)?; + if major.kind() != MajorKind::Map { + return error(FsError::UndecodableCborData("Unsupported major".into())); + } - Ok(Self { unix_fs, version }) + let _ = decode::read_uint(r, major)?; + + // Ordering the fields by name based on RFC-7049 which is also what libipld uses. + let mut cbor_order: Vec<&'static str> = Vec::from_iter(Metadata::FIELDS); + cbor_order.sort_unstable_by(|&a, &b| match a.len().cmp(&b.len()) { + Ordering::Greater => Ordering::Greater, + Ordering::Less => Ordering::Less, + Ordering::Equal => a.cmp(b), + }); + + // Iterate over the fields. + let mut unix_fs = None; + let mut version = String::new(); + for field in cbor_order.iter() { + // Decode field name. + String::decode(c, r)?; + + // Decode field value. + match *field { + "unix_fs" => { + unix_fs = Some(UnixFsMetadata::decode(c, r)?); + } + "version" => { + version = String::decode(c, r)?; + } + _ => unreachable!(), + } + } + + Ok(Self { + unix_fs: unix_fs + .ok_or_else(|| FsError::UndecodableCborData("Missing unix_fs".into()))?, + version: Version::from_str(&version)?, + }) } } impl Encode for Metadata { fn encode(&self, c: DagCborCodec, w: &mut W) -> Result<()> { - self.unix_fs.encode(c, w)?; - self.version.to_string().encode(c, w)?; + // Write the major of the section being written. + encode::write_u64(w, MajorKind::Map, Metadata::FIELDS.len() as u64)?; + + // Ordering the fields by name based on RFC-7049 which is also what libipld uses. + let mut cbor_order: Vec<&'static str> = Vec::from_iter(Metadata::FIELDS); + cbor_order.sort_unstable_by(|&a, &b| match a.len().cmp(&b.len()) { + Ordering::Greater => Ordering::Greater, + Ordering::Less => Ordering::Less, + Ordering::Equal => a.cmp(b), + }); + + // Iterate over the fields. + for field in cbor_order.iter() { + // Encode field name. + field.encode(c, w)?; + // Encode field value. + match *field { + "unix_fs" => { + self.unix_fs.encode(c, w)?; + } + "version" => { + self.version.to_string().encode(c, w)?; + } + _ => unreachable!(), + } + } Ok(()) } @@ -120,8 +185,28 @@ impl Encode for Metadata { #[cfg(test)] mod metadata_tests { + use std::io::Cursor; + + use chrono::Utc; + use libipld::{ + cbor::DagCborCodec, + codec::{Decode, Encode}, + }; + + use crate::{Metadata, UnixFsNodeKind}; + #[async_std::test] async fn metadata_encode_decode_successful() { - // TODO(appcypher): Implement this. + let metadata = Metadata::new(Utc::now(), UnixFsNodeKind::File); + + let mut encoded_bytes = vec![]; + + metadata.encode(DagCborCodec, &mut encoded_bytes).unwrap(); + + let mut cursor = Cursor::new(encoded_bytes); + + let decoded_metadata = Metadata::decode(DagCborCodec, &mut cursor).unwrap(); + + assert_eq!(metadata, decoded_metadata); } } diff --git a/crates/fs/public/directory.rs b/crates/fs/public/directory.rs index 0aa4d58e..d28fa21e 100644 --- a/crates/fs/public/directory.rs +++ b/crates/fs/public/directory.rs @@ -1,8 +1,11 @@ //! Public fs directory node. use std::{ + cell::RefCell, + cmp::Ordering, collections::BTreeMap, io::{Read, Seek}, + mem, rc::Rc, }; @@ -10,8 +13,9 @@ use crate::{error, BlockStore, FsError, Metadata, UnixFsNodeKind}; use anyhow::Result; use async_recursion::async_recursion; use chrono::{DateTime, Utc}; +use field_names::FieldNames; use libipld::{ - cbor::DagCborCodec, + cbor::{cbor::MajorKind, decode, encode, DagCborCodec}, codec::{Decode, Encode}, Cid, IpldCodec, }; @@ -19,7 +23,7 @@ use libipld::{ use super::{Link, PublicNode}; /// A directory in a WNFS public file system. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, FieldNames)] pub struct PublicDirectory { pub(crate) metadata: Metadata, pub(crate) userland: BTreeMap, @@ -41,31 +45,34 @@ impl PublicDirectory { &self, path_segments: &[String], store: &B, - ) -> Result> { + ) -> Result>>> { if path_segments.is_empty() { return error(FsError::InvalidPath); } - let mut working_node: Rc = Rc::new(PublicNode::Dir(self.clone())); + // TODO(appcypher): Any way to avoid this clone? + let mut working_node = Some(Rc::new(RefCell::new(PublicNode::Dir(self.clone())))); // Iterate over the path segments until we get the node of the last segment. for (index, segment) in path_segments.iter().enumerate() { // Cast working node to directory. - let dir = working_node.as_dir(); + let node_rc = working_node.unwrap(); + let node_ref = node_rc.borrow(); + let dir = node_ref.as_dir(); // Fetch node representing path segment in working directory. - if let Some(node) = dir.lookup_node(segment, store).await? { - match node.as_ref() { + if let Some(found_node) = dir.lookup_node(segment, store).await? { + match *found_node.borrow() { + // If the node is a directory, set it as the working node. PublicNode::Dir(_) => { - // If the node is a directory, set it as the working node. - working_node = Rc::clone(&node); + working_node = Some(Rc::clone(&node_rc)); } + // If the node is a file, we return it if it's the last segment. PublicNode::File(_) => { - // If the node is a file, we return it if it's the last segment. if index != path_segments.len() - 1 { return error(FsError::InvalidPath); } - working_node = Rc::clone(&node); + working_node = Some(Rc::clone(&node_rc)); break; } } @@ -74,8 +81,8 @@ impl PublicDirectory { continue; } - // If the node is not found, we return an error. - return error(FsError::NodeNotFound); + // If the node is not found, we return an none. + return Ok(None); } Ok(working_node) @@ -88,7 +95,7 @@ impl PublicDirectory { &self, path_segment: &str, store: &B, - ) -> Result>> { + ) -> Result>>> { Ok(match self.userland.get(path_segment) { Some(link) => Some(link.resolve(store).await?), None => None, @@ -98,24 +105,54 @@ impl PublicDirectory { /// Encode the directory as a CBOR object. pub async fn encode(&self, store: &mut B) -> Result> { let mut bytes = Vec::new(); - self.metadata.encode(DagCborCodec, &mut bytes)?; - let new_userland = { - let mut tmp = BTreeMap::new(); - for (k, link) in self.userland.iter() { - let cid = link.seal(store).await?; - tmp.insert(k.clone(), cid); - } - tmp - }; + // Write the major of the section being written. + encode::write_u64( + &mut bytes, + MajorKind::Map, + PublicDirectory::FIELDS.len() as u64, + )?; + + // Ordering the fields by name based on RFC-7049 which is also what libipld uses. + let mut cbor_order: Vec<&'static str> = Vec::from_iter(PublicDirectory::FIELDS); + cbor_order.sort_unstable_by(|&a, &b| match a.len().cmp(&b.len()) { + Ordering::Greater => Ordering::Greater, + Ordering::Less => Ordering::Less, + Ordering::Equal => a.cmp(b), + }); + + // Iterate over the fields. + for field in cbor_order.iter() { + // Encode field name. + field.encode(DagCborCodec, &mut bytes)?; + // Encode field value. + match *field { + "metadata" => { + self.metadata.encode(DagCborCodec, &mut bytes)?; + } + "userland" => { + let new_userland = { + let mut tmp = BTreeMap::new(); + for (k, link) in self.userland.iter() { + let cid = link.seal(store).await?; + tmp.insert(k.clone(), cid); + } + tmp + }; - new_userland.encode(DagCborCodec, &mut bytes)?; + new_userland.encode(DagCborCodec, &mut bytes)?; + } + "previous" => { + self.previous.encode(DagCborCodec, &mut bytes)?; + } + _ => unreachable!(), + } + } - self.previous.encode(DagCborCodec, &mut bytes)?; Ok(bytes) } - /// Stores WNFS directory as block(s) in chosen block store. + /// Stores a directory as block(s) in provided block store. /// /// This function can be recursive if the directory contains other directories. #[async_recursion(?Send)] @@ -124,32 +161,173 @@ impl PublicDirectory { store.put_block(bytes, IpldCodec::DagCbor).await } + /// Reads a file from the directory. + pub async fn read( + &self, + path_segments: &[String], + store: &mut B, + ) -> Result { + let node = self.get_node(path_segments, store).await?; + match node { + Some(node_rc) => match &*node_rc.borrow() { + PublicNode::File(file) => Ok(file.userland), + _ => error(FsError::NotAFile), + }, + _ => error(FsError::NotFound), + } + } + + pub async fn upsert() -> Result { + // TODO(appcypher): Implement this. + todo!() + } + /// Writes a file to the directory. /// /// Rather than mutate the directory directly, we create a new directory and return it. pub async fn write( &self, - path: &[String], - content_cid: &Cid, - store: &mut B, + path_segments: &[String], + time: DateTime, + content_cid: Cid, + store: &B, ) -> Result { + // Get the path segments for the file's parent directory. + let parent_path = match path_segments.split_last() { + Some((_, rest)) => rest, + None => return error(FsError::InvalidPath), + }; + + let mut directory = self.mkdir(parent_path, time, store).await?; + + todo!() + } + + /// Returns the children links of a directory. + pub async fn ls(&self) -> Result> { // TODO(appcypher): Implement this. todo!() } + + /// Creates a new directory with the given path. + pub async fn mkdir( + &self, + path_segments: &[String], + time: DateTime, + store: &B, + ) -> Result { + if path_segments.is_empty() { + return error(FsError::InvalidPath); + } + + // Clone the directory and create a new root. + let new_root = self.clone(); + let mut working_node = Rc::new(RefCell::new(PublicNode::Dir(new_root))); + + // Iterate over the path segments until the last segment. + for (index, segment) in path_segments.iter().enumerate() { + let mut _next_node = None; + + // This block helps us reduce the lifetime scope of the mutable borrow of working_node. + { + // Cast working node to directory. + let mut node_mut = working_node.borrow_mut(); + let dir = node_mut.as_mut_dir(); + + // Fetch node representing path segment in working directory. + if let Some(found_node) = dir.lookup_node(segment, store).await? { + match *found_node.borrow() { + // If the node is a directory, set it as the next working node. + PublicNode::Dir(_) => { + _next_node = Some(Rc::clone(&found_node)); + } + // If the node is a file, we return an error. + PublicNode::File(_) => { + return if index == path_segments.len() - 1 { + error(FsError::FileAlreadyExists) + } else { + error(FsError::InvalidPath) + } + } + } + } else { + // If the node is not found, we create it. + let new_node_rc = + Rc::new(RefCell::new(PublicNode::Dir(PublicDirectory::new(time)))); + + // Insert the new node into the working directory. + dir.userland + .insert(segment.clone(), Link::Node(Rc::clone(&new_node_rc))); + + // And set that as the new working node. + _next_node = Some(new_node_rc); + } + } + + working_node = _next_node.unwrap(); + } + + // Get the PublicNode behind the working_node `Rc`. + let node = mem::replace( + &mut *working_node.borrow_mut(), + PublicNode::Dir(PublicDirectory::new(time)), + ); + + Ok(node.into_dir()) + } } +// Decoding CBOR-encoded PublicDirectory from bytes. impl Decode for PublicDirectory { fn decode(c: DagCborCodec, r: &mut R) -> Result { - let metadata = Metadata::decode(c, r)?; - let userland = BTreeMap::::decode(c, r)? - .into_iter() - .map(|(k, cid)| (k, Link::Cid(cid))) - .collect(); + // Ensure the major kind is a map. + let major = decode::read_major(r)?; + if major.kind() != MajorKind::Map { + return error(FsError::UndecodableCborData("Unsupported major".into())); + } - let previous = Option::::decode(c, r)?; + // Decode the length of the map. + let _ = decode::read_uint(r, major)?; + + // Ordering the fields by name based on RFC-7049 which is also what libipld uses. + let mut cbor_order: Vec<&'static str> = Vec::from_iter(PublicDirectory::FIELDS); + cbor_order.sort_unstable_by(|&a, &b| match a.len().cmp(&b.len()) { + Ordering::Greater => Ordering::Greater, + Ordering::Less => Ordering::Less, + Ordering::Equal => a.cmp(b), + }); + + // Iterate over the fields. + let mut metadata = None; + let mut userland = BTreeMap::new(); + let mut previous = None; + + // Iterate over the fields. + for field in cbor_order.iter() { + // Decode field name. + String::decode(c, r)?; + + // Decode field value. + match *field { + "metadata" => { + metadata = Some(Metadata::decode(c, r)?); + } + "userland" => { + userland = BTreeMap::<_, Cid>::decode(c, r)? + .into_iter() + .map(|(k, cid)| (k, Link::Cid(cid))) + .collect(); + } + "previous" => { + previous = >::decode(c, r)?; + } + _ => unreachable!(), + } + } Ok(Self { - metadata, + metadata: metadata + .ok_or_else(|| FsError::UndecodableCborData("Missing unix_fs".into()))?, userland, previous, }) @@ -161,28 +339,36 @@ mod public_directory_tests { use std::io::Cursor; use super::*; - use crate::{BlockStoreLookup, MemoryBlockStore}; + use crate::{public::PublicFile, BlockStoreLookup, MemoryBlockStore}; use chrono::Utc; - // #[async_std::test] - // async fn files_added_to_directory_looked_up_unsuccessful() { - // let root = PublicDirectory::new(Utc::now()); + #[async_std::test] + async fn files_added_to_directory_looked_up_unsuccessful() { + let root = PublicDirectory::new(Utc::now()); - // let mut store = MemoryBlockStore::default(); + let mut store = MemoryBlockStore::default(); + + let content_cid = Cid::default(); - // let content_cid = &Cid::default(); + let time = Utc::now(); - // let root = root - // .write(&[String::from("text.txt")], content_cid, &mut store) - // .await - // .unwrap(); + // let root = root + // .write(&[String::from("text.txt")], time, content_cid, &mut store) + // .await + // .unwrap(); - // let node = root.lookup_node("text.txt", &store).await; + // let node = root.lookup_node("text.txt", &store).await; - // assert!(node.is_ok()); + // assert!(node.is_ok()); - // assert_eq!(node.unwrap(), None); - // } + // assert_eq!( + // node.unwrap(), + // Some(Rc::new(PublicNode::File(PublicFile::new( + // time, + // content_cid + // )))) + // ); + } #[async_std::test] async fn files_not_added_to_directory_not_looked_up_unsuccessful() { @@ -207,7 +393,7 @@ mod public_directory_tests { let bytes = store.get_block(&cid).await.unwrap(); - let mut cursor = Cursor::new(bytes); + let mut cursor = Cursor::new(bytes.as_ref()); let decoded_root = PublicDirectory::decode(DagCborCodec, &mut cursor).unwrap(); diff --git a/crates/fs/public/file.rs b/crates/fs/public/file.rs index 6a84714c..474d69d3 100644 --- a/crates/fs/public/file.rs +++ b/crates/fs/public/file.rs @@ -24,7 +24,7 @@ impl PublicFile { } } - /// Stores WNFS block(s) in chosen block store. + /// Stores a file as block(s) in provided block store. pub async fn store(&self, store: &mut B) -> Result { let bytes = { let mut tmp = vec![]; diff --git a/crates/fs/public/link.rs b/crates/fs/public/link.rs index 791066b0..7f544dcc 100644 --- a/crates/fs/public/link.rs +++ b/crates/fs/public/link.rs @@ -1,6 +1,6 @@ //! Node link. -use std::rc::Rc; +use std::{cell::RefCell, rc::Rc}; use anyhow::Result; use libipld::{cbor::DagCborCodec, Cid}; @@ -14,16 +14,16 @@ use crate::BlockStore; #[derive(Debug, Clone, PartialEq, Eq)] pub enum Link { Cid(Cid), - Node(Rc), + Node(Rc>), } impl Link { // Resolves a CID link in the file system to a node. - pub async fn resolve(&self, store: &B) -> Result> { + pub async fn resolve(&self, store: &B) -> Result>> { Ok(match self { Link::Cid(cid) => { let node = store.load(cid, DagCborCodec).await?; - Rc::new(node) + Rc::new(RefCell::new(node)) } Link::Node(node) => Rc::clone(node), }) @@ -33,7 +33,7 @@ impl Link { pub async fn seal(&self, store: &mut B) -> Result { Ok(match self { Link::Cid(cid) => *cid, - Link::Node(node) => node.store(store).await?, + Link::Node(node) => node.borrow().store(store).await?, }) } } @@ -42,11 +42,11 @@ impl Link { mod public_link_tests { #[async_std::test] async fn node_link_sealed_successfully() { - // TODO(appcypher): Implement. + // TODO(appcypher): Implement this. } #[async_std::test] async fn cid_link_resolved_successfully() { - // TODO(appcypher): Implement. + // TODO(appcypher): Implement this. } } diff --git a/crates/fs/public/node.rs b/crates/fs/public/node.rs index 303c8f64..3dc1b37e 100644 --- a/crates/fs/public/node.rs +++ b/crates/fs/public/node.rs @@ -1,15 +1,15 @@ //! Public file system in-memory representation. use std::{ - collections::BTreeMap, - io::{Read, Seek}, + io::{Cursor, Read, Seek}, + result, }; use anyhow::Result; use libipld::{cbor::DagCborCodec, codec::Decode, Cid}; -use super::{Link, PublicDirectory, PublicFile}; -use crate::{common::BlockStore, Metadata, UnixFsNodeKind}; +use super::{PublicDirectory, PublicFile}; +use crate::common::BlockStore; /// A node in a WNFS public file system. This can either be a file or a directory. #[derive(Debug, Clone, PartialEq, Eq)] @@ -27,6 +27,18 @@ impl PublicNode { }) } + /// Casts a node to an owned directory. + /// + /// # Panics + /// + /// Panics if the node is not a directory. + pub fn into_dir(self) -> PublicDirectory { + match self { + PublicNode::Dir(dir) => dir, + _ => unreachable!(), + } + } + /// Casts a node to a directory. /// /// # Panics @@ -38,33 +50,39 @@ impl PublicNode { _ => unreachable!(), } } + + /// Casts a node to a mutable directory. + /// + /// # Panics + /// + /// Panics if the node is not a directory. + pub fn as_mut_dir(&mut self) -> &mut PublicDirectory { + match self { + PublicNode::Dir(dir) => dir, + _ => unreachable!(), + } + } } impl Decode for PublicNode { fn decode(c: DagCborCodec, r: &mut R) -> Result { - let metadata = Metadata::decode(c, r)?; - let node = if matches!(metadata.unix_fs.kind, UnixFsNodeKind::File) { - let userland = Cid::decode(c, r)?; - let previous = >::decode(c, r)?; - - PublicNode::File(PublicFile { - metadata, - userland, - previous, - }) - } else { - let userland = BTreeMap::::decode(c, r)? - .into_iter() - .map(|(k, cid)| (k, Link::Cid(cid))) - .collect(); - - let previous = >::decode(c, r)?; - - PublicNode::Dir(PublicDirectory { - metadata, - userland, - previous, - }) + // NOTE(appcypher): There is really no great way to seek or peek at the data behind `r :: R: Read + Seek`. + // So we just copy the whole data behind the opaque type which allows us to cursor over the data multiple times. + // It is not ideal but it works. + let bytes: Vec = r.bytes().collect::>()?; + + // We first try to decode as a file. + let mut try_file_cursor = Cursor::new(bytes); + let try_file_decode = PublicFile::decode(c, &mut try_file_cursor); + + let node = match try_file_decode { + Ok(file) => PublicNode::File(file), + _ => { + // If the file decode failed, we try to decode as a directory. + let mut cursor = Cursor::new(try_file_cursor.into_inner()); + let dir = PublicDirectory::decode(c, &mut cursor)?; + PublicNode::Dir(dir) + } }; Ok(node) @@ -83,24 +101,20 @@ mod public_node_tests { MemoryBlockStore, }; - // #[async_std::test] - // async fn encoded_public_file_decoded_successfully() { - // let file = PublicFile::new(Utc::now(), Cid::default()); - - // dbg!(&file); - - // let mut encoded_bytes = vec![]; + #[async_std::test] + async fn encoded_public_file_decoded_successfully() { + let file = PublicFile::new(Utc::now(), Cid::default()); - // file.encode(DagCborCodec, &mut encoded_bytes).unwrap(); + let mut encoded_bytes = vec![]; - // dbg!(format!("{:02x?}", encoded_bytes)); + file.encode(DagCborCodec, &mut encoded_bytes).unwrap(); - // let mut cursor = Cursor::new(encoded_bytes); + let mut cursor = Cursor::new(encoded_bytes); - // let decoded_file = PublicNode::decode(DagCborCodec, &mut cursor).unwrap(); + let decoded_file = PublicNode::decode(DagCborCodec, &mut cursor).unwrap(); - // assert_eq!(PublicNode::File(file), decoded_file); - // } + assert_eq!(PublicNode::File(file), decoded_file); + } #[async_std::test] async fn encoded_public_directory_decoded_successfully() { diff --git a/crates/wasm/Cargo.toml b/crates/wasm/Cargo.toml index 5507ffe1..3b8e0788 100644 --- a/crates/wasm/Cargo.toml +++ b/crates/wasm/Cargo.toml @@ -17,16 +17,19 @@ authors = ["The Fission Authors"] [dependencies] wnfs = { path = "../fs", version = "0.1.0" } -wasm-bindgen = { version = "0.2.79", optional = true } -wasm-bindgen-futures = { version = "0.4.29", optional = true } -js-sys = { version = "0.3.56", optional = true } -web-sys = { version = "0.3.56", optional = true } -chrono = { version = "0.4.19", features = ["wasmbind", "clock", "js-sys"] } -anyhow = "1.0.56" -async-trait = "0.1.53" +wasm-bindgen = { version = "0.2", optional = true } +wasm-bindgen-futures = { version = "0.4", optional = true } +js-sys = { version = "0.3", optional = true } +web-sys = { version = "0.3", optional = true } +chrono = { version = "0.4", features = ["wasmbind"] } +anyhow = "1.0" +async-trait = "0.1" +console_error_panic_hook = { version = "0.1", optional = true } +wee_alloc = { version = "0.4", optional = true } +cfg-if = "1.0.0" [dev-dependencies] -wasm-bindgen-test = "0.3.30" +wasm-bindgen-test = "0.3" [lib] path = "lib.rs" @@ -41,8 +44,10 @@ js = [ "wasm-bindgen-futures", "js-sys", "chrono/wasmbind", - "chrono/clock", - "chrono/js-sys", + "wee_alloc", + "console_error_panic_hook", ] web = ["wasm", "web-sys"] +[build] +target = "wasm32-unknown-unknown" diff --git a/crates/wasm/README.md b/crates/wasm/README.md index f7c7bfe4..5a5220c5 100644 --- a/crates/wasm/README.md +++ b/crates/wasm/README.md @@ -11,5 +11,19 @@ cargo install wasm-pack - Build project ```bash -wasm-pack build --target nodejs +wasm-pack build --target web +``` + +### Testing the Project + +- Start the test + +```bash +wasm-pack test --chrome +``` + +- Run tests in the browser + +```bash +open http://127.0.0.1:8000 ``` diff --git a/crates/wasm/fs/blockstore.rs b/crates/wasm/fs/blockstore.rs index 80e1accb..bedca76d 100644 --- a/crates/wasm/fs/blockstore.rs +++ b/crates/wasm/fs/blockstore.rs @@ -1,80 +1,140 @@ -//! The bindgen API of WNFS block store implemenation. +//! The bindgen API for WNFS block store. -use std::{borrow::Cow, str::FromStr}; +use std::str::FromStr; +use std::{borrow::Cow, cell::RefCell, rc::Rc}; use async_trait::async_trait; -use wasm_bindgen::prelude::wasm_bindgen; +use js_sys::Promise; +use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; +use wasm_bindgen_futures::future_to_promise; use wnfs::{ BlockStore as WnfsBlockStore, BlockStoreCidLoad as WnfsBlockStoreCidLoad, BlockStoreLookup as WnfsBlockStoreLookup, Cid, Codec, Decode, IpldCodec, MemoryBlockStore as WnfsMemoryBlockStore, }; -use super::JsResult; +//-------------------------------------------------------------------------------------------------- +// Externs +//-------------------------------------------------------------------------------------------------- +#[wasm_bindgen] +extern "C" { + pub type ExternBlockStore; + + #[wasm_bindgen(js_name = "getBlock")] + fn get_block(this: ExternBlockStore, cid: String) -> Promise; + + #[wasm_bindgen(js_name = "putBlock")] + fn put_block(this: ExternBlockStore, cid: String) -> Promise; + + #[wasm_bindgen(js_name = "putBlock")] + fn load(this: ExternBlockStore, cid: String) -> Promise; +} + +//-------------------------------------------------------------------------------------------------- +// Type Definitions +//-------------------------------------------------------------------------------------------------- + +/// An in-memory block store to simulate IPFS. #[wasm_bindgen] #[derive(Default)] -pub struct MemoryBlockStore(WnfsMemoryBlockStore); +pub struct MemoryBlockStore(Rc>); + +/// A block store provided by the host (JavaScript) for csutom implementation like connection to the IPFS network. +#[wasm_bindgen] +pub struct ForeignBlockStore(ExternBlockStore); + +//-------------------------------------------------------------------------------------------------- +// Implementations +//-------------------------------------------------------------------------------------------------- #[wasm_bindgen] impl MemoryBlockStore { + /// Creates a new in-memory block store. #[wasm_bindgen(constructor)] pub fn new() -> Self { Self::default() } + /// Stores an array of bytes in the block store. #[wasm_bindgen(js_name = "putBlock")] - pub async fn put_block(&mut self, bytes: Vec, codec: u64) -> JsResult { - let codec = IpldCodec::try_from(codec).map_err(|_| js_sys::Error::new("Invalid codec"))?; + pub fn put_block(&self, bytes: Vec, codec: u64) -> Promise { + let store = Rc::clone(&self.0); + + future_to_promise(async move { + let codec = + IpldCodec::try_from(codec).map_err(|_| js_sys::Error::new("Invalid codec"))?; - let cid = self - .0 - .put_block(bytes, codec) - .await - .map_err(|_| js_sys::Error::new("Failed to put block"))?; + let cid = store + .borrow_mut() + .put_block(bytes, codec) + .await + .map_err(|_| js_sys::Error::new("Failed to put block"))?; - Ok(cid.to_string()) + let value = JsValue::from(cid.to_string()); + + Ok(value) + }) } + /// Gets a block of bytes from the store with provided CID. #[wasm_bindgen(js_name = "getBlock")] - pub async fn get_block(&self, cid: &str) -> JsResult { - let cid = Cid::from_str(cid).map_err(|_| js_sys::Error::new("Invalid CID"))?; + pub fn get_block(&self, cid: String) -> Promise { + let store = Rc::clone(&self.0); + + future_to_promise(async move { + let cid = Cid::from_str(&cid).map_err(|_| js_sys::Error::new("Invalid CID"))?; - let bytes = self - .0 - .get_block(&cid) - .await - .map_err(|_| js_sys::Error::new("Failed to get block"))?; + let store_ref = store.borrow(); - Ok(js_sys::Uint8Array::from(&bytes[..])) + let bytes = store_ref + .get_block(&cid) + .await + .map_err(|_| js_sys::Error::new("Failed to get block"))?; + + let value = JsValue::from(js_sys::Uint8Array::from(&bytes[..])); + + Ok(value) + }) } } -#[async_trait] +#[async_trait(?Send)] impl WnfsBlockStore for MemoryBlockStore { async fn put_block( &mut self, bytes: Vec, codec: wnfs::IpldCodec, ) -> Result { - self.0.put_block(bytes, codec).await + let mut store = self.0.borrow_mut(); + store.put_block(bytes, codec).await } } -#[async_trait] +#[async_trait(?Send)] impl WnfsBlockStoreLookup for MemoryBlockStore { - async fn get_block<'a>(&'a self, cid: &wnfs::Cid) -> Result, anyhow::Error> { - self.0.get_block(cid).await + async fn get_block<'a>(&'a self, cid: &wnfs::Cid) -> Result>, anyhow::Error> { + let store = self.0.borrow(); + store.get_block(cid).await.map(|x| Cow::Owned(x.to_vec())) } } -#[async_trait] +#[async_trait(?Send)] impl WnfsBlockStoreCidLoad for MemoryBlockStore { async fn load, C: Codec>( &self, cid: &Cid, decoder: C, ) -> Result { - self.0.load(cid, decoder).await + let store = self.0.borrow(); + store.load(cid, decoder).await } } + +#[cfg(test)] +mod public_file_tests { + use super::*; + use wasm_bindgen_test::*; + + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); +} diff --git a/crates/wasm/fs/public/directory.rs b/crates/wasm/fs/public/directory.rs index 7194d9e3..6dbda77a 100644 --- a/crates/wasm/fs/public/directory.rs +++ b/crates/wasm/fs/public/directory.rs @@ -1,31 +1,35 @@ -//! The bindgen API of PublicDirectory. +//! The bindgen API for PublicDirectory. use chrono::{DateTime, Utc}; use wasm_bindgen::prelude::wasm_bindgen; use wnfs::public::PublicDirectory as WnfsPublicDirectory; +/// A directory in a WNFS public file system. #[wasm_bindgen] pub struct PublicDirectory(WnfsPublicDirectory); +#[cfg(target_arch = "wasm32")] #[wasm_bindgen] impl PublicDirectory { + /// Creates a new directory using the given metadata. #[wasm_bindgen(constructor)] pub fn new(time: &js_sys::Date) -> PublicDirectory { - // let time = DateTime::::from(time); // TODO(appcypher): Fix this. - let time = Utc::now(); - + let time = DateTime::::from(time); PublicDirectory(WnfsPublicDirectory::new(time)) } } +#[cfg(target_arch = "wasm32")] #[cfg(test)] mod public_directory_tests { use super::*; use wasm_bindgen_test::*; + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + #[wasm_bindgen_test] - fn new() { + fn create_directory_successfully() { let time = &js_sys::Date::new_0(); - let public_directory = PublicDirectory::new(time); + let _public_directory = PublicDirectory::new(time); } } diff --git a/crates/wasm/fs/public/file.rs b/crates/wasm/fs/public/file.rs index 1ea549e2..443495d5 100644 --- a/crates/wasm/fs/public/file.rs +++ b/crates/wasm/fs/public/file.rs @@ -1,4 +1,4 @@ -//! The bindgen API of PublicFile. +//! The bindgen API for PublicFile. use std::str::FromStr; @@ -6,20 +6,36 @@ use chrono::{DateTime, Utc}; use wasm_bindgen::prelude::wasm_bindgen; use wnfs::{public::PublicFile as WnfsPublicFile, Cid}; -use crate::fs::{JsResult, MemoryBlockStore}; +use crate::fs::JsResult; +/// A file in a WNFS public file system. #[wasm_bindgen] pub struct PublicFile(WnfsPublicFile); +#[cfg(target_arch = "wasm32")] #[wasm_bindgen] impl PublicFile { + /// Creates a new file in a WNFS public file system. #[wasm_bindgen(constructor)] pub fn new(time: &js_sys::Date, cid: &str) -> JsResult { - // let time = DateTime::::from(time); // TODO(appcypher): Fix this. - let time = Utc::now(); - + let time = DateTime::::from(time); let cid = Cid::from_str(cid).map_err(|_| js_sys::Error::new("Invalid CID"))?; - Ok(PublicFile(WnfsPublicFile::new(time, cid))) } } + +#[cfg(target_arch = "wasm32")] +#[cfg(test)] +mod public_file_tests { + use super::*; + use wasm_bindgen_test::*; + + wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + + #[wasm_bindgen_test] + fn create_file_successfully() { + let time = &js_sys::Date::new_0(); + let cid = Cid::default(); + let _public_directory = PublicFile::new(time, &cid.to_string()); + } +} diff --git a/crates/wasm/lib.rs b/crates/wasm/lib.rs index 4d057181..62c2758b 100644 --- a/crates/wasm/lib.rs +++ b/crates/wasm/lib.rs @@ -1,3 +1,22 @@ #![allow(clippy::unused_unit)] // To prevent clippy screaming about wasm_bindgen macros. pub mod fs; + +/// Panic hook lets us get better error messages if our Rust code ever panics. +/// +/// This function needs to be called at least once during initialisation. +/// https://rustwasm.github.io/docs/wasm-pack/tutorials/npm-browser-packages/template-deep-dive/src-utils-rs.html#2-what-is-console_error_panic_hook +pub fn set_panic_hook() { + #[cfg(feature = "console_error_panic_hook")] + console_error_panic_hook::set_once(); +} + +// If wee_alloc is enabled, we use the the `wee_alloc` crate to allocate memory which helps us reduce bundle size. +// +// https://rustwasm.github.io/docs/wasm-pack/tutorials/npm-browser-packages/template-deep-dive/wee_alloc.html +cfg_if::cfg_if! { + if #[cfg(feature = "wee_alloc")] { + #[global_allocator] + static ALLOC: wee_alloc::WeeAlloc = wee_alloc::WeeAlloc::INIT; + } +}