Merge pull request #69 from richardschneider/cms

feat(FileSystemApi): CMS protected data
richardschneider · Jan 1, 2019 · 450ae57 · 450ae57
2 parents db82f5f + b693fc4
commit 450ae57
Show file tree

Hide file tree

Showing 29 changed files with 965 additions and 52 deletions.
diff --git a/IpfsCli/Commands/AddCommand.cs b/IpfsCli/Commands/AddCommand.cs
@@ -44,6 +44,9 @@ class AddCommand : CommandBase
         [Option("-r|--recursive", Description = "Add directory paths recursively")]
         public bool Recursive { get; set; }
 
+        [Option("--protect", Description = "protect the data with the key")]
+        public string ProtectionKey { get; set; }
+
         Program Parent { get; set; }
 
         protected override async Task<int> OnExecute(CommandLineApplication app)
@@ -57,7 +60,8 @@ protected override async Task<int> OnExecute(CommandLineApplication app)
                 Pin = Pin,
                 RawLeaves = RawLeaves,
                 Trickle = Trickle,
-                Wrap = Wrap
+                Wrap = Wrap,
+                ProtectionKey = ProtectionKey
             };
             IFileSystemNode node;
             if (Directory.Exists(FilePath))

diff --git a/IpfsCli/IpfsCli.csproj b/IpfsCli/IpfsCli.csproj
@@ -15,7 +15,7 @@
 
   <ItemGroup>
     <PackageReference Include="Common.Logging" Version="3.4.1" />
-    <PackageReference Include="Ipfs.Http.Client" Version="0.25.0" />
+    <PackageReference Include="Ipfs.Http.Client" Version="0.26.0" />
     <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="2.2.5" />
     <PackageReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.9.0" />

diff --git a/IpfsServer/HttpApi/V0/FileSystemController.cs b/IpfsServer/HttpApi/V0/FileSystemController.cs
@@ -209,7 +209,8 @@ public async Task<FileSystemNodeDto> Add(
             bool pin = false,
             [ModelBinder(Name = "raw-leaves")] bool rawLeaves = false,
             bool trickle = false,
-            [ModelBinder(Name = "wrap-with-directory")] bool wrap = false
+            [ModelBinder(Name = "wrap-with-directory")] bool wrap = false,
+            string protect = null
             )
         {
             if (file == null)
@@ -223,7 +224,8 @@ public async Task<FileSystemNodeDto> Add(
                 Pin = pin,
                 RawLeaves = rawLeaves,
                 Trickle = trickle,
-                Wrap = wrap
+                Wrap = wrap,
+                ProtectionKey = protect
             };
             if (chunker != null)
             {

diff --git a/IpfsServer/IpfsServer.csproj b/IpfsServer/IpfsServer.csproj
@@ -13,7 +13,7 @@
 
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Ipfs.Core" Version="0.38.1" />
+    <PackageReference Include="Ipfs.Core" Version="0.39.0" />
     <PackageReference Include="Microsoft.AspNetCore.App" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="4.0.1" />
   </ItemGroup>

diff --git a/PeerTalk/src/PeerTalk.csproj b/PeerTalk/src/PeerTalk.csproj
@@ -37,14 +37,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Ipfs.Core" Version="0.38.1" />
+    <PackageReference Include="Ipfs.Core" Version="0.39.0" />
     <PackageReference Include="Makaretu.Dns.Multicast" Version="0.15.1" />
     <PackageReference Include="Makaretu.KBucket" Version="0.4.1" />
     <PackageReference Include="Nito.AsyncEx" Version="5.0.0-pre-05" />
     <PackageReference Include="protobuf-net" Version="2.4.0" />
     <PackageReference Include="semver" Version="2.0.4" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.9.0" />
-    <PackageReference Include="Portable.BouncyCastle" Version="1.8.3.37" />
+    <PackageReference Include="Portable.BouncyCastle" Version="1.8.4" />
   </ItemGroup>
   <ItemGroup Condition=" '$(TargetFramework)' == 'netstandard14'">
     <PackageReference Include="System.Net.NameResolution" Version="4.3.0.0" />

diff --git a/doc/articles/filesystem.md b/doc/articles/filesystem.md
@@ -9,15 +9,9 @@ and [Object API](xref:Ipfs.CoreApi.IObjectApi).
 A file has a unique [content id (CID)](xref:Ipfs.Cid) which is the cryptographic hash of the content; see
 [CID concept](https://docs.ipfs.io/guides/concepts/cid/) for background information.  The file's content is not just the file's 
 data but is encapsulated with a [protocol buffer](https://en.wikipedia.org/wiki/Protocol_Buffers) encoding of the 
-[PBNode](https://github.com/ipfs/go-ipfs/blob/0cb22ccf359e05fb5b55a9bf2f9c515bf7d4dba7/merkledag/pb/merkledag.proto#L31-L39) 
+[Merkle DAG](https://github.com/ipfs/go-ipfs/blob/0cb22ccf359e05fb5b55a9bf2f9c515bf7d4dba7/merkledag/pb/merkledag.proto#L31-L39) 
 and [UnixFS Data](https://github.com/ipfs/go-ipfs/blob/0cb22ccf359e05fb5b55a9bf2f9c515bf7d4dba7/unixfs/pb/unixfs.proto#L3-L20).
 
-Where
-- `PBNode.Data` contains unixfs message Data
-- unixfs `Data.Data` contans file's data
-
-When the file's data exceeds the [chunking size](xref:Ipfs.CoreApi.AddFileOptions.ChunkSize), multiple [blocks](xref:Ipfs.CoreApi.IBlockApi) 
-are generated.  The returned CID points to a block that has `PBNode.Links` and no `PBNode.Data`.
 
 ### Adding a file
 
@@ -47,17 +41,3 @@ using (var stream = await ipfs.FileSystem.ReadFileAsyc(path))
 	// Do something with the data
 }
 ```
-
-### Getting a CID
-
-Normally, you get the CID by [adding](xref:Ipfs.CoreApi.IFileSystemApi.AddAsync*) the file to IPFS.  You can avoid adding it 
-to IPFS by using the [OnlyHash option](xref:Ipfs.CoreApi.AddFileOptions.OnlyHash).
-
-```csharp
-var options = new AddFileOptions { OnlyHash = true };
-var fsn = await ipfs.FileSystem.AddTextAsync("hello world", options);
-Console.WriteLine((string)fsn.Id)
-
-// Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD
-```
-
diff --git a/doc/articles/fs/cid-only.md b/doc/articles/fs/cid-only.md
@@ -0,0 +1,13 @@
+### Getting a CID
+
+Normally, you get the CID by [adding](xref:Ipfs.CoreApi.IFileSystemApi.AddAsync*) the file to IPFS.  You can avoid adding it 
+to IPFS by using the [OnlyHash option](xref:Ipfs.CoreApi.AddFileOptions.OnlyHash).
+
+```csharp
+var options = new AddFileOptions { OnlyHash = true };
+var fsn = await ipfs.FileSystem.AddTextAsync("hello world", options);
+Console.WriteLine((string)fsn.Id)
+
+// Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD
+```
+
diff --git a/doc/articles/fs/encryption.md b/doc/articles/fs/encryption.md
@@ -0,0 +1,29 @@
+# Encryption
+
+The [protection key option](xref:Ipfs.CoreApi.AddFileOptions.ProtectionKey) specifies that 
+the file's data blocks are encrypted using the specified [key name](../key.md).
+
+Each data block maps to a [RFC652 - Cryptographic Message Syntax (CMS)](https://tools.ietf.org/html/rfc5652)
+of type [Enveloped-data](https://tools.ietf.org/html/rfc5652#section-6) which is DER encoded 
+and has the following features:
+
+- The data block is encrypted with a random IV and key using [aes-256-cbc](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard)
+- The recipient is a key transport (ktri) with the Subject Key ID equal to the protection key's public ID
+- The protection key is used to obtain the `aes key` to decrypt the data block.
+
+The [Cid.ContentType](xref:Ipfs.Cid.ContentType) is set to `cms`.
+
+```csharp
+var options = new AddFileOptions
+{
+    ProtectionKey = "me"
+};
+var node = await ipfs.FileSystem.AddTextAsync("hello world", options);
+```
+
+## Reading
+
+The standard [read file methods](../filesystem.md#reading-a-file) are used to decrypt to file. 
+If the private key is not held by the local peer, then a `KeyNotFoundException` is thrown.
+
+
diff --git a/doc/articles/fs/format.md b/doc/articles/fs/format.md
@@ -0,0 +1,71 @@
+### Standard Format
+
+Here is the [Merkle DAG](https://github.com/ipfs/go-ipfs/blob/0cb22ccf359e05fb5b55a9bf2f9c515bf7d4dba7/merkledag/pb/merkledag.proto#L31-L39) 
+and [UnixFS Data](https://github.com/ipfs/go-ipfs/blob/0cb22ccf359e05fb5b55a9bf2f9c515bf7d4dba7/unixfs/pb/unixfs.proto#L3-L20) 
+of a file containing the "hello world" string.
+
+```json
+{
+  "Links": [],
+  "Data": "\u0008\u0002\u0012\u000bhello world\u0018\u000b"
+}
+```
+
+`Data` is the protobuf encoding of the UnixFS Data.
+
+```json
+{
+  "Type": 2,
+  "Data": "aGVsbG8gd29ybGQ=",
+  "FileSize": 11,
+  "BlockSizes": null,
+  "HashType": null,
+  "Fanout": null
+}
+```
+### Chunked Format
+
+When the file's data exceeds the [chunking size](xref:Ipfs.CoreApi.AddFileOptions.ChunkSize), multiple [blocks](xref:Ipfs.CoreApi.IBlockApi) 
+are generated.  The returned CID points to a block that has `Merkle.Links`. Each link 
+contains a chunk of the file.
+
+The following uses a chunking size of 6.  A primary and two secondary blocks are created for "hello world".
+
+#### Primary Block
+
+```json
+{
+  "Links": [
+    {"Name": "", "Hash": "QmPhmNbdBMtSQczNc4hnsMxRf5L4vfkU8jRTXDSHj8trSV", "Size": 14},
+	{"Name": "", "Hash": "QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk", "Size": 13}
+	],
+  "Data":"\u0008\u0002\u0018\u000b \u0006 \u0005"
+}
+{
+  "Type": 2, 
+  "Data": null,
+  "FileSize": 11,
+  "BlockSizes": [6,5],
+  "HashType":null,
+  "Fanout":null
+}
+```
+
+#### First Link
+
+```json
+{
+  "Links": [],
+  "Data": "\u0008\u0002\u0012\u0006hello \u0018\u0006"
+}
+```
+
+#### Second Link
+
+```json
+{
+  "Links": [],
+  "Data": "\u0008\u0002\u0012\u0005world\u0018\u0005"
+}
+```
+
diff --git a/doc/articles/fs/raw.md b/doc/articles/fs/raw.md
@@ -0,0 +1,19 @@
+# Raw Leaves
+
+The [raw leaves option](xref:Ipfs.CoreApi.AddFileOptions.RawLeaves) specifies that 
+the file's data blocks are not [encapsulated](format.md) 
+with a Merkle DAG but simply contain the file's data.
+
+The [Cid.ContentType](xref:Ipfs.Cid.ContentType) is set to `raw`.
+
+```csharp
+var options = new AddFileOptions
+{
+    RawLeaves = true
+};
+var node = await ipfs.FileSystem.AddTextAsync("hello world", options);
+
+// zb2rhj7crUKTQYRGCRATFaQ6YFLTde2YzdqbbhAASkL9uRDXn
+// base58btc cidv1 raw sha2-256 QmaozNR7DZHQK1ZcU9p7QdrshMvXqWK6gpu5rmrkPdT3L4
+
+```
diff --git a/doc/articles/fs/wrap.md b/doc/articles/fs/wrap.md
@@ -0,0 +1,34 @@
+# Wrapping
+
+The [wrap option](xref:Ipfs.CoreApi.AddFileOptions.Wrap) specifies that 
+the a directory is created for the file.
+
+```csharp
+var path = "hello.txt";
+File.WriteAllText("hello.txt", "hello world");
+var options = new AddFileOptions
+{
+    Wrap = true
+};
+var node = await ipfs.FileSystem.AddFileAsync(path, options);
+
+// QmNxvA5bwvPGgMXbmtyhxA1cKFdvQXnsGnZLCGor3AzYxJ
+
+```
+## Format
+
+Two blocks are created, a directory object and a file object.  The file object 
+is described in [standard format](format.md).  The directory object looks 
+like this.
+
+```json
+{
+  "Links": [
+    {"Name": "hello.txt", "Hash": "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD", "Size":19}
+  ],
+  "Data": "\u0008\u0001"
+}
+{
+  "Type": 1,
+}
+```
diff --git a/doc/articles/intro.md b/doc/articles/intro.md
@@ -15,7 +15,7 @@ package is published on [NuGet](https://www.nuget.org/packages/Ipfs.Engine).
 ## Related projects
 
 - [IPFS Core](https://github.com/richardschneider/net-ipfs-core)
-- [IPFS API](https://github.com/richardschneider/net-ipfs-api)
+- [IPFS HTTP Client](https://github.com/richardschneider/net-ipfs-http-client)
 
 ## Other implementations
 

diff --git a/doc/articles/toc.yml b/doc/articles/toc.yml
@@ -11,6 +11,17 @@
   href: core-api.md
 - name: File System
   href: filesystem.md
+  items:
+    - name: CID only
+      href: fs/cid-only.md
+    - name: Encryption
+      href: fs/encryption.md
+    - name: Raw Leaves
+      href: fs/raw.md
+    - name: Wrapping
+      href: fs/wrap.md
+    - name: Format
+      href: fs/format.md
 - name: Repository
   href: repository.md
   items:

diff --git a/src/BlockOptions.cs b/src/BlockOptions.cs
@@ -33,5 +33,13 @@ public class BlockOptions
         ///   Defaults to 64.
         /// </value>
         public int InlineCidLimit { get; set; } = 64;
+
+        /// <summary>
+        ///   The maximun length of data block.
+        /// </summary>
+        /// <value>
+        /// </value>
+        ///   1MB (1024 * 1024)
+        public int MaxBlockSize { get; } = 1024 * 1024;
     }
 }
diff --git a/src/CoreApi/BlockApi.cs b/src/CoreApi/BlockApi.cs
@@ -163,6 +163,11 @@ public async Task<Cid> PutAsync(
             bool pin = false,
             CancellationToken cancel = default(CancellationToken))
         {
+            if (data.Length > ipfs.Options.Block.MaxBlockSize)
+            {
+                throw new ArgumentOutOfRangeException("data.Length", $"Block length can not exceed { ipfs.Options.Block.MaxBlockSize}.");
+            }
+
             // Small enough for an inline CID?
             if (ipfs.Options.Block.AllowInlineCid && data.Length <= ipfs.Options.Block.InlineCidLimit)
             {

diff --git a/src/CoreApi/FileSystemApi.cs b/src/CoreApi/FileSystemApi.cs
@@ -57,10 +57,12 @@ public async Task<IFileSystemNode> AddAsync(
 
             // TODO: various options
             if (options.Trickle) throw new NotImplementedException("Trickle");
+
             var blockService = GetBlockService(options);
+            var keyChain = await ipfs.KeyChain(cancel);
 
             var chunker = new SizeChunker();
-            var nodes = await chunker.ChunkAsync(stream, options, blockService, cancel);
+            var nodes = await chunker.ChunkAsync(stream, options, blockService, keyChain, cancel);
 
             // Multiple nodes for the file?
             FileSystemNode node = null;
@@ -220,7 +222,8 @@ async Task<FileSystemNode> CreateDirectoryAsync (IEnumerable<IFileSystemLink> li
         public async Task<Stream> ReadFileAsync(string path, CancellationToken cancel = default(CancellationToken))
         {
             var cid = await ipfs.ResolveIpfsPathToCidAsync(path, cancel);
-            return await FileSystem.CreateReadStream(cid, ipfs.Block, cancel);
+            var keyChain = await ipfs.KeyChain(cancel);
+            return await FileSystem.CreateReadStream(cid, ipfs.Block, keyChain, cancel);
         }
 
         public async Task<Stream> ReadFileAsync(string path, long offset, long count = 0, CancellationToken cancel = default(CancellationToken))