Merge pull request #130 from richardschneider/repo-management

Repo management

doc/articles/repo/block.md

@@ -1,7 +1,7 @@
 # Block
 
 A block is typically a file (or portion of a file) that is content addressable by IPFS, e.g. 
-a [content ID](xref:Ipfs.Cid).  It is managed with the [BlockApi](xref:Ipfs.CoreApi.IBlockApi).
+a [content ID](xref:Ipfs.Cid).  It is managed with the [Block Api](xref:Ipfs.CoreApi.IBlockApi).
 
 A locally cached or pinned block can be found in the repository's `blocks` folder.  To support case insensitive file 
 systems the block's file name is the [base-32](xref:Ipfs.Base32) encoding of the 

doc/articles/repo/config.md

@@ -1,11 +1,11 @@
 # Configuration
 
 The configuration file is JSON formatted and is a general store for the 
-various [API](../core-api.md) settings.
+various [API](../core-api.md) settings.  The file can be changed manually
+or the [Config API](xref:Ipfs.CoreApi.IConfigApi) can be used.
 
-## Location
-
-TODO
+The `config` file is located in the repository folder.  The path is typically
+`$HOME/.csipfs/config`.
 
 ## Keys
 

doc/articles/repo/gc.md

@@ -0,0 +1,5 @@
+# Garbage Collection
+
+[Garbage collection](xref:Ipfs.CoreApi.IBlockRepositoryApi.RemoveGarbageAsync*) is used to reclaim hard disk space from the
+repository. It enumerates the local set of
+[blocks](block.md) and remove ones that are not [pinned](pin.md).

doc/articles/repo/pin.md

@@ -0,0 +1,21 @@
+# Pinning
+
+The IPFS engine treats the [blocks](block.md) it stores like a cache, 
+meaning that there is no guarantee that the data will 
+continue to be stored; see [garbage collection](gc.md). 
+
+The [Pin API](xref:Ipfs.CoreApi.IPinApi) is used to indicate that the data 
+is important and mustn’t be thrown away.
+
+## Pinning Services
+
+To ensure that your important data is retained, you may want
+to use a pinning service. Such a service normally trades 
+money for the service of guaranteeing they’ll keep your data
+pinned. 
+
+Some cases where this might be important to you:
+
+- You don’t have a lot of disk space, but you want to ensure some data sticks around.
+- Your computer is a laptop, phone, or tablet that will have intermittent connectivity to the network, but you want to be able to access your data on IPFS from anywhere at any time, even when the device you added it from is offline.
+- You want a backup that ensures your data is always available from another computer on the network in case you accidentally delete or garbage-collect on your own computer.

doc/articles/repository.md

@@ -7,6 +7,9 @@ will create it with the factory defaults.
 To change its name and/or location use the [environment variables](envvars.md) 
 or the [Repository Options](xref:Ipfs.Engine.RepositoryOptions).
 
+The [Block Repository API](xref:Ipfs.CoreApi.IBlockRepositoryApi) manages the [blocks](repo/block.md) in the repository.
+
+
 ## Creating
 
 The repository will be automatically created if it does not already exist.  

doc/articles/toc.yml

@@ -41,6 +41,9 @@
     - name: Keys
       href: repo/key.md
     - name: Pins
+      href: repo/pin.md
+    - name: Garbage Collection
+      href: repo/gc.md
 - name: Peer
   href: peer.md
   items:

src/CoreApi/BlockRepositoryApi.cs

@@ -20,9 +20,17 @@ public BlockRepositoryApi(IpfsEngine ipfs)
             this.ipfs = ipfs;
         }
 
-        public Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken))
+        public async Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken))
         {
-            throw new NotImplementedException();
+            var blockApi = (BlockApi)ipfs.Block;
+            var pinApi = (PinApi)ipfs.Pin;
+            foreach (var cid in blockApi.Store.Names)
+            {
+                if (!await pinApi.IsPinnedAsync(cid, cancel).ConfigureAwait(false))
+                {
+                    await ipfs.Block.RemoveAsync(cid, ignoreNonexistent: true, cancel: cancel).ConfigureAwait(false);
+                }
+            }
         }
 
         public Task<RepositoryData> StatisticsAsync(CancellationToken cancel = default(CancellationToken))
@@ -66,5 +74,6 @@ void GetDirStats(string path, RepositoryData data, CancellationToken cancel)
                 GetDirStats(dir, data, cancel);
             }
         }
+
     }
 }

src/CoreApi/PinApi.cs

@@ -10,7 +10,7 @@ namespace Ipfs.Engine.CoreApi
 {
     class Pin
     {
-        public static Pin Default = new Pin();
+        public Cid Id;
     }
 
     class PinApi : IPinApi
@@ -32,14 +32,11 @@ FileStore<Cid, Pin> Store
                     var folder = Path.Combine(ipfs.Options.Repository.Folder, "pins");
                     if (!Directory.Exists(folder))
                         Directory.CreateDirectory(folder);
-                    // TODO: Need cid.Encode("base32")
                     store = new FileStore<Cid, Pin>
                     {
                         Folder = folder,
-                        NameToKey = (cid) => cid.Encode(),
-                        KeyToName = (key) => Cid.Decode(key),
-                        Serialize = (stream, cid, block, cancel) => Task.CompletedTask,
-                        Deserialize = (stream, cid, cancel) => Task.FromResult(Pin.Default)
+                        NameToKey = (cid) => cid.Hash.ToBase32(),
+                        KeyToName = (key) => new MultiHash(key.FromBase32())
                     };
                 }
                 return store;
@@ -62,7 +59,7 @@ FileStore<Cid, Pin> Store
                 var current = todos.Pop();
 
                 // Add CID to PIN database.
-                await Store.PutAsync(current, Pin.Default).ConfigureAwait(false);
+                await Store.PutAsync(current, new Pin { Id = current }).ConfigureAwait(false);
 
                 // Make sure that the content is stored locally.
                 await ipfs.Block.GetAsync(current, cancel).ConfigureAwait(false);
@@ -85,8 +82,9 @@ FileStore<Cid, Pin> Store
 
         public Task<IEnumerable<Cid>> ListAsync(CancellationToken cancel = default(CancellationToken))
         {
-            var cids = Store.Names.ToArray();
-            return Task.FromResult((IEnumerable<Cid>)cids);
+            var cids = Store.Values
+                .Select(pin => pin.Id);
+            return Task.FromResult(cids);
         }
 
         public async Task<IEnumerable<Cid>> RemoveAsync(Cid id, bool recursive = true, CancellationToken cancel = default(CancellationToken))
@@ -98,21 +96,34 @@ FileStore<Cid, Pin> Store
             while (todos.Count > 0)
             {
                 var current = todos.Pop();
-                // TODO: exists is never set to true!
-                bool exists = false;
                 await Store.RemoveAsync(current, cancel).ConfigureAwait(false);
-                if (exists && recursive)
+                if (recursive)
                 {
-                    var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false);
-                    foreach (var link in links)
+                    if (null != await ipfs.Block.StatAsync(current, cancel).ConfigureAwait(false))
                     {
-                        todos.Push(link.Id);
+                        try
+                        {
+                            var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false);
+                            foreach (var link in links)
+                            {
+                                todos.Push(link.Id);
+                            }
+                        }
+                        catch (Exception)
+                        {
+                            // ignore if current is not an objcet.
+                        }
                     }
                 }
                 dones.Add(current);
             }
 
             return dones;
         }
+
+        public async Task<bool> IsPinnedAsync(Cid id, CancellationToken cancel = default(CancellationToken))
+        {
+            return await Store.ExistsAsync(id, cancel).ConfigureAwait(false);
+        }
     }
 }

test/CoreApi/BlockRepositoryApiTest.cs

@@ -28,5 +28,19 @@ public async Task Stats()
             Assert.AreEqual(stats.Version, version);
         }
 
+        [TestMethod]
+        public async Task GarbageCollection()
+        {
+            var pinned = await ipfs.Block.PutAsync(new byte[256], pin: true);
+            var unpinned = await ipfs.Block.PutAsync(new byte[512], pin: false);
+            Assert.AreNotEqual(pinned, unpinned);
+            Assert.IsNotNull(await ipfs.Block.StatAsync(pinned));
+            Assert.IsNotNull(await ipfs.Block.StatAsync(unpinned));
+
+            await ipfs.BlockRepository.RemoveGarbageAsync();
+            Assert.IsNotNull(await ipfs.Block.StatAsync(pinned));
+            Assert.IsNull(await ipfs.Block.StatAsync(unpinned));
+        }
+
     }
 }

test/CoreApi/PinApiTest.cs

@@ -1,6 +1,5 @@
 using Ipfs.CoreApi;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Newtonsoft.Json.Linq;
 using System;
 using System.Linq;
 using System.Text;
@@ -85,6 +84,25 @@ public async Task Add_Recursive()
             var cids = await ipfs.Pin.AddAsync(node.Id, true);
             Assert.AreEqual(6, cids.Count());
         }
+
+        [TestMethod]
+        public async Task Remove_Recursive()
+        {
+            var ipfs = TestFixture.Ipfs;
+            var options = new AddFileOptions
+            {
+                ChunkSize = 3,
+                Pin = false,
+                RawLeaves = true,
+                Wrap = true,
+            };
+            var node = await ipfs.FileSystem.AddTextAsync("hello world", options);
+            var cids = await ipfs.Pin.AddAsync(node.Id, true);
+            Assert.AreEqual(6, cids.Count());
+
+            var removedCids = await ipfs.Pin.RemoveAsync(node.Id, true);
+            CollectionAssert.AreEqual(cids.ToArray(), removedCids.ToArray());
+        }
     }
 }