From 9c98e0ca9a3601b7958c50b80138ad1285811b86 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 12:36:49 +1200 Subject: [PATCH 1/6] docs: more articles --- doc/articles/repo/block.md | 2 +- doc/articles/repo/config.md | 8 ++++---- doc/articles/repo/gc.md | 5 +++++ doc/articles/repo/pin.md | 21 +++++++++++++++++++++ doc/articles/repository.md | 3 +++ doc/articles/toc.yml | 3 +++ 6 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 doc/articles/repo/gc.md create mode 100644 doc/articles/repo/pin.md diff --git a/doc/articles/repo/block.md b/doc/articles/repo/block.md index 3e98a16d..151f2766 100644 --- a/doc/articles/repo/block.md +++ b/doc/articles/repo/block.md @@ -1,7 +1,7 @@ # Block A block is typically a file (or portion of a file) that is content addressable by IPFS, e.g. -a [content ID](xref:Ipfs.Cid). It is managed with the [BlockApi](xref:Ipfs.CoreApi.IBlockApi). +a [content ID](xref:Ipfs.Cid). It is managed with the [Block Api](xref:Ipfs.CoreApi.IBlockApi). A locally cached or pinned block can be found in the repository's `blocks` folder. To support case insensitive file systems the block's file name is the [base-32](xref:Ipfs.Base32) encoding of the diff --git a/doc/articles/repo/config.md b/doc/articles/repo/config.md index 3017103a..15dff55b 100644 --- a/doc/articles/repo/config.md +++ b/doc/articles/repo/config.md @@ -1,11 +1,11 @@ # Configuration The configuration file is JSON formatted and is a general store for the -various [API](../core-api.md) settings. +various [API](../core-api.md) settings. The file can be changed manually +or the [Config API](xref:Ipfs.CoreApi.IConfigApi) can be used. -## Location - -TODO +The `config` file is located in the repository folder. The path is typically +`$HOME/.csipfs/config`. ## Keys diff --git a/doc/articles/repo/gc.md b/doc/articles/repo/gc.md new file mode 100644 index 00000000..52634979 --- /dev/null +++ b/doc/articles/repo/gc.md @@ -0,0 +1,5 @@ +# Garbage Collection + +[Garbage collection](xref:Ipfs.CoreApi.IBlockRepositoryApi.RemoveGarbageAsync*) is used to reclaim hard disk space from the +repository. It enumerates the local set of +[blocks](block.md) and remove ones that are not [pinned](pin.md). \ No newline at end of file diff --git a/doc/articles/repo/pin.md b/doc/articles/repo/pin.md new file mode 100644 index 00000000..12263ead --- /dev/null +++ b/doc/articles/repo/pin.md @@ -0,0 +1,21 @@ +# Pinning + +The IPFS engine treats the [blocks](block.md) it stores like a cache, +meaning that there is no guarantee that the data will +continue to be stored; see [garbage collection](gc.md). + +The [Pin API](xref:Ipfs.CoreApi.IPinApi) is used to indicate that the data +is important and mustn’t be thrown away. + +## Pinning Services + +To ensure that your important data is retained, you may want +to use a pinning service. Such a service normally trades +money for the service of guaranteeing they’ll keep your data +pinned. + +Some cases where this might be important to you: + +- You don’t have a lot of disk space, but you want to ensure some data sticks around. +- Your computer is a laptop, phone, or tablet that will have intermittent connectivity to the network, but you want to be able to access your data on IPFS from anywhere at any time, even when the device you added it from is offline. +- You want a backup that ensures your data is always available from another computer on the network in case you accidentally delete or garbage-collect on your own computer. \ No newline at end of file diff --git a/doc/articles/repository.md b/doc/articles/repository.md index 71cf64e8..3b14844a 100644 --- a/doc/articles/repository.md +++ b/doc/articles/repository.md @@ -7,6 +7,9 @@ will create it with the factory defaults. To change its name and/or location use the [environment variables](envvars.md) or the [Repository Options](xref:Ipfs.Engine.RepositoryOptions). +The [Block Repository API](xref:Ipfs.CoreApi.IBlockRepositoryApi) manages the [blocks](repo/block.md) in the repository. + + ## Creating The repository will be automatically created if it does not already exist. diff --git a/doc/articles/toc.yml b/doc/articles/toc.yml index ee8e5076..f97c188c 100644 --- a/doc/articles/toc.yml +++ b/doc/articles/toc.yml @@ -41,6 +41,9 @@ - name: Keys href: repo/key.md - name: Pins + href: repo/pin.md + - name: Garbage Collection + href: repo/gc.md - name: Peer href: peer.md items: From 0a3401dab1c6854b57e31bfa6d546f7aee7f9d61 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 14:19:45 +1200 Subject: [PATCH 2/6] feat(BlockRepositoryApi): implement garbage collection --- src/CoreApi/BlockRepositoryApi.cs | 13 +++++++++++-- src/CoreApi/PinApi.cs | 5 +++++ test/CoreApi/BlockRepositoryApiTest.cs | 14 ++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/CoreApi/BlockRepositoryApi.cs b/src/CoreApi/BlockRepositoryApi.cs index 521d572b..e69b8102 100644 --- a/src/CoreApi/BlockRepositoryApi.cs +++ b/src/CoreApi/BlockRepositoryApi.cs @@ -20,9 +20,17 @@ public BlockRepositoryApi(IpfsEngine ipfs) this.ipfs = ipfs; } - public Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken)) + public async Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken)) { - throw new NotImplementedException(); + var blockApi = (BlockApi)ipfs.Block; + var pinApi = (PinApi)ipfs.Pin; + foreach (var cid in blockApi.Store.Names) + { + if (!await pinApi.IsPinnedAsync(cid, cancel).ConfigureAwait(false)) + { + await ipfs.Block.RemoveAsync(cid, ignoreNonexistent: true, cancel: cancel).ConfigureAwait(false); + } + } } public Task StatisticsAsync(CancellationToken cancel = default(CancellationToken)) @@ -66,5 +74,6 @@ void GetDirStats(string path, RepositoryData data, CancellationToken cancel) GetDirStats(dir, data, cancel); } } + } } diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index 14d8caa4..8e63c653 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -114,5 +114,10 @@ FileStore Store return dones; } + + public async Task IsPinnedAsync(Cid id, CancellationToken cancel = default(CancellationToken)) + { + return await Store.ExistsAsync(id, cancel).ConfigureAwait(false); + } } } diff --git a/test/CoreApi/BlockRepositoryApiTest.cs b/test/CoreApi/BlockRepositoryApiTest.cs index b537d186..bf9b93bd 100644 --- a/test/CoreApi/BlockRepositoryApiTest.cs +++ b/test/CoreApi/BlockRepositoryApiTest.cs @@ -28,5 +28,19 @@ public async Task Stats() Assert.AreEqual(stats.Version, version); } + [TestMethod] + public async Task GarbageCollection() + { + var pinned = await ipfs.Block.PutAsync(new byte[256], pin: true); + var unpinned = await ipfs.Block.PutAsync(new byte[512], pin: false); + Assert.AreNotEqual(pinned, unpinned); + Assert.IsNotNull(await ipfs.Block.StatAsync(pinned)); + Assert.IsNotNull(await ipfs.Block.StatAsync(unpinned)); + + await ipfs.BlockRepository.RemoveGarbageAsync(); + Assert.IsNotNull(await ipfs.Block.StatAsync(pinned)); + Assert.IsNull(await ipfs.Block.StatAsync(unpinned)); + } + } } From 928430c14364b0399911366de2dc0f5054c31eb0 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 14:31:46 +1200 Subject: [PATCH 3/6] fix(PinApi): cope with case insensitive file system --- src/CoreApi/PinApi.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index 8e63c653..d1c9c135 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -32,12 +32,11 @@ FileStore Store var folder = Path.Combine(ipfs.Options.Repository.Folder, "pins"); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); - // TODO: Need cid.Encode("base32") store = new FileStore { Folder = folder, - NameToKey = (cid) => cid.Encode(), - KeyToName = (key) => Cid.Decode(key), + NameToKey = (cid) => cid.Hash.ToBase32(), + KeyToName = (key) => new MultiHash(key.FromBase32()), Serialize = (stream, cid, block, cancel) => Task.CompletedTask, Deserialize = (stream, cid, cancel) => Task.FromResult(Pin.Default) }; From 02b4c245c271cf3ee77772ee10ac229b4a24fd12 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 16:45:09 +1200 Subject: [PATCH 4/6] fix(PinApi): store the original CID --- src/CoreApi/PinApi.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index d1c9c135..b80f97f0 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -10,7 +10,7 @@ namespace Ipfs.Engine.CoreApi { class Pin { - public static Pin Default = new Pin(); + public Cid Id; } class PinApi : IPinApi @@ -32,13 +32,12 @@ FileStore Store var folder = Path.Combine(ipfs.Options.Repository.Folder, "pins"); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); + // TODO: Need cid.Encode("base32") store = new FileStore { Folder = folder, NameToKey = (cid) => cid.Hash.ToBase32(), - KeyToName = (key) => new MultiHash(key.FromBase32()), - Serialize = (stream, cid, block, cancel) => Task.CompletedTask, - Deserialize = (stream, cid, cancel) => Task.FromResult(Pin.Default) + KeyToName = (key) => new MultiHash(key.FromBase32()) }; } return store; @@ -61,7 +60,7 @@ FileStore Store var current = todos.Pop(); // Add CID to PIN database. - await Store.PutAsync(current, Pin.Default).ConfigureAwait(false); + await Store.PutAsync(current, new Pin { Id = current }).ConfigureAwait(false); // Make sure that the content is stored locally. await ipfs.Block.GetAsync(current, cancel).ConfigureAwait(false); @@ -84,8 +83,9 @@ FileStore Store public Task> ListAsync(CancellationToken cancel = default(CancellationToken)) { - var cids = Store.Names.ToArray(); - return Task.FromResult((IEnumerable)cids); + var cids = Store.Values + .Select(pin => pin.Id); + return Task.FromResult(cids); } public async Task> RemoveAsync(Cid id, bool recursive = true, CancellationToken cancel = default(CancellationToken)) From 7624498af5b056b6d74fad2b61d3043f2f59a502 Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 17:40:25 +1200 Subject: [PATCH 5/6] fix(PinApi): recursive removal of pins --- src/CoreApi/PinApi.cs | 20 ++++++++++++++------ test/CoreApi/PinApiTest.cs | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index b80f97f0..0ef2c132 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -97,15 +97,23 @@ FileStore Store while (todos.Count > 0) { var current = todos.Pop(); - // TODO: exists is never set to true! - bool exists = false; await Store.RemoveAsync(current, cancel).ConfigureAwait(false); - if (exists && recursive) + if (recursive) { - var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false); - foreach (var link in links) + if (null != await ipfs.Block.StatAsync(current, cancel).ConfigureAwait(false)) { - todos.Push(link.Id); + try + { + var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false); + foreach (var link in links) + { + todos.Push(link.Id); + } + } + catch (Exception) + { + // ignore if current is not an objcet. + } } } dones.Add(current); diff --git a/test/CoreApi/PinApiTest.cs b/test/CoreApi/PinApiTest.cs index 3a085787..a61f8538 100644 --- a/test/CoreApi/PinApiTest.cs +++ b/test/CoreApi/PinApiTest.cs @@ -1,6 +1,5 @@ using Ipfs.CoreApi; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Newtonsoft.Json.Linq; using System; using System.Linq; using System.Text; @@ -85,6 +84,25 @@ public async Task Add_Recursive() var cids = await ipfs.Pin.AddAsync(node.Id, true); Assert.AreEqual(6, cids.Count()); } + + [TestMethod] + public async Task Remove_Recursive() + { + var ipfs = TestFixture.Ipfs; + var options = new AddFileOptions + { + ChunkSize = 3, + Pin = false, + RawLeaves = true, + Wrap = true, + }; + var node = await ipfs.FileSystem.AddTextAsync("hello world", options); + var cids = await ipfs.Pin.AddAsync(node.Id, true); + Assert.AreEqual(6, cids.Count()); + + var removedCids = await ipfs.Pin.RemoveAsync(node.Id, true); + CollectionAssert.AreEqual(cids.ToArray(), removedCids.ToArray()); + } } } From b08c0c092c7dbda375511af7a67a3197601455aa Mon Sep 17 00:00:00 2001 From: Richard Schneider Date: Sun, 28 Jul 2019 18:03:40 +1200 Subject: [PATCH 6/6] fix(PinApi): correct comments --- src/CoreApi/PinApi.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index 0ef2c132..790d10de 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -32,7 +32,6 @@ FileStore Store var folder = Path.Combine(ipfs.Options.Repository.Folder, "pins"); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); - // TODO: Need cid.Encode("base32") store = new FileStore { Folder = folder,