diff --git a/doc/articles/repo/block.md b/doc/articles/repo/block.md index 3e98a16d..151f2766 100644 --- a/doc/articles/repo/block.md +++ b/doc/articles/repo/block.md @@ -1,7 +1,7 @@ # Block A block is typically a file (or portion of a file) that is content addressable by IPFS, e.g. -a [content ID](xref:Ipfs.Cid). It is managed with the [BlockApi](xref:Ipfs.CoreApi.IBlockApi). +a [content ID](xref:Ipfs.Cid). It is managed with the [Block Api](xref:Ipfs.CoreApi.IBlockApi). A locally cached or pinned block can be found in the repository's `blocks` folder. To support case insensitive file systems the block's file name is the [base-32](xref:Ipfs.Base32) encoding of the diff --git a/doc/articles/repo/config.md b/doc/articles/repo/config.md index 3017103a..15dff55b 100644 --- a/doc/articles/repo/config.md +++ b/doc/articles/repo/config.md @@ -1,11 +1,11 @@ # Configuration The configuration file is JSON formatted and is a general store for the -various [API](../core-api.md) settings. +various [API](../core-api.md) settings. The file can be changed manually +or the [Config API](xref:Ipfs.CoreApi.IConfigApi) can be used. -## Location - -TODO +The `config` file is located in the repository folder. The path is typically +`$HOME/.csipfs/config`. ## Keys diff --git a/doc/articles/repo/gc.md b/doc/articles/repo/gc.md new file mode 100644 index 00000000..52634979 --- /dev/null +++ b/doc/articles/repo/gc.md @@ -0,0 +1,5 @@ +# Garbage Collection + +[Garbage collection](xref:Ipfs.CoreApi.IBlockRepositoryApi.RemoveGarbageAsync*) is used to reclaim hard disk space from the +repository. It enumerates the local set of +[blocks](block.md) and remove ones that are not [pinned](pin.md). \ No newline at end of file diff --git a/doc/articles/repo/pin.md b/doc/articles/repo/pin.md new file mode 100644 index 00000000..12263ead --- /dev/null +++ b/doc/articles/repo/pin.md @@ -0,0 +1,21 @@ +# Pinning + +The IPFS engine treats the [blocks](block.md) it stores like a cache, +meaning that there is no guarantee that the data will +continue to be stored; see [garbage collection](gc.md). + +The [Pin API](xref:Ipfs.CoreApi.IPinApi) is used to indicate that the data +is important and mustn’t be thrown away. + +## Pinning Services + +To ensure that your important data is retained, you may want +to use a pinning service. Such a service normally trades +money for the service of guaranteeing they’ll keep your data +pinned. + +Some cases where this might be important to you: + +- You don’t have a lot of disk space, but you want to ensure some data sticks around. +- Your computer is a laptop, phone, or tablet that will have intermittent connectivity to the network, but you want to be able to access your data on IPFS from anywhere at any time, even when the device you added it from is offline. +- You want a backup that ensures your data is always available from another computer on the network in case you accidentally delete or garbage-collect on your own computer. \ No newline at end of file diff --git a/doc/articles/repository.md b/doc/articles/repository.md index 71cf64e8..3b14844a 100644 --- a/doc/articles/repository.md +++ b/doc/articles/repository.md @@ -7,6 +7,9 @@ will create it with the factory defaults. To change its name and/or location use the [environment variables](envvars.md) or the [Repository Options](xref:Ipfs.Engine.RepositoryOptions). +The [Block Repository API](xref:Ipfs.CoreApi.IBlockRepositoryApi) manages the [blocks](repo/block.md) in the repository. + + ## Creating The repository will be automatically created if it does not already exist. diff --git a/doc/articles/toc.yml b/doc/articles/toc.yml index ee8e5076..f97c188c 100644 --- a/doc/articles/toc.yml +++ b/doc/articles/toc.yml @@ -41,6 +41,9 @@ - name: Keys href: repo/key.md - name: Pins + href: repo/pin.md + - name: Garbage Collection + href: repo/gc.md - name: Peer href: peer.md items: diff --git a/src/CoreApi/BlockRepositoryApi.cs b/src/CoreApi/BlockRepositoryApi.cs index 521d572b..e69b8102 100644 --- a/src/CoreApi/BlockRepositoryApi.cs +++ b/src/CoreApi/BlockRepositoryApi.cs @@ -20,9 +20,17 @@ public BlockRepositoryApi(IpfsEngine ipfs) this.ipfs = ipfs; } - public Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken)) + public async Task RemoveGarbageAsync(CancellationToken cancel = default(CancellationToken)) { - throw new NotImplementedException(); + var blockApi = (BlockApi)ipfs.Block; + var pinApi = (PinApi)ipfs.Pin; + foreach (var cid in blockApi.Store.Names) + { + if (!await pinApi.IsPinnedAsync(cid, cancel).ConfigureAwait(false)) + { + await ipfs.Block.RemoveAsync(cid, ignoreNonexistent: true, cancel: cancel).ConfigureAwait(false); + } + } } public Task StatisticsAsync(CancellationToken cancel = default(CancellationToken)) @@ -66,5 +74,6 @@ void GetDirStats(string path, RepositoryData data, CancellationToken cancel) GetDirStats(dir, data, cancel); } } + } } diff --git a/src/CoreApi/PinApi.cs b/src/CoreApi/PinApi.cs index 14d8caa4..790d10de 100644 --- a/src/CoreApi/PinApi.cs +++ b/src/CoreApi/PinApi.cs @@ -10,7 +10,7 @@ namespace Ipfs.Engine.CoreApi { class Pin { - public static Pin Default = new Pin(); + public Cid Id; } class PinApi : IPinApi @@ -32,14 +32,11 @@ FileStore Store var folder = Path.Combine(ipfs.Options.Repository.Folder, "pins"); if (!Directory.Exists(folder)) Directory.CreateDirectory(folder); - // TODO: Need cid.Encode("base32") store = new FileStore { Folder = folder, - NameToKey = (cid) => cid.Encode(), - KeyToName = (key) => Cid.Decode(key), - Serialize = (stream, cid, block, cancel) => Task.CompletedTask, - Deserialize = (stream, cid, cancel) => Task.FromResult(Pin.Default) + NameToKey = (cid) => cid.Hash.ToBase32(), + KeyToName = (key) => new MultiHash(key.FromBase32()) }; } return store; @@ -62,7 +59,7 @@ FileStore Store var current = todos.Pop(); // Add CID to PIN database. - await Store.PutAsync(current, Pin.Default).ConfigureAwait(false); + await Store.PutAsync(current, new Pin { Id = current }).ConfigureAwait(false); // Make sure that the content is stored locally. await ipfs.Block.GetAsync(current, cancel).ConfigureAwait(false); @@ -85,8 +82,9 @@ FileStore Store public Task> ListAsync(CancellationToken cancel = default(CancellationToken)) { - var cids = Store.Names.ToArray(); - return Task.FromResult((IEnumerable)cids); + var cids = Store.Values + .Select(pin => pin.Id); + return Task.FromResult(cids); } public async Task> RemoveAsync(Cid id, bool recursive = true, CancellationToken cancel = default(CancellationToken)) @@ -98,15 +96,23 @@ FileStore Store while (todos.Count > 0) { var current = todos.Pop(); - // TODO: exists is never set to true! - bool exists = false; await Store.RemoveAsync(current, cancel).ConfigureAwait(false); - if (exists && recursive) + if (recursive) { - var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false); - foreach (var link in links) + if (null != await ipfs.Block.StatAsync(current, cancel).ConfigureAwait(false)) { - todos.Push(link.Id); + try + { + var links = await ipfs.Object.LinksAsync(current, cancel).ConfigureAwait(false); + foreach (var link in links) + { + todos.Push(link.Id); + } + } + catch (Exception) + { + // ignore if current is not an objcet. + } } } dones.Add(current); @@ -114,5 +120,10 @@ FileStore Store return dones; } + + public async Task IsPinnedAsync(Cid id, CancellationToken cancel = default(CancellationToken)) + { + return await Store.ExistsAsync(id, cancel).ConfigureAwait(false); + } } } diff --git a/test/CoreApi/BlockRepositoryApiTest.cs b/test/CoreApi/BlockRepositoryApiTest.cs index b537d186..bf9b93bd 100644 --- a/test/CoreApi/BlockRepositoryApiTest.cs +++ b/test/CoreApi/BlockRepositoryApiTest.cs @@ -28,5 +28,19 @@ public async Task Stats() Assert.AreEqual(stats.Version, version); } + [TestMethod] + public async Task GarbageCollection() + { + var pinned = await ipfs.Block.PutAsync(new byte[256], pin: true); + var unpinned = await ipfs.Block.PutAsync(new byte[512], pin: false); + Assert.AreNotEqual(pinned, unpinned); + Assert.IsNotNull(await ipfs.Block.StatAsync(pinned)); + Assert.IsNotNull(await ipfs.Block.StatAsync(unpinned)); + + await ipfs.BlockRepository.RemoveGarbageAsync(); + Assert.IsNotNull(await ipfs.Block.StatAsync(pinned)); + Assert.IsNull(await ipfs.Block.StatAsync(unpinned)); + } + } } diff --git a/test/CoreApi/PinApiTest.cs b/test/CoreApi/PinApiTest.cs index 3a085787..a61f8538 100644 --- a/test/CoreApi/PinApiTest.cs +++ b/test/CoreApi/PinApiTest.cs @@ -1,6 +1,5 @@ using Ipfs.CoreApi; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Newtonsoft.Json.Linq; using System; using System.Linq; using System.Text; @@ -85,6 +84,25 @@ public async Task Add_Recursive() var cids = await ipfs.Pin.AddAsync(node.Id, true); Assert.AreEqual(6, cids.Count()); } + + [TestMethod] + public async Task Remove_Recursive() + { + var ipfs = TestFixture.Ipfs; + var options = new AddFileOptions + { + ChunkSize = 3, + Pin = false, + RawLeaves = true, + Wrap = true, + }; + var node = await ipfs.FileSystem.AddTextAsync("hello world", options); + var cids = await ipfs.Pin.AddAsync(node.Id, true); + Assert.AreEqual(6, cids.Count()); + + var removedCids = await ipfs.Pin.RemoveAsync(node.Id, true); + CollectionAssert.AreEqual(cids.ToArray(), removedCids.ToArray()); + } } }