diff --git a/config/init.go b/config/init.go index cc2351f1f4e..91ed75cc245 100644 --- a/config/init.go +++ b/config/init.go @@ -154,6 +154,19 @@ func badgerSpec() map[string]interface{} { } } +func badger2Spec() map[string]interface{} { + return map[string]interface{}{ + "type": "measure", + "prefix": "badger.datastore", + "child": map[string]interface{}{ + "type": "badger2ds", + "path": "badger2ds", + "syncWrites": false, + "truncate": true, + }, + } +} + func flatfsSpec() map[string]interface{} { return map[string]interface{}{ "type": "mount", diff --git a/config/profile.go b/config/profile.go index cbc7c976453..021a3776747 100644 --- a/config/profile.go +++ b/config/profile.go @@ -123,7 +123,7 @@ This profile may only be applied when first initializing the node. "flatfs": { Description: `Configures the node to use the flatfs datastore. -This is the most battle-tested and reliable datastore. +This is the most battle-tested and reliable datastore. You should use this datastore if: * You need a very simple and very reliable datastore, and you trust your @@ -146,7 +146,7 @@ This profile may only be applied when first initializing the node. "badgerds": { Description: `Configures the node to use the experimental badger datastore. -Use this datastore if some aspects of performance, +Use this datastore if some aspects of performance, especially the speed of adding many gigabytes of files, are critical. However, be aware that: @@ -154,7 +154,7 @@ However, be aware that: smaller than several gigabytes. If you run IPFS with --enable-gc, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using flatfs. -* This datastore uses up to several gigabytes of memory. +* This datastore uses up to several gigabytes of memory. * Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. * The current implementation is based on old badger 1.x @@ -168,6 +168,23 @@ This profile may only be applied when first initializing the node.`, return nil }, }, + "badger2ds": { + Description: `Configures the node to use the experimental badger datastore. + +Use this datastore if some aspects of performance, +especially the speed of adding many gigabytes of files, are critical. +However, be aware that: + +* This datastore uses up to several gigabytes of memory. + +This profile may only be applied when first initializing the node.`, + + InitOnly: true, + Transform: func(c *Config) error { + c.Datastore.Spec = badger2Spec() + return nil + }, + }, "lowpower": { Description: `Reduces daemon overhead on the system. May affect node functionality - performance of content discovery and data diff --git a/docs/config.md b/docs/config.md index 844109a799b..1594b19ac7e 100644 --- a/docs/config.md +++ b/docs/config.md @@ -226,16 +226,28 @@ documented in `ipfs config profile --help`. Configures the node to use the experimental badger datastore. Keep in mind that this **uses an outdated badger 1.x**. - Use this datastore if some aspects of performance, - especially the speed of adding many gigabytes of files, are critical. However, be aware that: + Use this datastore if performance, especially when adding many gigabytes of files, is critical. However: + + - This datastore will not properly reclaim space when your datastore is + smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in + your IPFS node, and disk usage is more critical than performance, consider using + flatfs. + - This datastore uses up to several gigabytes of memory. + This profile may only be applied when first initializing the node. + +- `badger2ds` + + Configures the node to use the badger2 datastore. + + Use this datastore if some aspects of performance, + especially the speed of adding many gigabytes of files, are critical. However, be aware that: + - This datastore will not properly reclaim space when your datastore is smaller than several gigabytes. If you run IPFS with `--enable-gc`, you plan on storing very little data in your IPFS node, and disk usage is more critical than performance, consider using - `flatfs`. + flatfs. - This datastore uses up to several gigabytes of memory. - - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte. - - The current implementation is based on old badger 1.x which is no longer supported by the upstream team. This profile may only be applied when first initializing the node. diff --git a/docs/datastores.md b/docs/datastores.md index db729bf97a0..e500855a5c8 100644 --- a/docs/datastores.md +++ b/docs/datastores.md @@ -41,6 +41,7 @@ Uses [badger](https://github.com/dgraph-io/badger) as a key value store. * `syncWrites`: Flush every write to disk before continuing. Setting this to false is safe as kubo will automatically flush writes to disk before and after performing critical operations like pinning. However, you can set this to true to be extra-safe (at the cost of a 2-3x slowdown when adding files). * `truncate`: Truncate the DB if a partially written sector is found (defaults to true). There is no good reason to set this to false unless you want to manually recover partially written (and unpinned) blocks if kubo crashes half-way through a adding a file. +* `vlogFileSize`: Sets the maximum size of a single value log file to the size specified, or to the default if unspecified. ```json { @@ -48,6 +49,29 @@ Uses [badger](https://github.com/dgraph-io/badger) as a key value store. "path": "", "syncWrites": true|false, "truncate": true|false, + "vlogFileSize": +} +``` + +## badger2ds + +Uses [badger2](https://github.com/dgraph-io/badger) as a key value store. + +* `syncWrites`: Flush every write to disk before continuing. Setting this to false is safe as go-ipfs will automatically flush writes to disk before and after performing critical operations like pinning. However, you can set this to true to be extra-safe (at the cost of a 2-3x slowdown when adding files). +* `truncate`: Truncate the DB if a partially written sector is found (defaults to true). There is no good reason to set this to false unless you want to manually recover partially written (and unpinned) blocks if go-ipfs crashes half-way through a adding a file. +* `compression`: Configure compression. When compression is enabled, every block is compressed using the specified algorithm. This option doesn't affect existing tables. Only the newly created tables are compressed. Compression can be configured to use the snappy algorithm or ZSTD with level 1, 2 or 3. An unspecified value uses the default configuration. +* `blockCacheSize`: Specifies how much data cache should hold in memory. If compression is disabled, adding a cache leads to unnecessary overhead which may affect read performance. A value of 0 means no block cache, and an unspecified value uses the default configuration. +* `vlogFileSize`: Sets the maximum size of a single value log file to the size specified, or to the default if unspecified. + +```json +{ + "type": "badger2ds", + "path": "", + "syncWrites": true|false, + "truncate": true|false, + "compression": "none"|"snappy"|"zstd1"|"zstd2"|"zstd3", + "blockCacheSize": , + "vlogFileSize": } ``` diff --git a/go.mod b/go.mod index 43688f8bfe0..d5c7882e0b9 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/cheggaaa/pb v1.0.29 github.com/coreos/go-systemd/v22 v22.4.0 + github.com/dgraph-io/badger/v2 v2.2007.4 + github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd // indirect github.com/dustin/go-humanize v1.0.0 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 @@ -25,6 +27,7 @@ require ( github.com/ipfs/go-delegated-routing v0.7.0 github.com/ipfs/go-detect-race v0.0.1 github.com/ipfs/go-ds-badger v0.3.0 + github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ds-flatfs v0.5.1 github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ds-measure v0.2.0 @@ -131,7 +134,7 @@ require ( github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/dgraph-io/badger v1.6.2 // indirect - github.com/dgraph-io/ristretto v0.0.2 // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/go.sum b/go.sum index 348263ffefe..30234b1d4cb 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,7 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= @@ -191,8 +192,13 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= +github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd h1:KoJOtZf+6wpQaDTuOWGuo61GxcPBIfhwRxRTaTWGCTc= +github.com/dgraph-io/ristretto v0.0.4-0.20200906165740-41ebdbffecfd/go.mod h1:YylP9MpCYGVZQrly/j/diqcdUetCRRePeBB0c2VGXsA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= @@ -328,6 +334,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -487,6 +495,7 @@ github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13X github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.5/go.mod h1:eXTcaaiN6uOlVCLS9GjJUJtlvJfM3xk23w3fyfrmmJs= github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= +github.com/ipfs/go-datastore v0.5.1/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-delegated-routing v0.7.0 h1:43FyMnKA+8XnyX68Fwg6aoGkqrf8NS5aG7p644s26PU= @@ -501,6 +510,8 @@ github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBR github.com/ipfs/go-ds-badger v0.2.7/go.mod h1:02rnztVKA4aZwDuaRPTf8mpqcKmXP7mLl6JPxd14JHA= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-badger2 v0.1.3 h1:Zo9JicXJ1DmXTN4KOw7oPXkspZ0AWHcAFCP1tQKnegg= +github.com/ipfs/go-ds-badger2 v0.1.3/go.mod h1:TPhhljfrgewjbtuL/tczP8dNrBYwwk+SdPYbms/NO9w= github.com/ipfs/go-ds-flatfs v0.5.1 h1:ZCIO/kQOS/PSh3vcF1H6a8fkRGS7pOfwfPdx4n/KJH4= github.com/ipfs/go-ds-flatfs v0.5.1/go.mod h1:RWTV7oZD/yZYBKdbVIFXTX2fdY2Tbvl94NsWqmoyAX4= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -721,6 +732,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.10 h1:Ai8UzuomSCDw90e1qNMtb15msBXsNpH6gzkkENQNcJo= github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= diff --git a/plugin/loader/preload.go b/plugin/loader/preload.go index 4304862119d..694095d84bf 100644 --- a/plugin/loader/preload.go +++ b/plugin/loader/preload.go @@ -1,6 +1,7 @@ package loader import ( + pluginbadger2ds "github.com/ipfs/kubo/plugin/plugins/badger2ds" pluginbadgerds "github.com/ipfs/kubo/plugin/plugins/badgerds" pluginiplddagjose "github.com/ipfs/kubo/plugin/plugins/dagjose" pluginflatfs "github.com/ipfs/kubo/plugin/plugins/flatfs" @@ -18,6 +19,7 @@ func init() { Preload(pluginipldgit.Plugins...) Preload(pluginiplddagjose.Plugins...) Preload(pluginbadgerds.Plugins...) + Preload(pluginbadger2ds.Plugins...) Preload(pluginflatfs.Plugins...) Preload(pluginlevelds.Plugins...) Preload(pluginpeerlog.Plugins...) diff --git a/plugin/loader/preload_list b/plugin/loader/preload_list index c18ea80ccd5..9e5c2ac0cc6 100644 --- a/plugin/loader/preload_list +++ b/plugin/loader/preload_list @@ -7,6 +7,7 @@ ipldgit github.com/ipfs/kubo/plugin/plugins/git * iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose * badgerds github.com/ipfs/kubo/plugin/plugins/badgerds * +badger2ds github.com/ipfs/kubo/plugin/plugins/badger2ds * flatfs github.com/ipfs/kubo/plugin/plugins/flatfs * levelds github.com/ipfs/kubo/plugin/plugins/levelds * peerlog github.com/ipfs/kubo/plugin/plugins/peerlog * diff --git a/plugin/plugins/badger2ds/badger2ds.go b/plugin/plugins/badger2ds/badger2ds.go new file mode 100644 index 00000000000..b25f126868e --- /dev/null +++ b/plugin/plugins/badger2ds/badger2ds.go @@ -0,0 +1,185 @@ +package badger2ds + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/ipfs/kubo/plugin" + "github.com/ipfs/kubo/repo" + "github.com/ipfs/kubo/repo/fsrepo" + + badgeropts "github.com/dgraph-io/badger/v2/options" + humanize "github.com/dustin/go-humanize" + badger2ds "github.com/ipfs/go-ds-badger2" +) + +// Plugins is exported list of plugins that will be loaded +var Plugins = []plugin.Plugin{ + &badger2dsPlugin{}, +} + +type badger2dsPlugin struct{} + +var _ plugin.PluginDatastore = (*badger2dsPlugin)(nil) + +func (*badger2dsPlugin) Name() string { + return "ds-badger2ds" +} + +func (*badger2dsPlugin) Version() string { + return "0.1.0" +} + +func (*badger2dsPlugin) Init(_ *plugin.Environment) error { + return nil +} + +func (*badger2dsPlugin) DatastoreTypeName() string { + return "badger2ds" +} + +type datastoreConfig struct { + path string + syncWrites bool + truncate bool + + compression badgeropts.CompressionType + zstdCompressionLevel int + + blockCacheSize int64 + vlogFileSize int64 +} + +// DatastoreConfigParser returns a function that creates a new badger2 +// datastore config from a map of badger2 configuration parameters. +func (*badger2dsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { + return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { + var c datastoreConfig + var ok bool + + c.path, ok = params["path"].(string) + if !ok { + return nil, fmt.Errorf("'path' field is missing or not string") + } + + sw, ok := params["syncWrites"] + if !ok { + c.syncWrites = false + } else { + if swb, ok := sw.(bool); ok { + c.syncWrites = swb + } else { + return nil, fmt.Errorf("'syncWrites' field was not a boolean") + } + } + + truncate, ok := params["truncate"] + if !ok { + c.truncate = true + } else { + if truncate, ok := truncate.(bool); ok { + c.truncate = truncate + } else { + return nil, fmt.Errorf("'truncate' field was not a boolean") + } + } + + compression, ok := params["compression"] + if !ok { + // If not specified, use go-ds-badger2 defaults + c.compression = badger2ds.DefaultOptions.Compression + c.zstdCompressionLevel = badger2ds.DefaultOptions.ZSTDCompressionLevel + } else { + if compression, ok := compression.(string); ok { + switch compression { + case "none": + c.compression = badgeropts.None + case "snappy": + c.compression = badgeropts.Snappy + case "zstd1": + c.compression = badgeropts.ZSTD + c.zstdCompressionLevel = 1 + case "zstd2": + c.compression = badgeropts.ZSTD + c.zstdCompressionLevel = 2 + case "zstd3": + c.compression = badgeropts.ZSTD + c.zstdCompressionLevel = 3 + case "": + // If empty string, use go-ds-badger2 defaults + c.compression = badger2ds.DefaultOptions.Compression + c.zstdCompressionLevel = badger2ds.DefaultOptions.ZSTDCompressionLevel + default: + return nil, fmt.Errorf("unrecognized value for compression: %s", compression) + } + } else { + return nil, fmt.Errorf("'compression' field is not string") + } + } + + bcs, ok := params["blockCacheSize"] + if !ok { + // If not specified, use go-ds-badger2 defaults + c.blockCacheSize = badger2ds.DefaultOptions.BlockCacheSize + } else { + if blockCacheSize, ok := bcs.(string); ok { + s, err := humanize.ParseBytes(blockCacheSize) + if err != nil { + return nil, err + } + c.blockCacheSize = int64(s) + } else { + return nil, fmt.Errorf("'blockCacheSize' field was not a string") + } + } + + vls, ok := params["vlogFileSize"] + if !ok { + // If not specified, use go-ds-badger2 defaults + c.vlogFileSize = badger2ds.DefaultOptions.ValueLogFileSize + } else { + if vlogSize, ok := vls.(string); ok { + s, err := humanize.ParseBytes(vlogSize) + if err != nil { + return nil, err + } + c.vlogFileSize = int64(s) + } else { + return nil, fmt.Errorf("'vlogFileSize' field was not a string") + } + } + + return &c, nil + } +} + +func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec { + return map[string]interface{}{ + "type": "badger2ds", + "path": c.path, + } +} + +func (c *datastoreConfig) Create(path string) (repo.Datastore, error) { + p := c.path + if !filepath.IsAbs(p) { + p = filepath.Join(path, p) + } + + err := os.MkdirAll(p, 0755) + if err != nil { + return nil, err + } + + defopts := badger2ds.DefaultOptions + defopts.SyncWrites = c.syncWrites + defopts.Truncate = c.truncate + defopts.Compression = c.compression + defopts.ZSTDCompressionLevel = c.zstdCompressionLevel + defopts.BlockCacheSize = c.blockCacheSize + defopts.ValueLogFileSize = c.vlogFileSize + defopts.NumVersionsToKeep = 1 + + return badger2ds.NewDatastore(p, &defopts) +} diff --git a/plugin/plugins/badgerds/badgerds.go b/plugin/plugins/badgerds/badgerds.go index 3d3f69061ab..1e12e3dff15 100644 --- a/plugin/plugins/badgerds/badgerds.go +++ b/plugin/plugins/badgerds/badgerds.go @@ -46,8 +46,8 @@ type datastoreConfig struct { vlogFileSize int64 } -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore -// from the given parameters +// DatastoreConfigParser returns a function that creates a new badger +// datastore config from a map of badger configuration parameters. func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) { var c datastoreConfig @@ -82,7 +82,7 @@ func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap { vls, ok := params["vlogFileSize"] if !ok { - // default to 1GiB + // If not specified, use go-ds-badger defaults c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize } else { if vlogSize, ok := vls.(string); ok { diff --git a/repo/fsrepo/config_test.go b/repo/fsrepo/config_test.go index 03af75a9631..e84afe46ab5 100644 --- a/repo/fsrepo/config_test.go +++ b/repo/fsrepo/config_test.go @@ -2,6 +2,7 @@ package fsrepo_test import ( "encoding/json" + "io/ioutil" "os" "reflect" "testing" @@ -48,6 +49,18 @@ var defaultConfig = []byte(`{ "BloomFilterSize": 0 }`) +var badgerConfig = []byte(`{ + "path": "datastore", + "type": "badgerds" +}`) + +var badger2Config = []byte(`{ + "compression": "none", + "blockCacheSize": "0", + "path": "datastore", + "type": "badger2ds" +}`) + var leveldbConfig = []byte(`{ "compression": "none", "path": "datastore", @@ -233,3 +246,81 @@ func TestMeasureConfig(t *testing.T) { t.Errorf("expected '*measure.measure' got '%s'", typ) } } + +func TestBadgerConfig(t *testing.T) { + config := new(config.Datastore) + err := json.Unmarshal(defaultConfig, config) + if err != nil { + t.Fatal(err) + } + dir, err := ioutil.TempDir("", "ipfs-datastore-config-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) // clean up + + spec := make(map[string]interface{}) + err = json.Unmarshal(badgerConfig, &spec) + if err != nil { + t.Fatal(err) + } + + dsc, err := fsrepo.AnyDatastoreConfig(spec) + if err != nil { + t.Fatal(err) + } + + expected := `{"path":"datastore","type":"badgerds"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) + } + + ds, err := dsc.Create(dir) + if err != nil { + t.Fatal(err) + } + + if typ := reflect.TypeOf(ds).String(); typ != "*badger.Datastore" { + t.Errorf("expected '*badger.datastore' got '%s'", typ) + } +} + +func TestBadger2Config(t *testing.T) { + config := new(config.Datastore) + err := json.Unmarshal(defaultConfig, config) + if err != nil { + t.Fatal(err) + } + dir, err := ioutil.TempDir("", "ipfs-datastore-config-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) // clean up + + spec := make(map[string]interface{}) + err = json.Unmarshal(badger2Config, &spec) + if err != nil { + t.Fatal(err) + } + + dsc, err := fsrepo.AnyDatastoreConfig(spec) + if err != nil { + t.Fatal(err) + } + + expected := `{"path":"datastore","type":"badger2ds"}` + if dsc.DiskSpec().String() != expected { + t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String()) + } + + ds, err := dsc.Create(dir) + if err != nil { + t.Fatal(err) + } + + // The name of the package in go-ds-badger2 is "badger", so the type + // returned by dsc.Create() is "*badger.Datastore" + if typ := reflect.TypeOf(ds).String(); typ != "*badger.Datastore" { + t.Errorf("expected '*badger.datastore' got '%s'", typ) + } +} diff --git a/test/sharness/t0025-datastores.sh b/test/sharness/t0025-datastores.sh index ec99accb5e7..1d7b421743c 100755 --- a/test/sharness/t0025-datastores.sh +++ b/test/sharness/t0025-datastores.sh @@ -5,7 +5,6 @@ test_description="Test non-standard datastores" . lib/test-lib.sh test_expect_success "'ipfs init --profile=badgerds' succeeds" ' - BITS="2048" && ipfs init --profile=badgerds ' @@ -13,4 +12,16 @@ test_expect_success "'ipfs pin ls' works" ' ipfs pin ls | wc -l | grep 9 ' +test_expect_success "cleanup repo" ' + rm -rf "$IPFS_PATH" +' + +test_expect_success "'ipfs init --profile=badger2ds' succeeds" ' + ipfs init --profile=badger2ds +' + +test_expect_success "'ipfs pin ls' works" ' + ipfs pin ls | wc -l | grep 9 +' + test_done diff --git a/test/sharness/t0060-daemon.sh b/test/sharness/t0060-daemon.sh index d448e035b46..2cdaf10063a 100755 --- a/test/sharness/t0060-daemon.sh +++ b/test/sharness/t0060-daemon.sh @@ -30,6 +30,28 @@ test_expect_success "cleanup repo" ' rm -rf "$IPFS_PATH" ' +test_expect_success "create badger2 config" ' + ipfs init --profile=badger2ds,test > /dev/null && + cp "$IPFS_PATH/config" init-config +' + +test_expect_success "cleanup repo" ' + rm -rf "$IPFS_PATH" +' + +test_launch_ipfs_daemon --init --init-config="$(pwd)/init-config" --init-profile=test +test_kill_ipfs_daemon + +test_expect_success "daemon initialization with existing config works" ' + ipfs config "Datastore.Spec.child.path" >actual && + test $(cat actual) = "badger2ds" && + ipfs config Addresses > orig_addrs +' + +test_expect_success "cleanup repo" ' + rm -rf "$IPFS_PATH" +' + test_launch_ipfs_daemon --init --init-config="$(pwd)/init-config" --init-profile=test,randomports test_kill_ipfs_daemon