Skip to content

Commit

Permalink
ipfs: fix HAMT sharded directories
Browse files Browse the repository at this point in the history
n8maninger committed Aug 12, 2024

Verified

This commit was signed with the committer’s verified signature.
n8maninger Nate Maninger
1 parent 79dca13 commit bf0e382
Showing 5 changed files with 325 additions and 225 deletions.
15 changes: 15 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file

version: 2
updates:
- package-ecosystem: "gomod" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
groups:
all-dependencies:
patterns:
- "*"
18 changes: 11 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
@@ -22,25 +22,29 @@ api: # The API is used to pin and unpin CIDs and access metrics about the node
address: :8081
password: sia is cool
renterd: # The backing store for fsd. This configures the renterd node that stores the block data.
address: http://localhost:9980/api/worker
password: sia is extra cool
workerAddress: http://localhost:9980/api/worker
workerPassword: sia is extra cool
busAddress: http://localhost:9980/api/bus
busPassword: sia is extra cool
bucket: ipfs
blockstore:
maxConcurrent: 100 # the maximum number of concurrent fetches the blockstore will allow
cacheSize: 1000000 # the maximum number of blocks that will be cached in memory at any given time
ipfs: # The IPFS node configuration.
privateKey: # The private key for the IPFS node. If not set, a new key will be generated on startup and must be manually saved to the configuration file.
gateway: # configure the HTTP gateway
listenAddress: :8080
redirectPathStyle: true
listenAddress: :8080
redirectPathStyle: true
fetch:
enabled: false # enable or disable fetching blocks from the IPFS network If false, will only serve pinned blocks.
enabled: false # enable or disable fetching blocks from the IPFS network. If false, will only serve pinned blocks.
allowlist: [] # contains the CIDs that are allowed to be fetched remotely by the gateway. If empty, all CIDs are allowed.
listenAddresses:
- /ip4/0.0.0.0/tcp/4001 # the listen address for bitswap. Since no announce addresses are configured, no bits will be swapped
announceAddresses:
- /ipv4/127.0.0.1/tcp/4001 # the announce address for bitswap. This is the address that will be shared with other nodes on the network.
provider:
batchSize: 50000 # configures the reprovide batch size
interval: 18h # the interval at which the node will re-provide blocks to the IPFS network
batchSize: 50000 # configures the reprovide batch size
interval: 18h # the interval at which the node will re-provide blocks to the IPFS network
```
## Build
126 changes: 75 additions & 51 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,59 +1,62 @@
module go.sia.tech/fsd

go 1.21.7
go 1.22

toolchain go1.22.5

require (
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/ipfs/boxo v0.19.0
github.com/ipfs/boxo v0.22.0
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
github.com/ipfs/go-ds-leveldb v0.5.0
github.com/ipfs/go-ipld-format v0.6.0
github.com/ipld/go-car/v2 v2.13.1
github.com/libp2p/go-libp2p v0.33.2
github.com/libp2p/go-libp2p v0.36.1
github.com/libp2p/go-libp2p-kad-dht v0.25.2
github.com/mattn/go-sqlite3 v1.14.22
github.com/multiformats/go-multiaddr v0.12.3
github.com/multiformats/go-multiaddr v0.13.0
github.com/multiformats/go-multicodec v0.9.0
github.com/multiformats/go-multihash v0.2.3
go.sia.tech/jape v0.11.2-0.20240124024603-93559895d640
go.sia.tech/renterd v1.0.6
go.sia.tech/jape v0.12.0
go.sia.tech/renterd v1.0.8
go.uber.org/zap v1.27.0
gopkg.in/yaml.v3 v3.0.1
lukechampine.com/frand v1.4.2
)

require (
github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect
github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
github.com/aws/aws-sdk-go v1.50.1 // indirect
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
github.com/cskr/pubsub v1.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
github.com/dchest/threefish v0.0.0-20120919164726-3ecf4c494abf // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/elastic/gosigar v0.14.3 // indirect
github.com/flynn/noise v1.1.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.5 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/websocket v1.5.1 // indirect
github.com/gotd/contrib v0.19.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/gotd/contrib v0.20.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
@@ -78,17 +81,17 @@ require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/julienschmidt/httprouter v1.3.0 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/klauspost/reedsolomon v1.12.1 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/klauspost/reedsolomon v1.12.3 // indirect
github.com/koron/go-ssdp v0.0.4 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.3 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
github.com/libp2p/go-msgio v0.3.0 // indirect
github.com/libp2p/go-nat v0.2.0 // indirect
@@ -97,7 +100,7 @@ require (
github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/miekg/dns v1.1.58 // indirect
github.com/miekg/dns v1.1.61 // indirect
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
github.com/minio/sha256-simd v1.0.1 // indirect
@@ -110,29 +113,50 @@ require (
github.com/multiformats/go-multibase v0.2.0 // indirect
github.com/multiformats/go-multistream v0.5.0 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/onsi/ginkgo/v2 v2.15.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/onsi/ginkgo/v2 v2.20.0 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
github.com/pion/datachannel v1.5.8 // indirect
github.com/pion/dtls/v2 v2.2.12 // indirect
github.com/pion/ice/v2 v2.3.32 // indirect
github.com/pion/interceptor v0.1.29 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns v0.0.12 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.14 // indirect
github.com/pion/rtp v1.8.8 // indirect
github.com/pion/sctp v1.8.20 // indirect
github.com/pion/sdp/v3 v3.0.9 // indirect
github.com/pion/srtp/v2 v2.0.20 // indirect
github.com/pion/stun v0.6.1 // indirect
github.com/pion/transport/v2 v2.2.9 // indirect
github.com/pion/turn/v2 v2.1.6 // indirect
github.com/pion/webrtc/v3 v3.2.50 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/polydawn/refmt v0.89.0 // indirect
github.com/prometheus/client_golang v1.18.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.47.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/prometheus/client_golang v1.19.1 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/quic-go/qpack v0.4.0 // indirect
github.com/quic-go/quic-go v0.42.0 // indirect
github.com/quic-go/webtransport-go v0.6.0 // indirect
github.com/quic-go/quic-go v0.46.0 // indirect
github.com/quic-go/webtransport-go v0.8.0 // indirect
github.com/raulk/go-watchdog v1.3.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stretchr/testify v1.9.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20240109153615-66e95c3e8a87 // indirect
github.com/whyrusleeping/cbor-gen v0.1.2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/wlynxg/anet v0.0.3 // indirect
gitlab.com/NebulousLabs/bolt v1.4.4 // indirect
gitlab.com/NebulousLabs/encoding v0.0.0-20200604091946-456c3dc907fe // indirect
gitlab.com/NebulousLabs/entropy-mnemonics v0.0.0-20181018051301-7532f67e3500 // indirect
@@ -146,29 +170,29 @@ require (
gitlab.com/NebulousLabs/siamux v0.0.2-0.20220630142132-142a1443a259 // indirect
gitlab.com/NebulousLabs/threadgroup v0.0.0-20200608151952-38921fbef213 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
go.sia.tech/core v0.2.1 // indirect
go.sia.tech/coreutils v0.0.3 // indirect
go.sia.tech/gofakes3 v0.0.0-20231109151325-e0d47c10dce2 // indirect
go.opentelemetry.io/otel v1.28.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/trace v1.28.0 // indirect
go.sia.tech/core v0.4.3 // indirect
go.sia.tech/coreutils v0.2.5 // indirect
go.sia.tech/gofakes3 v0.0.4 // indirect
go.sia.tech/mux v1.2.0 // indirect
go.sia.tech/siad v1.5.10-0.20230228235644-3059c0b930ca // indirect
go.uber.org/dig v1.17.1 // indirect
go.uber.org/fx v1.20.1 // indirect
go.uber.org/dig v1.18.0 // indirect
go.uber.org/fx v1.22.2 // indirect
go.uber.org/mock v0.4.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
golang.org/x/mod v0.15.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sync v0.6.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.18.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
gonum.org/v1/gonum v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
gorm.io/gorm v1.25.7 // indirect
lukechampine.com/blake3 v1.2.1 // indirect
golang.org/x/crypto v0.26.0 // indirect
golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
golang.org/x/mod v0.20.0 // indirect
golang.org/x/net v0.28.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.24.0 // indirect
golang.org/x/text v0.17.0 // indirect
golang.org/x/tools v0.24.0 // indirect
golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
gonum.org/v1/gonum v0.15.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gorm.io/gorm v1.25.11 // indirect
lukechampine.com/blake3 v1.3.0 // indirect
)
345 changes: 203 additions & 142 deletions go.sum

Large diffs are not rendered by default.

46 changes: 21 additions & 25 deletions ipfs/unixfs.go
Original file line number Diff line number Diff line change
@@ -8,12 +8,12 @@

chunker "github.com/ipfs/boxo/chunker"
"github.com/ipfs/boxo/ipld/merkledag"
"github.com/ipfs/boxo/ipld/unixfs"
"github.com/ipfs/boxo/ipld/unixfs/importer/balanced"
ihelpers "github.com/ipfs/boxo/ipld/unixfs/importer/helpers"
fsio "github.com/ipfs/boxo/ipld/unixfs/io"
"github.com/ipfs/go-cid"
format "github.com/ipfs/go-ipld-format"
"go.uber.org/zap"
)

type (
@@ -55,26 +55,21 @@
}
}

func traverseNode(ctx context.Context, ng format.NodeGetter, parent format.Node, path []string) (format.Node, error) {
func traverseUnixFSNode(ctx context.Context, ng format.DAGService, parent format.Node, path []string) (format.Node, error) {
if len(path) == 0 {
return parent, nil
}

childLink, rem, err := parent.Resolve(path)
dir, err := fsio.NewDirectoryFromNode(ng, parent)
if err != nil {
return nil, fmt.Errorf("failed to resolve path %q: %w", strings.Join(path, "/"), err)
return nil, fmt.Errorf("failed to create directory from node: %w", err)
}

switch v := childLink.(type) {
case *format.Link:
childNode, err := ng.Get(ctx, v.Cid)
if err != nil {
return nil, fmt.Errorf("failed to get child node %q: %w", v.Cid, err)
}
return traverseNode(ctx, ng, childNode, rem)
default:
return nil, fmt.Errorf("expected link node, got %T", childLink)
child, err := dir.Find(ctx, path[0])
if err != nil {
return nil, fmt.Errorf("failed to find child %q: %w", path[0], err)
}
return traverseUnixFSNode(ctx, ng, child, path[1:])
}

// DownloadUnixFile downloads a UnixFS CID from IPFS
@@ -86,7 +81,7 @@
return nil, fmt.Errorf("failed to get root node: %w", err)
}

node, err := traverseNode(ctx, n.dagService, rootNode, path)
node, err := traverseUnixFSNode(ctx, n.dagService, rootNode, path)
if err != nil {
return nil, fmt.Errorf("failed to get root node: %w", err)
}
@@ -122,7 +117,7 @@
return dr, "index.html", err
}

node, err := traverseNode(ctx, n.dagService, rootNode, path)
child, err := traverseUnixFSNode(ctx, n.dagService, rootNode, path)
if err != nil && strings.Contains(err.Error(), "failed to resolve path") {
if indexFileCid != nil {
return serveIndex()
@@ -133,20 +128,21 @@

// if the resolved node is a directory and the index file exists
// serve it.
fsnode, err := unixfs.ExtractFSNode(node)
dir, err := fsio.NewDirectoryFromNode(n.dagService, child)
if err == nil {
if fsnode.IsDir() && indexFileCid != nil {
return serveIndex()
n.log.Debug("serving index.html")
child, err := dir.Find(ctx, "index.html")
if err != nil {
return nil, "", fmt.Errorf("failed to find index.html: %w", err)
}
dr, err := fsio.NewDagReader(ctx, child, dagSess)
return dr, "index.html", err
} else {

Check warning on line 140 in ipfs/unixfs.go

GitHub Actions / test

indent-error-flow: if block ends with a return statement, so drop this else and outdent its block (revive)

Check warning on line 140 in ipfs/unixfs.go

GitHub Actions / test (ubuntu-latest, 1.22)

indent-error-flow: if block ends with a return statement, so drop this else and outdent its block (revive)
n.log.Debug("failed to create directory from node", zap.Error(err))
}

filename := c.String()
if len(path) > 0 {
filename = path[len(path)-1]
}

dr, err := fsio.NewDagReader(ctx, node, dagSess)
return dr, filename, err
dr, err := fsio.NewDagReader(ctx, child, dagSess)
return dr, path[len(path)-1], err
}

// UploadUnixFile uploads a UnixFS file to IPFS

0 comments on commit bf0e382

Please sign in to comment.