From b4852e3ecf8a87fb9e51e60979175462113459f5 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 21 Nov 2023 11:27:15 -0800 Subject: [PATCH 1/3] http, sia: use v1 CIDs by default --- go.mod | 2 +- http/api.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++------- sia/sia.go | 41 ++++++++++++++++++++++++++++--------- 3 files changed, 84 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 6aa70bc..5b78a24 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.24.4 github.com/multiformats/go-multiaddr v0.11.0 github.com/multiformats/go-multibase v0.2.0 + github.com/multiformats/go-multicodec v0.9.0 github.com/multiformats/go-multihash v0.2.3 go.sia.tech/jape v0.11.0 go.sia.tech/renterd v0.6.1-0.20231117113258-d4afcc97a585 @@ -117,7 +118,6 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nxadm/tail v1.4.11 // indirect diff --git a/http/api.go b/http/api.go index 7fdb81a..0385a8d 100644 --- a/http/api.go +++ b/http/api.go @@ -1,9 +1,12 @@ package http import ( + "errors" "net/http" "github.com/ipfs/go-cid" + "github.com/multiformats/go-multicodec" + "github.com/multiformats/go-multihash" "go.sia.tech/fsd/config" "go.sia.tech/fsd/ipfs" "go.sia.tech/fsd/sia" @@ -29,7 +32,20 @@ func (as *apiServer) handleCalculate(jc jape.Context) { body := jc.Request.Body defer body.Close() - blocks, err := as.sia.CalculateBlocks(ctx, body) + opts := sia.CIDOptions{ + CIDBuilder: cid.V1Builder{Codec: uint64(multicodec.DagPb), MhType: multihash.SHA2_256}, + RawLeaves: true, + } + + if err := jc.DecodeForm("rawLeaves", &opts.RawLeaves); err != nil { + return + } else if err := jc.DecodeForm("maxLinks", &opts.MaxLinks); err != nil { + return + } else if err := jc.DecodeForm("blockSize", &opts.BlockSize); err != nil { + return + } + + blocks, err := as.sia.CalculateBlocks(ctx, body, opts) if err != nil { jc.Error(err, http.StatusInternalServerError) return @@ -43,20 +59,32 @@ func (as *apiServer) handlePin(jc jape.Context) { if err := jc.DecodeParam("cid", &cidStr); err != nil { return } - cid, err := cid.Parse(cidStr) + c, err := cid.Parse(cidStr) if err != nil { jc.Error(err, http.StatusBadRequest) return } - r, err := as.ipfs.DownloadCID(ctx, cid, nil) + // TODO: break this out for better support, the current implementation will + // not handle anything but standard unixfs files with the default block size + r, err := as.ipfs.DownloadCID(ctx, c, nil) if err != nil { jc.Error(err, http.StatusInternalServerError) return } defer r.Close() - if err := as.sia.UploadCID(ctx, cid, r); err != nil { + var opts sia.CIDOptions + switch c.Version() { + case 1: + prefix := c.Prefix() + opts.CIDBuilder = cid.V1Builder{Codec: prefix.Codec, MhType: prefix.MhType, MhLength: prefix.MhLength} + opts.RawLeaves = true + case 0: + opts.CIDBuilder = cid.V0Builder{} + } + + if err := as.sia.UploadCID(ctx, c, r, opts); err != nil { jc.Error(err, http.StatusInternalServerError) return } @@ -68,23 +96,40 @@ func (as *apiServer) handleUpload(jc jape.Context) { if err := jc.DecodeParam("cid", &cidStr); err != nil { return } - cid, err := cid.Parse(cidStr) + c, err := cid.Parse(cidStr) if err != nil { jc.Error(err, http.StatusBadRequest) return + } else if c.Version() != 1 { + jc.Error(errors.New("only v1 CIDs are supported"), http.StatusBadRequest) + return } body := jc.Request.Body defer body.Close() - err = as.sia.UploadCID(ctx, cid, body) + prefix := c.Prefix() + opts := sia.CIDOptions{ + CIDBuilder: cid.V1Builder{Codec: prefix.Codec, MhType: prefix.MhType, MhLength: prefix.MhLength}, + RawLeaves: true, + } + + if err := jc.DecodeForm("rawLeaves", &opts.RawLeaves); err != nil { + return + } else if err := jc.DecodeForm("maxLinks", &opts.MaxLinks); err != nil { + return + } else if err := jc.DecodeForm("blockSize", &opts.BlockSize); err != nil { + return + } + + err = as.sia.UploadCID(ctx, c, body, opts) if err != nil { jc.Error(err, http.StatusInternalServerError) return } // the root cid is the first block - jc.Encode(cid.String()) + jc.Encode(c.String()) } func (as *apiServer) handleVerifyCID(jc jape.Context) { diff --git a/sia/sia.go b/sia/sia.go index 9845d97..7286692 100644 --- a/sia/sia.go +++ b/sia/sia.go @@ -44,13 +44,30 @@ type ( renterd config.Renterd } + + CIDOptions struct { + CIDBuilder cid.Builder + RawLeaves bool + MaxLinks int + BlockSize int64 + } ) // ErrNotFound is returned when a CID is not found in the store var ErrNotFound = errors.New("not found") +func setDefaultCIDOpts(opts *CIDOptions) { + if opts.MaxLinks <= 0 { + opts.MaxLinks = ihelpers.DefaultLinksPerBlock + } + + if opts.BlockSize <= 0 { + opts.BlockSize = chunker.DefaultBlockSize + } +} + // UploadCID uploads a CID to the renterd node -func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error { +func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader, opts CIDOptions) error { log := n.log.Named("upload").With(zap.Stringer("cid", c), zap.String("bucket", n.renterd.Bucket)) dataKey := c.String() @@ -86,11 +103,13 @@ func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error { } }() - spl := chunker.NewSizeSplitter(r, chunker.DefaultBlockSize) - + setDefaultCIDOpts(&opts) + spl := chunker.NewSizeSplitter(r, opts.BlockSize) dbp := ihelpers.DagBuilderParams{ - Maxlinks: ihelpers.DefaultLinksPerBlock, - Dagserv: dagSvc, + Dagserv: dagSvc, + CidBuilder: opts.CIDBuilder, + RawLeaves: opts.RawLeaves, + Maxlinks: opts.MaxLinks, } db, err := dbp.New(spl) if err != nil { @@ -189,14 +208,16 @@ func (n *Node) ProxyHTTPDownload(cid cid.Cid, r *http.Request, w http.ResponseWr } // CalculateBlocks calculates the blocks for a given reader and returns them -func (n *Node) CalculateBlocks(ctx context.Context, r io.Reader) ([]Block, error) { +func (n *Node) CalculateBlocks(ctx context.Context, r io.Reader, opts CIDOptions) ([]Block, error) { dagSvc := NewUnixFileUploader("", io.Discard, io.Discard, n.log.Named("calculate")) - spl := chunker.NewSizeSplitter(r, chunker.DefaultBlockSize) - + setDefaultCIDOpts(&opts) + spl := chunker.NewSizeSplitter(r, opts.BlockSize) dbp := ihelpers.DagBuilderParams{ - Maxlinks: ihelpers.DefaultLinksPerBlock, - Dagserv: dagSvc, + Dagserv: dagSvc, + CidBuilder: opts.CIDBuilder, + RawLeaves: opts.RawLeaves, + Maxlinks: opts.MaxLinks, } db, err := dbp.New(spl) if err != nil { From 011a0aed1efd2401345931c2aeea62a6ba4a3d3e Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 21 Nov 2023 11:57:27 -0800 Subject: [PATCH 2/3] doc: update examples --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8d81e50..6ef86ed 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ ```sh # calculate the CID of the Bored Ape 0 image -curl -X POST --data-binary @ape0.png http://localhost:8001/api/cid/calculate +curl -X POST -H 'Content-Type: application/octet-stream' --data-binary @ape0.png http://localhost:8001/api/cid/calculate # Upload Bored Ape 0 image -curl -X POST --data-binary @ape0.png http://localhost:8001/api/upload/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ +curl -X POST -H 'Content-Type: application/octet-stream' --data-binary @ape0.png http://localhost:8001/api/upload/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ "QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ" curl http://localhost:8080/ipfs/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ ``` @@ -20,3 +20,6 @@ curl http://localhost:8080/ipfs/QmRRPWG96cmgTn2qSzjwr2qvfNEuhunv6FNeMFGa9bx6mQ ## todo - [ ] IPLD / directory support, eg: ipfs://CID/images/cat.png, ipfs://QmeSjSinHpPnmXmspMjwiXyN6zS4E9zccariGR3jxcaWtq/0 +- [ ] IPNS support +- [ ] Better PIN support +- [ ] CAR support From 4755e758bcf7260d1861313747c88f5032071312 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Tue, 21 Nov 2023 11:59:08 -0800 Subject: [PATCH 3/3] sia: fix lint --- sia/sia.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sia/sia.go b/sia/sia.go index 7286692..15c0109 100644 --- a/sia/sia.go +++ b/sia/sia.go @@ -45,6 +45,7 @@ type ( renterd config.Renterd } + // CIDOptions holds configuration options for CID uploads CIDOptions struct { CIDBuilder cid.Builder RawLeaves bool