From 68efd0d22a3a32a460e676f88b1ed05bf16e78d0 Mon Sep 17 00:00:00 2001 From: Nate Maninger Date: Mon, 20 Nov 2023 07:19:42 -0800 Subject: [PATCH] http,sia: use String() for CIDs instead of .Hash().B58String() --- go.mod | 4 +- http/ipfs.go | 4 +- sia/sia.go | 16 ++--- sia/store.go | 8 +-- sia/uploader.go | 2 +- sia/uploader_test.go | 136 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 sia/uploader_test.go diff --git a/go.mod b/go.mod index 79e2daa..6aa70bc 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,8 @@ require ( github.com/libp2p/go-libp2p v0.31.0 github.com/libp2p/go-libp2p-kad-dht v0.24.4 github.com/multiformats/go-multiaddr v0.11.0 + github.com/multiformats/go-multibase v0.2.0 + github.com/multiformats/go-multihash v0.2.3 go.sia.tech/jape v0.11.0 go.sia.tech/renterd v0.6.1-0.20231117113258-d4afcc97a585 go.uber.org/zap v1.25.0 @@ -115,9 +117,7 @@ require ( github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect - github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/nxadm/tail v1.4.11 // indirect diff --git a/http/ipfs.go b/http/ipfs.go index 8fbcf34..0ccb33f 100644 --- a/http/ipfs.go +++ b/http/ipfs.go @@ -45,12 +45,12 @@ func (is *ipfsGatewayServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - is.log.Info("serving", zap.String("cid", cid.Hash().B58String()), zap.String("path", r.URL.Path)) + is.log.Info("serving", zap.Stringer("cid", cid), zap.String("path", r.URL.Path)) // TODO: support paths in Sia proxied downloads err = is.sia.ProxyHTTPDownload(cid, r, w) if errors.Is(err, sia.ErrNotFound) && is.config.IPFS.FetchRemote { - is.log.Info("downloading from ipfs", zap.String("cid", cid.Hash().B58String())) + is.log.Info("downloading from ipfs", zap.Stringer("cid", cid)) r, err := is.ipfs.DownloadCID(ctx, cid, path) if err != nil { http.Error(w, "", http.StatusInternalServerError) diff --git a/sia/sia.go b/sia/sia.go index d5727c7..86bc012 100644 --- a/sia/sia.go +++ b/sia/sia.go @@ -50,16 +50,16 @@ var ErrNotFound = errors.New("not found") // UploadCID uploads a CID to the renterd node func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error { - log := n.log.Named("upload").With(zap.String("cid", c.Hash().B58String()), zap.String("bucket", n.renterd.Bucket)) + log := n.log.Named("upload").With(zap.Stringer("cid", c), zap.String("bucket", n.renterd.Bucket)) - dataKey := c.Hash().B58String() + dataKey := c.String() metaKey := dataKey + ".meta" metaR, metaW := io.Pipe() dataR, dataW := io.Pipe() client := worker.NewClient(n.renterd.Address, n.renterd.Password) - dagSvc := NewUnixFileUploader(c.Hash().B58String(), dataW, metaW, log) + dagSvc := NewUnixFileUploader(c.String(), dataW, metaW, log) errCh := make(chan error, 2) @@ -115,8 +115,8 @@ func (n *Node) UploadCID(ctx context.Context, c cid.Cid, r io.Reader) error { blocks := dagSvc.Blocks() - if rootNode.Cid().Hash().B58String() != c.Hash().B58String() { - return fmt.Errorf("unexpected root cid: %s", rootNode.Cid().Hash().B58String()) + if !rootNode.Cid().Equals(c) { + return fmt.Errorf("unexpected root cid: %s", rootNode.Cid().String()) } else if err := n.store.AddBlocks(ctx, blocks); err != nil { return fmt.Errorf("failed to add blocks to store: %w", err) } @@ -154,7 +154,7 @@ func (n *Node) ProxyHTTPDownload(cid cid.Cid, r *http.Request, w http.ResponseWr return errors.New("cannot proxy partial downloads") } - target, err := url.Parse(n.renterd.Address + "/objects/" + cid.Hash().B58String()) + target, err := url.Parse(n.renterd.Address + "/objects/" + cid.String()) if err != nil { panic(err) } @@ -207,8 +207,8 @@ func (n *Node) VerifyCID(ctx context.Context, c cid.Cid) error { block, err := rbs.Get(ctx, c) if err != nil { return fmt.Errorf("failed to get block: %w", err) - } else if block.Cid().Hash().B58String() != c.Hash().B58String() { - return fmt.Errorf("unexpected root cid: %s", block.Cid().Hash().B58String()) + } else if block.Cid().String() != c.String() { + return fmt.Errorf("unexpected root cid: %s", block.Cid().String()) } return nil } diff --git a/sia/store.go b/sia/store.go index c1eaffd..68b0388 100644 --- a/sia/store.go +++ b/sia/store.go @@ -37,7 +37,7 @@ func (bs *RenterdBlockStore) DeleteBlock(context.Context, cid.Cid) error { // Has returns whether or not a given block is in the blockstore. func (bs *RenterdBlockStore) Has(ctx context.Context, c cid.Cid) (bool, error) { - bs.log.Debug("has block", zap.String("cid", c.Hash().B58String())) + bs.log.Debug("has block", zap.Stringer("cid", c)) return bs.store.HasBlock(ctx, c) } @@ -50,7 +50,7 @@ func (bs *RenterdBlockStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, return nil, fmt.Errorf("failed to get cid: %w", err) } - bs.log.Debug("get block", zap.String("cid", c.Hash().B58String()), zap.Uint64("blockSize", cm.Data.BlockSize), zap.Uint64("blockOffset", cm.Data.Offset), zap.Uint64("metadataSize", cm.Metadata.Length), zap.Uint64("metadataOffset", cm.Metadata.Offset)) + bs.log.Debug("get block", zap.Stringer("cid", c), zap.Uint64("blockSize", cm.Data.BlockSize), zap.Uint64("blockOffset", cm.Data.Offset), zap.Uint64("metadataSize", cm.Metadata.Length), zap.Uint64("metadataOffset", cm.Metadata.Offset)) errCh := make(chan error, 2) defer close(errCh) @@ -123,7 +123,7 @@ func (bs *RenterdBlockStore) Get(ctx context.Context, c cid.Cid) (blocks.Block, }) } if actual := node.Cid(); !actual.Equals(c) { - panic(fmt.Errorf("unexpected cid: requested %q got %q", c.Hash().B58String(), actual.Hash().B58String())) // developer error + panic(fmt.Errorf("unexpected cid: requested %q got %q", c.String(), actual.String())) // developer error } return node, nil } @@ -136,7 +136,7 @@ func (bs *RenterdBlockStore) GetSize(ctx context.Context, c cid.Cid) (int, error } else if err != nil { return 0, fmt.Errorf("failed to get cid: %w", err) } - bs.log.Debug("get block size", zap.String("cid", c.Hash().B58String()), zap.Uint64("size", cm.Data.BlockSize)) + bs.log.Debug("get block size", zap.Stringer("cid", c), zap.Uint64("size", cm.Data.BlockSize)) return int(cm.Data.BlockSize + cm.Metadata.Length), nil } diff --git a/sia/uploader.go b/sia/uploader.go index 134ba66..0500bf9 100644 --- a/sia/uploader.go +++ b/sia/uploader.go @@ -96,7 +96,7 @@ func (ufs *UnixFileUploader) Add(ctx context.Context, node format.Node) error { dataOffset := ufs.dataOffset + dataSize - fileSize ufs.log.Debug("adding node", - zap.String("cid", node.Cid().Hash().B58String()), + zap.Stringer("cid", node.Cid()), zap.Uint64("filesize", fileSize), zap.Uint64("dataOffset", dataOffset), zap.Uint64("metaOffset", ufs.metaOffset), diff --git a/sia/uploader_test.go b/sia/uploader_test.go new file mode 100644 index 0000000..864ef62 --- /dev/null +++ b/sia/uploader_test.go @@ -0,0 +1,136 @@ +package sia + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "testing" + + "github.com/gogo/protobuf/proto" + chunker "github.com/ipfs/boxo/chunker" + "github.com/ipfs/boxo/ipld/merkledag" + "github.com/ipfs/boxo/ipld/unixfs/importer/balanced" + ihelpers "github.com/ipfs/boxo/ipld/unixfs/importer/helpers" + pb "github.com/ipfs/boxo/ipld/unixfs/pb" + "github.com/ipfs/go-cid" + format "github.com/ipfs/go-ipld-format" + "github.com/multiformats/go-multibase" + "github.com/multiformats/go-multihash" + "go.uber.org/zap/zaptest" +) + +func TestUploader(t *testing.T) { + c := cid.MustParse("QmRtaUc7FYQvijR3FHDtyu5M1PXfp6NCJmN4gG8jmEJgpj") + log := zaptest.NewLogger(t) + + dataBuf, metaBuf := bytes.NewBuffer(nil), bytes.NewBuffer(nil) + + uploader := NewUnixFileUploader(c.Hash().B58String(), dataBuf, metaBuf, log) + + f, err := os.Open("/Users/n8maninger/Downloads/QmRtaUc7FYQvijR3FHDtyu5M1PXfp6NCJmN4gG8jmEJgpj.png") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + spl := chunker.NewSizeSplitter(f, chunker.DefaultBlockSize) + + dbp := ihelpers.DagBuilderParams{ + Maxlinks: ihelpers.DefaultLinksPerBlock, + RawLeaves: true, + CidBuilder: cid.V1Builder{ + Codec: cid.DagProtobuf, + MhType: multihash.SHA2_256, + }, + Dagserv: uploader, + } + db, err := dbp.New(spl) + if err != nil { + t.Fatal(err) + } + + rootNode, err := balanced.Layout(db) + if err != nil { + t.Fatal(err) + } + + if !rootNode.Cid().Equals(c) { + t.Fatalf("expected cid %s, got %s", c, rootNode.Cid()) + } else if metaBuf.Len() != 46 { + t.Fatalf("expected metadata size 46, got %d", metaBuf.Len()) + } else if dataBuf.Len() != 904060 { + t.Fatalf("expected data size 904060, got %d", dataBuf.Len()) + } + + blocks := uploader.Blocks() + + if len(blocks) != 5 { + t.Fatalf("expected 5 blocks, got %d", len(blocks)) + } + + for _, b := range blocks { + metaStart, metaEnd := b.Metadata.Offset, b.Metadata.Offset+b.Metadata.Length + dataStart, dataEnd := b.Data.Offset, b.Data.Offset+b.Data.BlockSize + + t.Log(b.CID, metaStart, metaEnd, dataStart, dataEnd) + + mb := metaBuf.Bytes()[metaStart:metaEnd] + db := dataBuf.Bytes()[dataStart:dataEnd] + + t.Log(hex.EncodeToString(mb)) + + var meta pb.Data + if err := proto.Unmarshal(mb, &meta); err != nil { + t.Fatal(err) + } + + if len(db) != 0 { + meta.Data = db + } + + buf, err := proto.Marshal(&meta) + if err != nil { + t.Fatal(err) + } + + node := merkledag.NodeWithData(buf) + for _, link := range b.Links { + node.AddRawLink(link.Name, &format.Link{ + Name: link.Name, + Size: link.Size, + Cid: link.CID, + }) + } + + raw, err := node.EncodeProtobuf(true) + if err != nil { + t.Fatal(err) + } + + if b.CID.Version() != 1 { + t.Fatal("expected cid version 1") + } + + if b.CID.Hash().B58String() == "QmRtaUc7FYQvijR3FHDtyu5M1PXfp6NCJmN4gG8jmEJgpj" { + t.Log("RAW", raw, len(raw)) + } + + if actual := node.Cid(); !actual.Equals(b.CID) { + t.Fatalf("expected cid %s, got %s", b.CID, actual) + } + t.Log("success", node.Cid()) + } + + h := sha256.New() + + h.Write(metaBuf.Bytes()[18:24]) + t.Log(hex.EncodeToString(h.Sum(nil)), len(metaBuf.Bytes()[18:24])) +} + +func TestTest(t *testing.T) { + c := cid.MustParse("QmRtaUc7FYQvijR3FHDtyu5M1PXfp6NCJmN4gG8jmEJgpj") + v1Cid := cid.NewCidV1(c.Type(), c.Hash()) + + t.Fatal(v1Cid.StringOfBase(multibase.Base32)) +}