From 88db4f3a9f98aa1530d18e92f3c94febb6549b14 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 8 Mar 2024 13:27:52 +0100 Subject: [PATCH] wip? --- examples/gateway/proxy/main_test.go | 6 +- gateway/backend_remote.go | 174 ++++++++++++++++++++++++++++ gateway/backend_remote_2.go | 4 +- 3 files changed, 178 insertions(+), 6 deletions(-) diff --git a/examples/gateway/proxy/main_test.go b/examples/gateway/proxy/main_test.go index 22010a8a7..82f63f56a 100644 --- a/examples/gateway/proxy/main_test.go +++ b/examples/gateway/proxy/main_test.go @@ -9,9 +9,7 @@ import ( "strings" "testing" - "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/examples/gateway/common" - "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/gateway" blocks "github.com/ipfs/go-block-format" "github.com/stretchr/testify/assert" @@ -25,10 +23,10 @@ const ( func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server { blockStore, err := gateway.NewProxyBlockstore([]string{rs.URL}, nil) require.NoError(t, err) - blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) + // blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) routing := newProxyRouting(rs.URL, nil) - backend, err := gateway.NewBlocksBackend(blockService, gateway.WithValueStore(routing)) + backend, err := gateway.NewGraphGatewayBackend(blockStore, gateway.WithValueStore(routing)) require.NoError(t, err) handler := common.NewHandler(backend) diff --git a/gateway/backend_remote.go b/gateway/backend_remote.go index 71bf82b91..f837a8be6 100644 --- a/gateway/backend_remote.go +++ b/gateway/backend_remote.go @@ -9,15 +9,23 @@ import ( "sync" "time" + bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/verifcid" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-unixfsnode" + "github.com/ipfs/go-unixfsnode/data" "github.com/ipld/go-car" + dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" "github.com/ipld/go-ipld-prime/linking" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/schema" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/multiformats/go-multihash" ) @@ -135,3 +143,169 @@ func getLinksystem(fn getBlock) *ipld.LinkSystem { unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) return &lsys } + +// walkGatewaySimpleSelector2 walks the subgraph described by the path and terminal element parameters +func walkGatewaySimpleSelector2(ctx context.Context, terminalBlk blocks.Block, dagScope DagScope, entityRange *DagByteRange, lsys *ipld.LinkSystem) error { + lctx := ipld.LinkContext{Ctx: ctx} + var err error + + // If the scope is the block, we only need the root block of the last element of the path, which we have. + if dagScope == DagScopeBlock { + return nil + } + + // decode the terminal block into a node + pc := dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { + if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { + return tlnkNd.LinkTargetNodePrototype(), nil + } + return basicnode.Prototype.Any, nil + }) + + pathTerminalCidLink := cidlink.Link{Cid: terminalBlk.Cid()} + np, err := pc(pathTerminalCidLink, lctx) + if err != nil { + return err + } + + decoder, err := lsys.DecoderChooser(pathTerminalCidLink) + if err != nil { + return err + } + nb := np.NewBuilder() + blockData := terminalBlk.RawData() + if err := decoder(nb, bytes.NewReader(blockData)); err != nil { + return err + } + lastCidNode := nb.Build() + + // TODO: Evaluate: + // Does it matter that we're ignoring the "remainder" portion of the traversal in GetCAR? + // Does it matter that we're using a linksystem with the UnixFS reifier for dagscope=all? + + // If we're asking for everything then give it + if dagScope == DagScopeAll { + sel, err := selector.ParseSelector(selectorparse.CommonSelector_ExploreAllRecursively) + if err != nil { + return err + } + + progress := traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: ctx, + LinkSystem: *lsys, + LinkTargetNodePrototypeChooser: bsfetcher.DefaultPrototypeChooser, + LinkVisitOnlyOnce: false, // Despite being safe for the "all" selector we do this walk anyway since this is how we will be receiving the blocks + }, + } + + if err := progress.WalkMatching(lastCidNode, sel, func(progress traversal.Progress, node datamodel.Node) error { + return nil + }); err != nil { + return err + } + return nil + } + + // From now on, dag-scope=entity! + // Since we need more of the graph load it to figure out what we have + // This includes determining if the terminal node is UnixFS or not + if pbn, ok := lastCidNode.(dagpb.PBNode); !ok { + // If it's not valid dag-pb then we're done + return nil + } else if !pbn.FieldData().Exists() { + // If it's not valid UnixFS then we're done + return nil + } else if unixfsFieldData, decodeErr := data.DecodeUnixFSData(pbn.Data.Must().Bytes()); decodeErr != nil { + // If it's not valid dag-pb and UnixFS then we're done + return nil + } else { + switch unixfsFieldData.FieldDataType().Int() { + case data.Data_Directory, data.Data_Symlink: + // These types are non-recursive so we're done + return nil + case data.Data_Raw, data.Data_Metadata: + // TODO: for now, we decided to return nil here. The different implementations are inconsistent + // and UnixFS is not properly specified: https://github.com/ipfs/specs/issues/316. + // - Is Data_Raw different from Data_File? + // - Data_Metadata is handled differently in boxo/ipld/unixfs and go-unixfsnode. + return nil + case data.Data_HAMTShard: + // Return all elements in the map + _, err := lsys.KnownReifiers["unixfs-preload"](lctx, lastCidNode, lsys) + if err != nil { + return err + } + return nil + case data.Data_File: + nd, err := unixfsnode.Reify(lctx, lastCidNode, lsys) + if err != nil { + return err + } + + fnd, ok := nd.(datamodel.LargeBytesNode) + if !ok { + return fmt.Errorf("could not process file since it did not present as large bytes") + } + f, err := fnd.AsLargeBytes() + if err != nil { + return err + } + + // Get the entity range. If it's empty, assume the defaults (whole file). + effectiveRange := entityRange + if effectiveRange == nil { + effectiveRange = &DagByteRange{ + From: 0, + } + } + + from := effectiveRange.From + + // If we're starting to read based on the end of the file, find out where that is. + var fileLength int64 + foundFileLength := false + if effectiveRange.From < 0 { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + from = fileLength + effectiveRange.From + foundFileLength = true + } + + // If we're reading until the end of the file then do it + if effectiveRange.To == nil { + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.Copy(io.Discard, f) + return err + } + + to := *effectiveRange.To + if (*effectiveRange.To) < 0 && !foundFileLength { + fileLength, err = f.Seek(0, io.SeekEnd) + if err != nil { + return err + } + to = fileLength + *effectiveRange.To + foundFileLength = true + } + + numToRead := 1 + to - from + if numToRead < 0 { + return fmt.Errorf("tried to read less than zero bytes") + } + + if _, err := f.Seek(from, io.SeekStart); err != nil { + return err + } + _, err = io.CopyN(io.Discard, f, numToRead) + return err + default: + // Not a supported type, so we're done + return nil + } + } +} diff --git a/gateway/backend_remote_2.go b/gateway/backend_remote_2.go index 934ed6336..518da2f3e 100644 --- a/gateway/backend_remote_2.go +++ b/gateway/backend_remote_2.go @@ -1108,7 +1108,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, p path.ImmutablePath, param l := getLinksystem(teeBlock) // First resolve the path since we always need to. - _, terminalCid, remainder, _, err := resolvePathWithRootsAndBlock(ctx, p, l) + _, terminalCid, remainder, terminalBlk, err := resolvePathWithRootsAndBlock(ctx, p, l) if err != nil { return err } @@ -1133,7 +1133,7 @@ func (api *GraphGateway) GetCAR(ctx context.Context, p path.ImmutablePath, param blockBuffer = nil } - err = walkGatewaySimpleSelector(ctx, terminalCid, remainder, params, l) + err = walkGatewaySimpleSelector2(ctx, terminalBlk, params.Scope, params.Range, l) if err != nil { return err }