From 4564ef2af05407fe65334683b2d488d85729bc38 Mon Sep 17 00:00:00 2001 From: Ferran Borreguero Date: Mon, 2 Oct 2023 13:07:37 +0200 Subject: [PATCH] Engine store api (#53) * Create engine store api * Fix test --- core/vm/contracts_suave.go | 8 +++- core/vm/contracts_suave_eth.go | 3 +- core/vm/contracts_suave_test.go | 10 ++--- core/vm/suave.go | 4 +- suave/backends/backend_testing.go | 36 ++++++++++++++++ suave/backends/redis_backends_test.go | 31 +------------- suave/backends/redis_store_backend.go | 50 +++++++++++----------- suave/backends/redis_store_backend_test.go | 14 ++++++ suave/core/engine.go | 19 +++----- suave/core/engine_test.go | 6 +-- suave/core/types.go | 19 +++++--- suave/e2e/workflow_test.go | 12 +++--- 12 files changed, 120 insertions(+), 92 deletions(-) create mode 100644 suave/backends/backend_testing.go create mode 100644 suave/backends/redis_store_backend_test.go diff --git a/core/vm/contracts_suave.go b/core/vm/contracts_suave.go index e4e2805cf7..07580e4c60 100644 --- a/core/vm/contracts_suave.go +++ b/core/vm/contracts_suave.go @@ -300,7 +300,13 @@ func (c *fetchBids) RunConfidential(suaveContext *SuaveContext, input []byte) ([ } func (c *fetchBids) runImpl(suaveContext *SuaveContext, targetBlock uint64, namespace string) ([]types.Bid, error) { - bids := suaveContext.Backend.ConfidentialStoreEngine.FetchBidsByProtocolAndBlock(targetBlock, namespace) + bids1 := suaveContext.Backend.ConfidentialStoreEngine.FetchBidsByProtocolAndBlock(targetBlock, namespace) + + bids := make([]types.Bid, 0, len(bids1)) + for _, bid := range bids1 { + bids = append(bids, bid.ToInnerBid()) + } + return bids, nil } diff --git a/core/vm/contracts_suave_eth.go b/core/vm/contracts_suave_eth.go index 937323be4f..a44ee04362 100644 --- a/core/vm/contracts_suave_eth.go +++ b/core/vm/contracts_suave_eth.go @@ -217,10 +217,11 @@ func (c *buildEthBlock) runImpl(suaveContext *SuaveContext, blockArgs types.Buil for i, bidId := range bidIds { var err error - bidsToMerge[i], err = suaveContext.Backend.ConfidentialStoreEngine.FetchBidById(bidId) + bid, err := suaveContext.Backend.ConfidentialStoreEngine.FetchBidById(bidId) if err != nil { return nil, nil, fmt.Errorf("could not fetch bid id %v: %w", bidId, err) } + bidsToMerge[i] = bid.ToInnerBid() } var mergedBundles []types.SBundle diff --git a/core/vm/contracts_suave_test.go b/core/vm/contracts_suave_test.go index e320980123..ec1e27d6d6 100644 --- a/core/vm/contracts_suave_test.go +++ b/core/vm/contracts_suave_test.go @@ -45,11 +45,11 @@ func (m *mockSuaveBackend) FetchEngineBidById(suave.BidId) (suave.Bid, error) { return suave.Bid{}, nil } -func (m *mockSuaveBackend) FetchBidById(suave.BidId) (types.Bid, error) { - return types.Bid{}, nil +func (m *mockSuaveBackend) FetchBidById(suave.BidId) (suave.Bid, error) { + return suave.Bid{}, nil } -func (m *mockSuaveBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid { +func (m *mockSuaveBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []suave.Bid { return nil } @@ -179,10 +179,10 @@ func TestSuave_BidWorkflow(t *testing.T) { namespace string bids []types.Bid }{ - {0, "a", nil}, + {0, "a", []types.Bid{}}, {5, "a", []types.Bid{bid5}}, {10, "a", []types.Bid{bid10, bid10b}}, - {11, "a", nil}, + {11, "a", []types.Bid{}}, } for _, c := range cases { diff --git a/core/vm/suave.go b/core/vm/suave.go index 5285df8f69..e58810122e 100644 --- a/core/vm/suave.go +++ b/core/vm/suave.go @@ -17,8 +17,8 @@ type ConfidentialStore interface { InitializeBid(bid types.Bid, creationTx *types.Transaction) (types.Bid, error) Store(bidId suave.BidId, sourceTx *types.Transaction, caller common.Address, key string, value []byte) (suave.Bid, error) Retrieve(bid types.BidId, caller common.Address, key string) ([]byte, error) - FetchBidById(suave.BidId) (types.Bid, error) - FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid + FetchBidById(suave.BidId) (suave.Bid, error) + FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []suave.Bid } type SuaveContext struct { diff --git a/suave/backends/backend_testing.go b/suave/backends/backend_testing.go new file mode 100644 index 0000000000..49232b2dbc --- /dev/null +++ b/suave/backends/backend_testing.go @@ -0,0 +1,36 @@ +package backends + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + suave "github.com/ethereum/go-ethereum/suave/core" + "github.com/stretchr/testify/require" +) + +func testBackendStore(t *testing.T, store suave.ConfidentialStoreBackend) { + bid := suave.Bid{ + Id: suave.RandomBidId(), + DecryptionCondition: 10, + AllowedPeekers: []common.Address{common.HexToAddress("0x424344")}, + Version: "default:v0:ethBundles", + } + + err := store.InitializeBid(bid) + require.NoError(t, err) + + bidRes, err := store.FetchBidById(bid.Id) + require.NoError(t, err) + require.Equal(t, bid, bidRes) + + _, err = store.Store(bid, bid.AllowedPeekers[0], "xx", []byte{0x43, 0x14}) + require.NoError(t, err) + + retrievedData, err := store.Retrieve(bid, bid.AllowedPeekers[0], "xx") + require.NoError(t, err) + require.Equal(t, []byte{0x43, 0x14}, retrievedData) + + bids := store.FetchBidsByProtocolAndBlock(10, "default:v0:ethBundles") + require.Len(t, bids, 1) + require.Equal(t, bid, bids[0]) +} diff --git a/suave/backends/redis_backends_test.go b/suave/backends/redis_backends_test.go index 5d27271672..4bbc53be5b 100644 --- a/suave/backends/redis_backends_test.go +++ b/suave/backends/redis_backends_test.go @@ -60,35 +60,6 @@ func TestRedisTransport(t *testing.T) { } } -func TestRedisStore(t *testing.T) { - mr := miniredis.RunT(t) - - redisStoreBackend := NewRedisStoreBackend(mr.Addr()) - redisStoreBackend.Start() - t.Cleanup(func() { redisStoreBackend.Stop() }) - - bid := suave.Bid{ - Id: suave.BidId{0x42}, - DecryptionCondition: uint64(13), - AllowedPeekers: []common.Address{{0x41, 0x39}}, - Version: string("vv"), - } - - err := redisStoreBackend.InitializeBid(bid) - require.NoError(t, err) - - fetchedBid, err := redisStoreBackend.FetchEngineBidById(bid.Id) - require.NoError(t, err) - require.Equal(t, bid, fetchedBid) - - _, err = redisStoreBackend.Store(bid, bid.AllowedPeekers[0], "xx", []byte{0x43, 0x14}) - require.NoError(t, err) - - retrievedData, err := redisStoreBackend.Retrieve(bid, bid.AllowedPeekers[0], "xx") - require.NoError(t, err) - require.Equal(t, []byte{0x43, 0x14}, retrievedData) -} - func TestEngineOnRedis(t *testing.T) { mrStore1 := miniredis.RunT(t) mrStore2 := miniredis.RunT(t) @@ -174,7 +145,7 @@ func TestEngineOnRedis(t *testing.T) { require.NoError(t, err) require.Equal(t, []byte{0x43, 0x14}, retrievedData) - fetchedBid, err := redisStoreBackend2.FetchEngineBidById(bid.Id) + fetchedBid, err := redisStoreBackend2.FetchBidById(bid.Id) require.NoError(t, err) fetchedBidJson, err := json.Marshal(fetchedBid) diff --git a/suave/backends/redis_store_backend.go b/suave/backends/redis_store_backend.go index 5b859e5622..a005e424fa 100644 --- a/suave/backends/redis_store_backend.go +++ b/suave/backends/redis_store_backend.go @@ -112,10 +112,15 @@ func (r *RedisStoreBackend) InitializeBid(bid suave.Bid) error { return err } + err = r.indexBid(bid) + if err != nil { + return err + } + return nil } -func (r *RedisStoreBackend) FetchEngineBidById(bidId suave.BidId) (suave.Bid, error) { +func (r *RedisStoreBackend) FetchBidById(bidId suave.BidId) (suave.Bid, error) { key := formatRedisBidKey(bidId) data, err := r.client.Get(r.ctx, key).Bytes() @@ -158,44 +163,39 @@ var ( mempoolConfidentialStoreBid = suave.Bid{Id: mempoolConfStoreId, AllowedPeekers: []common.Address{mempoolConfStoreAddr}} ) -func (r *RedisStoreBackend) SubmitBid(bid types.Bid) error { +func (r *RedisStoreBackend) indexBid(bid suave.Bid) error { defer log.Info("bid submitted", "bid", bid, "store", r.Store) - var bidsByBlockAndProtocol []types.Bid + var bidsByBlockAndProtocol []suave.BidId bidsByBlockAndProtocolBytes, err := r.Retrieve(mempoolConfidentialStoreBid, mempoolConfStoreAddr, fmt.Sprintf("protocol-%s-bn-%d", bid.Version, bid.DecryptionCondition)) if err == nil { - bidsByBlockAndProtocol = suave.MustDecode[[]types.Bid](bidsByBlockAndProtocolBytes) + bidsByBlockAndProtocol = suave.MustDecode[[]suave.BidId](bidsByBlockAndProtocolBytes) } // store bid by block number and by protocol + block number - bidsByBlockAndProtocol = append(bidsByBlockAndProtocol, bid) + bidsByBlockAndProtocol = append(bidsByBlockAndProtocol, bid.Id) r.Store(mempoolConfidentialStoreBid, mempoolConfStoreAddr, fmt.Sprintf("protocol-%s-bn-%d", bid.Version, bid.DecryptionCondition), suave.MustEncode(bidsByBlockAndProtocol)) return nil } -func (r *RedisStoreBackend) FetchBidById(bidId suave.BidId) (types.Bid, error) { - engineBid, err := r.FetchEngineBidById(bidId) - if err != nil { - log.Error("bid missing!", "id", bidId, "err", err) - return types.Bid{}, errors.New("not found") - } - - return types.Bid{ - Id: engineBid.Id, - Salt: engineBid.Salt, - DecryptionCondition: engineBid.DecryptionCondition, - AllowedPeekers: engineBid.AllowedPeekers, - AllowedStores: engineBid.AllowedStores, - Version: engineBid.Version, - }, nil -} - -func (r *RedisStoreBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid { +func (r *RedisStoreBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []suave.Bid { bidsByProtocolBytes, err := r.Retrieve(mempoolConfidentialStoreBid, mempoolConfStoreAddr, fmt.Sprintf("protocol-%s-bn-%d", namespace, blockNumber)) if err != nil { return nil } - defer log.Info("bids fetched", "bids", string(bidsByProtocolBytes)) - return suave.MustDecode[[]types.Bid](bidsByProtocolBytes) + + res := []suave.Bid{} + + bidIDs := suave.MustDecode[[]suave.BidId](bidsByProtocolBytes) + for _, id := range bidIDs { + bid, err := r.FetchBidById(id) + if err != nil { + continue + } + res = append(res, bid) + } + + // defer log.Info("bids fetched", "bids", string(bidsByProtocolBytes)) + return res } diff --git a/suave/backends/redis_store_backend_test.go b/suave/backends/redis_store_backend_test.go new file mode 100644 index 0000000000..07db156de7 --- /dev/null +++ b/suave/backends/redis_store_backend_test.go @@ -0,0 +1,14 @@ +package backends + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRedis_StoreSuite(t *testing.T) { + store := NewRedisStoreBackend("") + require.NoError(t, store.Start()) + + testBackendStore(t, store) +} diff --git a/suave/core/engine.go b/suave/core/engine.go index aeb9cb63b5..d4cbadd3af 100644 --- a/suave/core/engine.go +++ b/suave/core/engine.go @@ -170,28 +170,23 @@ func (e *ConfidentialStoreEngine) InitializeBid(bid types.Bid, creationTx *types return types.Bid{}, fmt.Errorf("confidential engine: store backend failed to initialize bid: %w", err) } - // send the bid to the internal mempool - if err := e.backend.SubmitBid(bid); err != nil { - return types.Bid{}, fmt.Errorf("failed to submit to mempool: %w", err) - } - return bid, nil } -func (e *ConfidentialStoreEngine) SubmitBid(bid types.Bid) error { - return e.backend.SubmitBid(bid) +func (e *ConfidentialStoreEngine) StoreBid(bid Bid) error { + return e.backend.InitializeBid(bid) } -func (e *ConfidentialStoreEngine) FetchBidById(bidId BidId) (types.Bid, error) { +func (e *ConfidentialStoreEngine) FetchBidById(bidId BidId) (Bid, error) { return e.backend.FetchBidById(bidId) } -func (e *ConfidentialStoreEngine) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid { +func (e *ConfidentialStoreEngine) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []Bid { return e.backend.FetchBidsByProtocolAndBlock(blockNumber, namespace) } func (e *ConfidentialStoreEngine) Store(bidId BidId, sourceTx *types.Transaction, caller common.Address, key string, value []byte) (Bid, error) { - bid, err := e.backend.FetchEngineBidById(bidId) + bid, err := e.backend.FetchBidById(bidId) if err != nil { return Bid{}, fmt.Errorf("confidential engine: could not fetch bid %x while storing: %w", bidId, err) } @@ -231,7 +226,7 @@ func (e *ConfidentialStoreEngine) Store(bidId BidId, sourceTx *types.Transaction } func (e *ConfidentialStoreEngine) Retrieve(bidId BidId, caller common.Address, key string) ([]byte, error) { - bid, err := e.backend.FetchEngineBidById(bidId) + bid, err := e.backend.FetchBidById(bidId) if err != nil { return []byte{}, fmt.Errorf("confidential engine: could not fetch bid %x while retrieving: %w", bidId, err) } @@ -329,8 +324,6 @@ func (e *ConfidentialStoreEngine) NewMessage(message DAMessage) error { if !errors.Is(err, ErrBidAlreadyPresent) { return fmt.Errorf("unexpected error while initializing bid from transport: %w", err) } - } else { - e.backend.SubmitBid(innerBid) } _, err = e.backend.Store(message.Bid, message.Caller, message.Key, message.Value) diff --git a/suave/core/engine_test.go b/suave/core/engine_test.go index c85e6459ea..4d4403cdce 100644 --- a/suave/core/engine_test.go +++ b/suave/core/engine_test.go @@ -42,11 +42,11 @@ func (*FakeStoreBackend) Retrieve(bid Bid, caller common.Address, key string) ([ return nil, errors.New("not implemented") } -func (*FakeStoreBackend) FetchBidById(BidId) (types.Bid, error) { - return types.Bid{}, nil +func (*FakeStoreBackend) FetchBidById(BidId) (Bid, error) { + return Bid{}, nil } -func (*FakeStoreBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid { +func (*FakeStoreBackend) FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []Bid { return nil } diff --git a/suave/core/types.go b/suave/core/types.go index 8a2ebbbdd8..b222016bd4 100644 --- a/suave/core/types.go +++ b/suave/core/types.go @@ -26,6 +26,17 @@ type Bid struct { Signature []byte } +func (b *Bid) ToInnerBid() types.Bid { + return types.Bid{ + Id: b.Id, + Salt: b.Salt, + DecryptionCondition: b.DecryptionCondition, + AllowedPeekers: b.AllowedPeekers, + AllowedStores: b.AllowedStores, + Version: b.Version, + } +} + type MEVMBid = types.Bid type BuildBlockArgs = types.BuildBlockArgs @@ -38,14 +49,10 @@ type ConfidentialStoreBackend interface { node.Lifecycle InitializeBid(bid Bid) error - FetchEngineBidById(bidId BidId) (Bid, error) Store(bid Bid, caller common.Address, key string, value []byte) (Bid, error) Retrieve(bid Bid, caller common.Address, key string) ([]byte, error) - FetchBidById(BidId) (types.Bid, error) - FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []types.Bid - - // TODO: remove this - SubmitBid(types.Bid) error + FetchBidById(BidId) (Bid, error) + FetchBidsByProtocolAndBlock(blockNumber uint64, namespace string) []Bid } type ConfidentialEthBackend interface { diff --git a/suave/e2e/workflow_test.go b/suave/e2e/workflow_test.go index 15c4809a8e..4640477c8d 100644 --- a/suave/e2e/workflow_test.go +++ b/suave/e2e/workflow_test.go @@ -137,22 +137,22 @@ func TestMempool(t *testing.T) { { targetBlock := uint64(16103213) - bid1 := types.Bid{ + bid1 := suave.Bid{ Id: suave.RandomBidId(), DecryptionCondition: targetBlock, AllowedPeekers: []common.Address{common.HexToAddress("0x424344")}, Version: "default:v0:ethBundles", } - bid2 := types.Bid{ + bid2 := suave.Bid{ Id: suave.RandomBidId(), DecryptionCondition: targetBlock, AllowedPeekers: []common.Address{common.HexToAddress("0x424344")}, Version: "default:v0:ethBundles", } - require.NoError(t, fr.ConfidentialStore().SubmitBid(bid1)) - require.NoError(t, fr.ConfidentialStore().SubmitBid(bid2)) + require.NoError(t, fr.ConfidentialStore().StoreBid(bid1)) + require.NoError(t, fr.ConfidentialStore().StoreBid(bid2)) inoutAbi := mustParseMethodAbi(`[ { "inputs": [ { "internalType": "uint64", "name": "cond", "type": "uint64" }, { "internalType": "string", "name": "namespace", "type": "string" } ], "name": "fetchBids", "outputs": [ { "components": [ { "internalType": "Suave.BidId", "name": "id", "type": "bytes16" }, { "internalType": "Suave.BidId", "name": "salt", "type": "bytes16" }, { "internalType": "uint64", "name": "decryptionCondition", "type": "uint64" }, { "internalType": "address[]", "name": "allowedPeekers", "type": "address[]" }, { "internalType": "address[]", "name": "allowedStores", "type": "address[]" }, { "internalType": "string", "name": "version", "type": "string" } ], "internalType": "struct Suave.Bid[]", "name": "", "type": "tuple[]" } ], "stateMutability": "view", "type": "function" } ]`, "fetchBids") @@ -174,13 +174,13 @@ func TestMempool(t *testing.T) { var bids []suave.Bid require.NoError(t, mapstructure.Decode(unpacked[0], &bids)) - require.Equal(t, bid1, types.Bid{ + require.Equal(t, bid1, suave.Bid{ Id: bids[0].Id, DecryptionCondition: bids[0].DecryptionCondition, AllowedPeekers: bids[0].AllowedPeekers, Version: bids[0].Version, }) - require.Equal(t, bid2, types.Bid{ + require.Equal(t, bid2, suave.Bid{ Id: bids[1].Id, DecryptionCondition: bids[1].DecryptionCondition, AllowedPeekers: bids[1].AllowedPeekers,