diff --git a/cmd/node/config/config.toml b/cmd/node/config/config.toml index 7f07d4dd380..2225aedccb6 100644 --- a/cmd/node/config/config.toml +++ b/cmd/node/config/config.toml @@ -199,6 +199,20 @@ MaxBatchSize = 30000 MaxOpenFiles = 10 +[ReceiptsDataStorage] + [ReceiptsDataStorage.Cache] + Name = "ReceiptsDataStorage" + Capacity = 500000 + Type = "SizeLRU" + SizeInBytes = 52428800 #50MB + [ReceiptsDataStorage.DB] + FilePath = "ReceiptsData" + Type = "LvlDBSerial" + BatchDelaySeconds = 2 + MaxBatchSize = 30000 + MaxOpenFiles = 10 + + [UnsignedTransactionStorage] [UnsignedTransactionStorage.Cache] Name = "UnsignedTransactionStorage" @@ -392,6 +406,15 @@ Type = "TxCache" Shards = 16 +[ReceiptsDataPool] + Name = "ReceiptsDataPool" + Capacity = 600000 + SizePerSender = 20000 + SizeInBytes = 104857600 #100MB + SizeInBytesPerSender = 12288000 + Type = "SizeLRU" + Shards = 16 + [TrieNodesChunksDataPool] Name = "TrieNodesDataPool" Capacity = 400 diff --git a/config/config.go b/config/config.go index cf0ba12fc80..84ca3402401 100644 --- a/config/config.go +++ b/config/config.go @@ -146,6 +146,7 @@ type Config struct { PeerBlockBodyStorage StorageConfig BlockHeaderStorage StorageConfig TxStorage StorageConfig + ReceiptsDataStorage StorageConfig UnsignedTransactionStorage StorageConfig RewardTxStorage StorageConfig ShardHdrNonceHashStorage StorageConfig @@ -171,6 +172,7 @@ type Config struct { TxBlockBodyDataPool CacheConfig PeerBlockBodyDataPool CacheConfig TxDataPool CacheConfig + ReceiptsDataPool CacheConfig UnsignedTransactionDataPool CacheConfig RewardTransactionDataPool CacheConfig TrieNodesChunksDataPool CacheConfig diff --git a/dataRetriever/dataPool/dataPool.go b/dataRetriever/dataPool/dataPool.go index 67b55cbfaee..82490fb8063 100644 --- a/dataRetriever/dataPool/dataPool.go +++ b/dataRetriever/dataPool/dataPool.go @@ -26,6 +26,7 @@ type dataPool struct { peerAuthentications storage.Cacher heartbeats storage.Cacher validatorsInfo dataRetriever.ShardedDataCacherNotifier + receipts storage.Cacher } // DataPoolArgs represents the data pool's constructor structure @@ -44,6 +45,7 @@ type DataPoolArgs struct { PeerAuthentications storage.Cacher Heartbeats storage.Cacher ValidatorsInfo dataRetriever.ShardedDataCacherNotifier + Receipts storage.Cacher } // NewDataPool creates a data pools holder object @@ -90,6 +92,9 @@ func NewDataPool(args DataPoolArgs) (*dataPool, error) { if check.IfNil(args.ValidatorsInfo) { return nil, dataRetriever.ErrNilValidatorInfoPool } + if check.IfNil(args.Receipts) { + return nil, dataRetriever.ErrNilReceiptsPool + } return &dataPool{ transactions: args.Transactions, @@ -106,6 +111,7 @@ func NewDataPool(args DataPoolArgs) (*dataPool, error) { peerAuthentications: args.PeerAuthentications, heartbeats: args.Heartbeats, validatorsInfo: args.ValidatorsInfo, + receipts: args.Receipts, }, nil } @@ -179,6 +185,11 @@ func (dp *dataPool) ValidatorsInfo() dataRetriever.ShardedDataCacherNotifier { return dp.validatorsInfo } +// Receipts returns the holder for receipts info +func (dp *dataPool) Receipts() storage.Cacher { + return dp.receipts +} + // Close closes all the components func (dp *dataPool) Close() error { var lastError error diff --git a/dataRetriever/dataPool/dataPool_test.go b/dataRetriever/dataPool/dataPool_test.go index b948b7f2d44..9ad5ff96059 100644 --- a/dataRetriever/dataPool/dataPool_test.go +++ b/dataRetriever/dataPool/dataPool_test.go @@ -30,6 +30,7 @@ func createMockDataPoolArgs() dataPool.DataPoolArgs { PeerAuthentications: testscommon.NewCacherStub(), Heartbeats: testscommon.NewCacherStub(), ValidatorsInfo: testscommon.NewShardedDataStub(), + Receipts: testscommon.NewCacherStub(), } } diff --git a/dataRetriever/errors.go b/dataRetriever/errors.go index a015e6e10ed..b024c0c8639 100644 --- a/dataRetriever/errors.go +++ b/dataRetriever/errors.go @@ -71,9 +71,15 @@ var ErrResolveTypeUnknown = errors.New("unknown resolve type") // ErrNilMiniblocksPool signals that a nil miniblocks pool has been provided var ErrNilMiniblocksPool = errors.New("nil miniblocks pool") +// ErrNilReceiptsPool signals that a nil receipts pool has been provided +var ErrNilReceiptsPool = errors.New("nil receipts pool") + // ErrNilMiniblocksStorage signals that a nil miniblocks storage has been provided var ErrNilMiniblocksStorage = errors.New("nil miniblocks storage") +// ErrNilReceiptsStorage signals that a nil receipts storage has been provided +var ErrNilReceiptsStorage = errors.New("nil receipts storage") + // ErrNilDataPoolHolder signals that the data pool holder is nil var ErrNilDataPoolHolder = errors.New("nil data pool holder") diff --git a/dataRetriever/factory/dataPoolFactory.go b/dataRetriever/factory/dataPoolFactory.go index 6e1415ddfd8..79d2878a78a 100644 --- a/dataRetriever/factory/dataPoolFactory.go +++ b/dataRetriever/factory/dataPoolFactory.go @@ -146,6 +146,12 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error) return nil, fmt.Errorf("%w while creating the cache for the validator info results", err) } + cacherCfg = factory.GetCacherFromConfig(mainConfig.ReceiptsDataPool) + receiptsDataPool, err := storageunit.NewCache(cacherCfg) + if err != nil { + return nil, fmt.Errorf("%w while creating the cache for the receipts data", err) + } + currBlockTransactions := dataPool.NewCurrentBlockTransactionsPool() currEpochValidatorInfo := dataPool.NewCurrentEpochValidatorInfoPool() dataPoolArgs := dataPool.DataPoolArgs{ @@ -163,6 +169,7 @@ func NewDataPoolFromConfig(args ArgsDataPool) (dataRetriever.PoolsHolder, error) PeerAuthentications: peerAuthPool, Heartbeats: heartbeatPool, ValidatorsInfo: validatorsInfo, + Receipts: receiptsDataPool, } return dataPool.NewDataPool(dataPoolArgs) } diff --git a/dataRetriever/factory/requestersContainer/baseRequestersContainerFactory.go b/dataRetriever/factory/requestersContainer/baseRequestersContainerFactory.go index 2ec10054d8d..3a93901ed26 100644 --- a/dataRetriever/factory/requestersContainer/baseRequestersContainerFactory.go +++ b/dataRetriever/factory/requestersContainer/baseRequestersContainerFactory.go @@ -116,7 +116,6 @@ func (brcf *baseRequestersContainerFactory) generateCommonRequesters() error { } func (brcf *baseRequestersContainerFactory) generateTxRequesters(topic string) error { - shardC := brcf.shardCoordinator noOfShards := shardC.NumberOfShards() @@ -172,6 +171,26 @@ func (brcf *baseRequestersContainerFactory) createTxRequester( return requesters.NewTransactionRequester(arg) } +func (brcf *baseRequestersContainerFactory) createReceiptRequester( + topic string, + excludedTopic string, + targetShardID uint32, + numIntraShardPeers int, +) (dataRetriever.Requester, error) { + requestSender, err := brcf.createOneRequestSenderWithSpecifiedNumRequests(topic, excludedTopic, targetShardID, 0, numIntraShardPeers) + if err != nil { + return nil, err + } + + arg := requesters.ArgReceiptRequester{ + ArgBaseRequester: requesters.ArgBaseRequester{ + RequestSender: requestSender, + Marshaller: brcf.marshaller, + }, + } + return requesters.NewReceiptRequester(arg) +} + func (brcf *baseRequestersContainerFactory) generateMiniBlocksRequesters() error { shardC := brcf.shardCoordinator noOfShards := shardC.NumberOfShards() diff --git a/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory.go b/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory.go index c718f5b22a1..f025eacacc7 100644 --- a/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory.go +++ b/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory.go @@ -85,6 +85,11 @@ func (mrcf *metaRequestersContainerFactory) Create() (dataRetriever.RequestersCo return nil, err } + err = mrcf.generateReceiptRequesters() + if err != nil { + return nil, err + } + return mrcf.container, nil } @@ -177,6 +182,17 @@ func (mrcf *metaRequestersContainerFactory) generateMetaChainHeaderRequesters() return mrcf.container.Add(identifierHeader, requester) } +func (mrcf *metaRequestersContainerFactory) generateReceiptRequesters() error { + // only one receipts topic + identifierReceipt := factory.ReceiptTopic + mrcf.shardCoordinator.CommunicationIdentifier(core.MetachainShardId) + requester, err := mrcf.createReceiptRequester(identifierReceipt, EmptyExcludePeersOnTopic, core.MetachainShardId, mrcf.numTotalPeers) + if err != nil { + return err + } + + return mrcf.container.Add(identifierReceipt, requester) +} + func (mrcf *metaRequestersContainerFactory) createMetaChainHeaderRequester( identifier string, shardId uint32, diff --git a/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory_test.go b/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory_test.go index e68f4c7e5a5..42a506321ea 100644 --- a/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory_test.go +++ b/dataRetriever/factory/requestersContainer/metaRequestersContainerFactory_test.go @@ -223,8 +223,9 @@ func TestMetaRequestersContainerFactory_With4ShardsShouldWork(t *testing.T) { numRequestersTrieNodes := 2 numRequestersPeerAuth := 1 numRequesterValidatorInfo := 1 + numRequesterReceiptsData := 1 totalRequesters := numRequestersShardHeadersForMetachain + numRequesterMetablocks + numRequestersMiniBlocks + - numRequestersUnsigned + numRequestersTxs + numRequestersTrieNodes + numRequestersRewards + numRequestersPeerAuth + numRequesterValidatorInfo + numRequestersUnsigned + numRequestersTxs + numRequestersTrieNodes + numRequestersRewards + numRequestersPeerAuth + numRequesterValidatorInfo + numRequesterReceiptsData assert.Equal(t, totalRequesters, container.Len()) diff --git a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go index d7468d5302d..af8623c16ee 100644 --- a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go +++ b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory.go @@ -84,6 +84,11 @@ func (srcf *shardRequestersContainerFactory) Create() (dataRetriever.RequestersC return nil, err } + err = srcf.generateReceiptRequesters() + if err != nil { + return nil, err + } + return srcf.container, nil } @@ -136,6 +141,18 @@ func (srcf *shardRequestersContainerFactory) generateMetablockHeaderRequesters() return srcf.container.Add(identifierHdr, requester) } +func (srcf *shardRequestersContainerFactory) generateReceiptRequesters() error { + // only one receipts topic + selfShardID := srcf.shardCoordinator.SelfId() + identifierReceipt := factory.ReceiptTopic + requester, err := srcf.createReceiptRequester(identifierReceipt, EmptyExcludePeersOnTopic, selfShardID, srcf.numTotalPeers) + if err != nil { + return err + } + + return srcf.container.Add(identifierReceipt, requester) +} + func (srcf *shardRequestersContainerFactory) generateTrieNodesRequesters() error { shardC := srcf.shardCoordinator diff --git a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory_test.go b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory_test.go index e4c94491487..f166f92a7b5 100644 --- a/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory_test.go +++ b/dataRetriever/factory/requestersContainer/shardRequestersContainerFactory_test.go @@ -253,8 +253,9 @@ func TestShardRequestersContainerFactory_With4ShardsShouldWork(t *testing.T) { numRequesterTrieNodes := 1 numRequesterPeerAuth := 1 numRequesterValidatorInfo := 1 + numRequesterReceiptsData := 1 totalRequesters := numRequesterTxs + numRequesterHeaders + numRequesterMiniBlocks + numRequesterMetaBlockHeaders + - numRequesterSCRs + numRequesterRewardTxs + numRequesterTrieNodes + numRequesterPeerAuth + numRequesterValidatorInfo + numRequesterSCRs + numRequesterRewardTxs + numRequesterTrieNodes + numRequesterPeerAuth + numRequesterValidatorInfo + numRequesterReceiptsData assert.Equal(t, totalRequesters, container.Len()) } diff --git a/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go index 3d0eff8eaa9..b9c8e7bbf7e 100644 --- a/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/baseResolversContainerFactory.go @@ -417,3 +417,48 @@ func (brcf *baseResolversContainerFactory) generateValidatorInfoResolver() error return brcf.container.Add(identifierValidatorInfo, validatorInfoResolver) } + +func (brcf *baseResolversContainerFactory) generateReceiptResolver() error { + shardC := brcf.shardCoordinator + + // only one shard receipt topic + identifier := factory.ReceiptTopic + shardC.CommunicationIdentifier(shardC.SelfId()) + + receiptDataStorer, err := brcf.store.GetStorer(dataRetriever.ReceiptDataUnit) + if err != nil { + return err + } + resolverSender, err := brcf.createOneResolverSenderWithSpecifiedNumRequests(identifier, EmptyExcludePeersOnTopic, shardC.SelfId()) + if err != nil { + return err + } + + arg := resolvers.ArgReceiptResolver{ + ArgBaseResolver: resolvers.ArgBaseResolver{ + SenderResolver: resolverSender, + Marshaller: brcf.marshalizer, + AntifloodHandler: brcf.inputAntifloodHandler, + Throttler: brcf.throttler, + }, + ReceiptPool: brcf.dataPools.Receipts(), + ReceiptStorage: receiptDataStorer, + DataPacker: brcf.dataPacker, + IsFullHistoryNode: brcf.isFullHistoryNode, + } + resolver, err := resolvers.NewReceiptResolver(arg) + if err != nil { + return err + } + + err = brcf.mainMessenger.RegisterMessageProcessor(resolver.RequestTopic(), common.DefaultResolversIdentifier, resolver) + if err != nil { + return err + } + + err = brcf.fullArchiveMessenger.RegisterMessageProcessor(resolver.RequestTopic(), common.DefaultResolversIdentifier, resolver) + if err != nil { + return err + } + + return brcf.container.Add(identifier, resolver) +} diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go index b72f8c3154a..9f0973e5c2a 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory.go @@ -131,6 +131,11 @@ func (mrcf *metaResolversContainerFactory) Create() (dataRetriever.ResolversCont return nil, err } + err = mrcf.generateReceiptResolver() + if err != nil { + return nil, err + } + return mrcf.container, nil } diff --git a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go index 755672384cd..21cfea5ba9d 100644 --- a/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/metaResolversContainerFactory_test.go @@ -67,6 +67,9 @@ func createDataPoolsForMeta() dataRetriever.PoolsHolder { RewardTransactionsCalled: func() dataRetriever.ShardedDataCacherNotifier { return testscommon.NewShardedDataStub() }, + ReceiptsCalled: func() storage.Cacher { + return testscommon.NewCacherStub() + }, } return pools @@ -338,8 +341,9 @@ func TestMetaResolversContainerFactory_With4ShardsShouldWork(t *testing.T) { numResolversTrieNodes := 2 numResolversPeerAuth := 1 numResolverValidatorInfo := 1 + numRezolverReceiptsData := 1 totalResolvers := numResolversShardHeadersForMetachain + numResolverMetablocks + numResolversMiniBlocks + - numResolversUnsigned + numResolversTxs + numResolversTrieNodes + numResolversRewards + numResolversPeerAuth + numResolverValidatorInfo + numResolversUnsigned + numResolversTxs + numResolversTrieNodes + numResolversRewards + numResolversPeerAuth + numResolverValidatorInfo + numRezolverReceiptsData assert.Equal(t, totalResolvers, container.Len()) assert.Equal(t, totalResolvers, registerMainCnt) diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go index f24beaa4331..f2001ab2b6f 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory.go @@ -129,6 +129,11 @@ func (srcf *shardResolversContainerFactory) Create() (dataRetriever.ResolversCon return nil, err } + err = srcf.generateReceiptResolver() + if err != nil { + return nil, err + } + return srcf.container, nil } diff --git a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go index ca97015f3ae..9d94adb5c4e 100644 --- a/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go +++ b/dataRetriever/factory/resolverscontainer/shardResolversContainerFactory_test.go @@ -2,6 +2,7 @@ package resolverscontainer_test import ( "errors" + "github.com/stretchr/testify/require" "strings" "testing" @@ -74,6 +75,9 @@ func createDataPoolsForShard() dataRetriever.PoolsHolder { pools.RewardTransactionsCalled = func() dataRetriever.ShardedDataCacherNotifier { return testscommon.NewShardedDataStub() } + pools.ReceiptsCalled = func() storage.Cacher { + return testscommon.NewCacherStub() + } return pools } @@ -439,7 +443,8 @@ func TestShardResolversContainerFactory_With4ShardsShouldWork(t *testing.T) { args.ShardCoordinator = shardCoordinator rcf, _ := resolverscontainer.NewShardResolversContainerFactory(args) - container, _ := rcf.Create() + container, err := rcf.Create() + require.Nil(t, err) numResolverSCRs := noOfShards + 1 numResolverTxs := noOfShards + 1 @@ -450,8 +455,9 @@ func TestShardResolversContainerFactory_With4ShardsShouldWork(t *testing.T) { numResolverTrieNodes := 1 numResolverPeerAuth := 1 numResolverValidatorInfo := 1 + numResolverReceiptData := 1 totalResolvers := numResolverTxs + numResolverHeaders + numResolverMiniBlocks + numResolverMetaBlockHeaders + - numResolverSCRs + numResolverRewardTxs + numResolverTrieNodes + numResolverPeerAuth + numResolverValidatorInfo + numResolverSCRs + numResolverRewardTxs + numResolverTrieNodes + numResolverPeerAuth + numResolverValidatorInfo + numResolverReceiptData assert.Equal(t, totalResolvers, container.Len()) assert.Equal(t, totalResolvers, registerMainCnt) diff --git a/dataRetriever/interface.go b/dataRetriever/interface.go index 930b6aca124..372a5df3ac5 100644 --- a/dataRetriever/interface.go +++ b/dataRetriever/interface.go @@ -240,6 +240,7 @@ type PoolsHolder interface { PeerAuthentications() storage.Cacher Heartbeats() storage.Cacher ValidatorsInfo() ShardedDataCacherNotifier + Receipts() storage.Cacher Close() error IsInterfaceNil() bool } diff --git a/dataRetriever/requestHandlers/requestHandler.go b/dataRetriever/requestHandlers/requestHandler.go index 7166715dd3c..a95cd19f7ef 100644 --- a/dataRetriever/requestHandlers/requestHandler.go +++ b/dataRetriever/requestHandlers/requestHandler.go @@ -31,6 +31,7 @@ const uniqueHeadersSuffix = "hdr" const uniqueMetaHeadersSuffix = "mhdr" const uniqueTrieNodesSuffix = "tn" const uniqueValidatorInfoSuffix = "vi" +const uniqueReceiptSuffix = "rec" // TODO move the keys definitions that are whitelisted in core and use them in InterceptedData implementations, Identifiers() function @@ -115,6 +116,11 @@ func (rrh *resolverRequestHandler) RequestTransaction(destShardID uint32, txHash rrh.requestByHashes(destShardID, txHashes, factory.TransactionTopic, uniqueTxSuffix) } +// RequestReceipts method ask for receipts from the connected peers +func (rrh *resolverRequestHandler) RequestReceipts(receiptsHashes [][]byte) { + rrh.requestByHashes(rrh.shardID, receiptsHashes, factory.ReceiptTopic, uniqueReceiptSuffix) +} + func (rrh *resolverRequestHandler) requestByHashes(destShardID uint32, hashes [][]byte, topic string, abbreviatedTopic string) { suffix := fmt.Sprintf("%s_%d", abbreviatedTopic, destShardID) unrequestedHashes := rrh.getUnrequestedHashes(hashes, suffix) diff --git a/dataRetriever/requestHandlers/requesters/receiptRequester.go b/dataRetriever/requestHandlers/requesters/receiptRequester.go new file mode 100644 index 00000000000..ede9ea14d3b --- /dev/null +++ b/dataRetriever/requestHandlers/requesters/receiptRequester.go @@ -0,0 +1,32 @@ +package requesters + +// ArgReceiptRequester is the argument structure used to create a new receipt requester instance +type ArgReceiptRequester struct { + ArgBaseRequester +} + +type receiptRequester struct { + *baseRequester +} + +// NewReceiptRequester returns a new instance of receipt requester +func NewReceiptRequester(args ArgReceiptRequester) (*receiptRequester, error) { + err := checkArgBase(args.ArgBaseRequester) + if err != nil { + return nil, err + } + + return &receiptRequester{ + baseRequester: createBaseRequester(args.ArgBaseRequester), + }, nil +} + +// RequestDataFromHashArray requests receipt from other peers by hash array +func (requester *receiptRequester) RequestDataFromHashArray(hashes [][]byte, epoch uint32) error { + return requester.requestDataFromHashArray(hashes, epoch) +} + +// IsInterfaceNil returns true if there is no value under the interface +func (requester *receiptRequester) IsInterfaceNil() bool { + return requester == nil +} diff --git a/dataRetriever/requestHandlers/requesters/receiptRequester_test.go b/dataRetriever/requestHandlers/requesters/receiptRequester_test.go new file mode 100644 index 00000000000..a507e95ebbb --- /dev/null +++ b/dataRetriever/requestHandlers/requesters/receiptRequester_test.go @@ -0,0 +1,45 @@ +package requesters + +import ( + "testing" + + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/stretchr/testify/assert" +) + +func createMockArgReceiptRequester(argBase ArgBaseRequester) ArgReceiptRequester { + return ArgReceiptRequester{ + ArgBaseRequester: argBase, + } +} + +func TestNewReceiptRequester(t *testing.T) { + t.Parallel() + + t.Run("nil request sender should error", func(t *testing.T) { + t.Parallel() + + argsBase := createMockArgBaseRequester() + argsBase.RequestSender = nil + requester, err := NewReceiptRequester(createMockArgReceiptRequester(argsBase)) + assert.Equal(t, dataRetriever.ErrNilRequestSender, err) + assert.True(t, check.IfNil(requester)) + }) + t.Run("nil marshaller should error", func(t *testing.T) { + t.Parallel() + + argsBase := createMockArgBaseRequester() + argsBase.Marshaller = nil + requester, err := NewReceiptRequester(createMockArgReceiptRequester(argsBase)) + assert.Equal(t, dataRetriever.ErrNilMarshalizer, err) + assert.True(t, check.IfNil(requester)) + }) + t.Run("should work", func(t *testing.T) { + t.Parallel() + + requester, err := NewReceiptRequester(createMockArgReceiptRequester(createMockArgBaseRequester())) + assert.Nil(t, err) + assert.False(t, check.IfNil(requester)) + }) +} diff --git a/dataRetriever/resolvers/receiptResolver.go b/dataRetriever/resolvers/receiptResolver.go new file mode 100644 index 00000000000..d9baf4976ba --- /dev/null +++ b/dataRetriever/resolvers/receiptResolver.go @@ -0,0 +1,191 @@ +package resolvers + +import ( + "fmt" + + "github.com/multiversx/mx-chain-core-go/core" + "github.com/multiversx/mx-chain-core-go/core/check" + "github.com/multiversx/mx-chain-core-go/data/batch" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/p2p" + "github.com/multiversx/mx-chain-go/storage" + logger "github.com/multiversx/mx-chain-logger-go" +) + +// maxBuffToSendBulkReceipts represents max buffer size to send in bytes +const maxBuffToSendBulkReceipts = 1 << 18 // 256KB + +// ArgReceiptResolver is the argument structure used to create a new receiptResolver instance +type ArgReceiptResolver struct { + ArgBaseResolver + ReceiptPool storage.Cacher + ReceiptStorage storage.Storer + DataPacker dataRetriever.DataPacker + IsFullHistoryNode bool +} + +// receiptResolver is a wrapper over Resolver that is specialized in resolving receipts requests +type receiptResolver struct { + *baseResolver + messageProcessor + baseStorageResolver + receiptPool storage.Cacher + dataPacker dataRetriever.DataPacker +} + +// NewReceiptResolver creates a receipt resolver +func NewReceiptResolver(arg ArgReceiptResolver) (*receiptResolver, error) { + err := checkArgReceiptResolver(arg) + if err != nil { + return nil, err + } + + rcResolver := &receiptResolver{ + baseResolver: &baseResolver{ + TopicResolverSender: arg.SenderResolver, + }, + receiptPool: arg.ReceiptPool, + baseStorageResolver: createBaseStorageResolver(arg.ReceiptStorage, arg.IsFullHistoryNode), + dataPacker: arg.DataPacker, + messageProcessor: messageProcessor{ + marshalizer: arg.Marshaller, + antifloodHandler: arg.AntifloodHandler, + topic: arg.SenderResolver.RequestTopic(), + throttler: arg.Throttler, + }, + } + + return rcResolver, nil +} + +// ProcessReceivedMessage will be the callback func from the p2p.Messenger and will be called each time a new message was received +// (for the topic this validator was registered to, usually a request topic) +func (rcRes *receiptResolver) ProcessReceivedMessage(message p2p.MessageP2P, fromConnectedPeer core.PeerID, source p2p.MessageHandler) error { + err := rcRes.canProcessMessage(message, fromConnectedPeer) + if err != nil { + return err + } + + rcRes.throttler.StartProcessing() + defer rcRes.throttler.EndProcessing() + + rd, err := rcRes.parseReceivedMessage(message, fromConnectedPeer) + if err != nil { + return err + } + + switch rd.Type { + case dataRetriever.HashType: + err = rcRes.resolveReceiptRequestByHash(rd.Value, message.Peer(), rd.Epoch, source) + case dataRetriever.HashArrayType: + err = rcRes.resolveReceiptRequestByHashArray(rd.Value, message.Peer(), rd.Epoch, source) + default: + err = dataRetriever.ErrRequestTypeNotImplemented + } + + if err != nil { + err = fmt.Errorf("%w for hash %s", err, logger.DisplayByteSlice(rd.Value)) + } + + return err +} + +func (rcRes *receiptResolver) resolveReceiptRequestByHash(hash []byte, pid core.PeerID, epoch uint32, source p2p.MessageHandler) error { + receipt, err := rcRes.fetchReceiptAsByteSlice(hash, epoch) + if err != nil { + return err + } + + b := &batch.Batch{ + Data: [][]byte{receipt}, + } + buffToSend, err := rcRes.marshalizer.Marshal(b) + if err != nil { + return err + } + + return rcRes.Send(buffToSend, pid, source) +} + +func (rcRes *receiptResolver) fetchReceiptAsByteSlice(hash []byte, epoch uint32) ([]byte, error) { + value, ok := rcRes.receiptPool.Peek(hash) + if ok { + return rcRes.marshalizer.Marshal(value) + } + + buff, err := rcRes.getFromStorage(hash, epoch) + if err != nil { + rcRes.DebugHandler().LogFailedToResolveData( + rcRes.topic, + hash, + err, + ) + + return nil, err + } + + rcRes.DebugHandler().LogSucceededToResolveData(rcRes.topic, hash) + + return buff, nil +} + +func (rcRes *receiptResolver) resolveReceiptRequestByHashArray(mbBuff []byte, pid core.PeerID, epoch uint32, source p2p.MessageHandler) error { + b := batch.Batch{} + err := rcRes.marshalizer.Unmarshal(&b, mbBuff) + if err != nil { + return err + } + hashes := b.Data + + var errFetch error + errorsFound := 0 + rcpBuffSlice := make([][]byte, 0, len(hashes)) + for _, hash := range hashes { + receiptBytes, errTemp := rcRes.fetchReceiptAsByteSlice(hash, epoch) + if errTemp != nil { + errFetch = fmt.Errorf("%w for hash %s", errTemp, logger.DisplayByteSlice(hash)) + log.Trace("fetchReceiptAsByteSlice missing", + "error", errFetch.Error(), + "hash", hash) + errorsFound++ + + continue + } + rcpBuffSlice = append(rcpBuffSlice, receiptBytes) + } + + buffsToSend, errPack := rcRes.dataPacker.PackDataInChunks(rcpBuffSlice, maxBuffToSendBulkReceipts) + if errPack != nil { + return errPack + } + + for _, buff := range buffsToSend { + errSend := rcRes.Send(buff, pid, source) + if errSend != nil { + return errSend + } + } + + if errFetch != nil { + errFetch = fmt.Errorf("resolveReceiptRequestByHashArray last error %w from %d encountered errors", errFetch, errorsFound) + } + + return errFetch +} + +func checkArgReceiptResolver(arg ArgReceiptResolver) error { + err := checkArgBase(arg.ArgBaseResolver) + if err != nil { + return err + } + if check.IfNil(arg.ReceiptPool) { + return dataRetriever.ErrNilReceiptsPool + } + if check.IfNil(arg.ReceiptStorage) { + return dataRetriever.ErrNilReceiptsStorage + } + if check.IfNil(arg.DataPacker) { + return dataRetriever.ErrNilDataPacker + } + return nil +} diff --git a/dataRetriever/resolvers/receiptResolver_test.go b/dataRetriever/resolvers/receiptResolver_test.go new file mode 100644 index 00000000000..acf92419bcf --- /dev/null +++ b/dataRetriever/resolvers/receiptResolver_test.go @@ -0,0 +1,118 @@ +package resolvers_test + +import ( + "errors" + "testing" + + "github.com/multiversx/mx-chain-core-go/data/batch" + "github.com/multiversx/mx-chain-core-go/data/state" + "github.com/multiversx/mx-chain-go/dataRetriever" + "github.com/multiversx/mx-chain-go/dataRetriever/mock" + "github.com/multiversx/mx-chain-go/dataRetriever/resolvers" + "github.com/multiversx/mx-chain-go/testscommon" + "github.com/multiversx/mx-chain-go/testscommon/p2pmocks" + storageStubs "github.com/multiversx/mx-chain-go/testscommon/storage" + "github.com/stretchr/testify/require" +) + +func createMockArgReceiptResolver() resolvers.ArgReceiptResolver { + return resolvers.ArgReceiptResolver{ + ArgBaseResolver: createMockArgBaseResolver(), + ReceiptPool: testscommon.NewCacherStub(), + ReceiptStorage: &storageStubs.StorerStub{}, + DataPacker: &mock.DataPackerStub{}, + } +} + +func TestNewReceiptResolver(t *testing.T) { + t.Parallel() + + t.Run("nil receipt pool should error", func(t *testing.T) { + args := createMockArgReceiptResolver() + args.ReceiptPool = nil + + receiptResolver, err := resolvers.NewReceiptResolver(args) + require.Equal(t, dataRetriever.ErrNilReceiptsPool, err) + require.Nil(t, receiptResolver) + }) + t.Run("nil receipt storage should error", func(t *testing.T) { + args := createMockArgReceiptResolver() + args.ReceiptStorage = nil + + receiptResolver, err := resolvers.NewReceiptResolver(args) + require.Equal(t, dataRetriever.ErrNilReceiptsStorage, err) + require.Nil(t, receiptResolver) + }) + + t.Run("should work", func(t *testing.T) { + args := createMockArgReceiptResolver() + + receiptResolver, err := resolvers.NewReceiptResolver(args) + require.Nil(t, err) + require.NotNil(t, receiptResolver) + }) +} + +func TestReceiptResolver_ProcessReceivedMessageUnmarshalFails(t *testing.T) { + t.Parallel() + + goodMarshalizer := &mock.MarshalizerMock{} + cnt := 0 + marshalizer := &mock.MarshalizerStub{ + MarshalCalled: goodMarshalizer.Marshal, + UnmarshalCalled: func(obj interface{}, buff []byte) error { + cnt++ + if cnt > 1 { + return expectedErr + } + return goodMarshalizer.Unmarshal(obj, buff) + }, + } + receiptHash := []byte("aaa") + receiptList := make([][]byte, 0) + receiptList = append(receiptList, receiptHash) + requestedBuff, merr := goodMarshalizer.Marshal(&batch.Batch{Data: receiptList}) + require.Nil(t, merr) + + cache := testscommon.NewCacherStub() + cache.PeekCalled = func(key []byte) (value interface{}, ok bool) { + return nil, false + } + + arg := createMockArgReceiptResolver() + arg.ReceiptPool = cache + arg.ReceiptStorage = &storageStubs.StorerStub{ + GetCalled: func(key []byte) (i []byte, e error) { + body := state.Receipt{} + buff, _ := goodMarshalizer.Marshal(&body) + return buff, nil + }, + } + arg.Marshaller = marshalizer + arg.DataPacker = &mock.DataPackerStub{ + PackDataInChunksCalled: func(data [][]byte, limit int) ([][]byte, error) { + require.Fail(t, "should not have been called") + return nil, nil + }, + } + mbRes, _ := resolvers.NewReceiptResolver(arg) + + err := mbRes.ProcessReceivedMessage( + createRequestMsg(dataRetriever.HashArrayType, requestedBuff), + fromConnectedPeerId, + &p2pmocks.MessengerStub{}, + ) + + require.True(t, errors.Is(err, expectedErr)) + require.True(t, arg.Throttler.(*mock.ThrottlerStub).StartWasCalled()) + require.True(t, arg.Throttler.(*mock.ThrottlerStub).EndWasCalled()) +} + +func TestReceiptResolver_Close(t *testing.T) { + t.Parallel() + + arg := createMockArgReceiptResolver() + mbRes, _ := resolvers.NewReceiptResolver(arg) + + require.Nil(t, mbRes.Close()) +} diff --git a/dataRetriever/unitType.go b/dataRetriever/unitType.go index 22bba7dc2b8..da037492762 100644 --- a/dataRetriever/unitType.go +++ b/dataRetriever/unitType.go @@ -49,6 +49,8 @@ const ( PeerAccountsUnit UnitType = 21 // ScheduledSCRsUnit is the scheduled SCRs storage unit identifier ScheduledSCRsUnit UnitType = 22 + // ReceiptDataUnit is the receipt storage unit identifier + ReceiptDataUnit UnitType = 23 // ShardHdrNonceHashDataUnit is the header nonce-hash pair data unit identifier //TODO: Add only unit types lower than 100 diff --git a/epochStart/bootstrap/process_test.go b/epochStart/bootstrap/process_test.go index 552148003d6..33117a335dc 100644 --- a/epochStart/bootstrap/process_test.go +++ b/epochStart/bootstrap/process_test.go @@ -142,6 +142,8 @@ func createMockEpochStartBootstrapArgs( PeerAccountsTrieStorage: generalCfg.PeerAccountsTrieStorage, HeartbeatV2: generalCfg.HeartbeatV2, Hardfork: generalCfg.Hardfork, + ReceiptsDataStorage: generalCfg.ReceiptsDataStorage, + ReceiptsDataPool: generalCfg.ReceiptsDataPool, EvictionWaitingList: config.EvictionWaitingListConfig{ HashesSize: 100, RootHashesSize: 100, diff --git a/go.mod b/go.mod index e2d3cb99819..cc810550931 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 github.com/mitchellh/mapstructure v1.5.0 github.com/multiversx/mx-chain-communication-go v1.1.0 - github.com/multiversx/mx-chain-core-go v1.2.21 + github.com/multiversx/mx-chain-core-go v1.2.22-0.20240821080303-1673ce1a0fd8 github.com/multiversx/mx-chain-crypto-go v1.2.12 github.com/multiversx/mx-chain-es-indexer-go v1.7.4 github.com/multiversx/mx-chain-logger-go v1.0.15 diff --git a/go.sum b/go.sum index 5c4d74b40ab..21fd4313714 100644 --- a/go.sum +++ b/go.sum @@ -387,8 +387,8 @@ github.com/multiversx/concurrent-map v0.1.4 h1:hdnbM8VE4b0KYJaGY5yJS2aNIW9TFFsUY github.com/multiversx/concurrent-map v0.1.4/go.mod h1:8cWFRJDOrWHOTNSqgYCUvwT7c7eFQ4U2vKMOp4A/9+o= github.com/multiversx/mx-chain-communication-go v1.1.0 h1:J7bX6HoN3HiHY7cUeEjG8AJWgQDDPcY+OPDOsSUOkRE= github.com/multiversx/mx-chain-communication-go v1.1.0/go.mod h1:WK6bP4pGEHGDDna/AYRIMtl6G9OA0NByI1Lw8PmOnRM= -github.com/multiversx/mx-chain-core-go v1.2.21 h1:+XVKznPTlUU5EFS1A8chtS8fStW60upRIyF4Pgml19I= -github.com/multiversx/mx-chain-core-go v1.2.21/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= +github.com/multiversx/mx-chain-core-go v1.2.22-0.20240821080303-1673ce1a0fd8 h1:zlm+Brw3jPj5lQ8m7wGLwYWD4ebWArGd5qE5/99qIWQ= +github.com/multiversx/mx-chain-core-go v1.2.22-0.20240821080303-1673ce1a0fd8/go.mod h1:B5zU4MFyJezmEzCsAHE9YNULmGCm2zbPHvl9hazNxmE= github.com/multiversx/mx-chain-crypto-go v1.2.12 h1:zWip7rpUS4CGthJxfKn5MZfMfYPjVjIiCID6uX5BSOk= github.com/multiversx/mx-chain-crypto-go v1.2.12/go.mod h1:HzcPpCm1zanNct/6h2rIh+MFrlXbjA5C8+uMyXj3LI4= github.com/multiversx/mx-chain-es-indexer-go v1.7.4 h1:SjJk9G9SN8baz0sFIU2jymYCfx3XiikGEB2wW0jwvfw= diff --git a/integrationTests/testInitializer.go b/integrationTests/testInitializer.go index 06dc1a24866..e9e7274c5ee 100644 --- a/integrationTests/testInitializer.go +++ b/integrationTests/testInitializer.go @@ -405,6 +405,7 @@ func CreateStore(numOfShards uint32) dataRetriever.StorageService { store.AddStorer(dataRetriever.StatusMetricsUnit, CreateMemUnit()) store.AddStorer(dataRetriever.ReceiptsUnit, CreateMemUnit()) store.AddStorer(dataRetriever.ScheduledSCRsUnit, CreateMemUnit()) + store.AddStorer(dataRetriever.ReceiptDataUnit, CreateMemUnit()) for i := uint32(0); i < numOfShards; i++ { hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(i) diff --git a/node/chainSimulator/components/storageService.go b/node/chainSimulator/components/storageService.go index 9a2a7c4860f..36ae6f10ad8 100644 --- a/node/chainSimulator/components/storageService.go +++ b/node/chainSimulator/components/storageService.go @@ -29,6 +29,7 @@ func CreateStore(numOfShards uint32) dataRetriever.StorageService { store.AddStorer(dataRetriever.EpochByHashUnit, CreateMemUnit()) store.AddStorer(dataRetriever.ResultsHashesByTxHashUnit, CreateMemUnit()) store.AddStorer(dataRetriever.TrieEpochRootHashUnit, CreateMemUnit()) + store.AddStorer(dataRetriever.ReceiptDataUnit, CreateMemUnit()) for i := uint32(0); i < numOfShards; i++ { hdrNonceHashDataUnit := dataRetriever.ShardHdrNonceHashDataUnit + dataRetriever.UnitType(i) diff --git a/node/chainSimulator/components/storageService_test.go b/node/chainSimulator/components/storageService_test.go index 3be398b53e6..dfd2076859c 100644 --- a/node/chainSimulator/components/storageService_test.go +++ b/node/chainSimulator/components/storageService_test.go @@ -37,6 +37,7 @@ func TestCreateStore(t *testing.T) { dataRetriever.ResultsHashesByTxHashUnit, dataRetriever.TrieEpochRootHashUnit, dataRetriever.ShardHdrNonceHashDataUnit, + dataRetriever.ReceiptDataUnit, dataRetriever.UnitType(101), // shard 2 } diff --git a/process/factory/factory.go b/process/factory/factory.go index ce1ade743ac..eab1cdd1387 100644 --- a/process/factory/factory.go +++ b/process/factory/factory.go @@ -19,6 +19,8 @@ const ( AccountTrieNodesTopic = "accountTrieNodes" // ValidatorTrieNodesTopic is used for sharding validator state trie nodes ValidatorTrieNodesTopic = "validatorTrieNodes" + // ReceiptTopic is the topic used for sharing receipts + ReceiptTopic = "receipts" ) // SystemVirtualMachine is a byte array identifier for the smart contract address created for system VM diff --git a/storage/factory/storageServiceFactory.go b/storage/factory/storageServiceFactory.go index c153e6b2cc8..256a752da20 100644 --- a/storage/factory/storageServiceFactory.go +++ b/storage/factory/storageServiceFactory.go @@ -225,6 +225,16 @@ func (psf *StorageServiceFactory) createAndAddBaseStorageUnits( } store.AddStorer(dataRetriever.MiniBlockUnit, miniBlockUnit) + receiptDataUnitArgs, err := psf.createPruningStorerArgs(psf.generalConfig.ReceiptsDataStorage, disabledCustomDatabaseRemover) + if err != nil { + return err + } + receiptDataUnit, err := psf.createPruningPersister(receiptDataUnitArgs) + if err != nil { + return fmt.Errorf("%w for ReceiptDataStorage", err) + } + store.AddStorer(dataRetriever.ReceiptDataUnit, receiptDataUnit) + metaBlockUnitArgs, err := psf.createPruningStorerArgs(psf.generalConfig.MetaBlockStorage, disabledCustomDatabaseRemover) if err != nil { return err diff --git a/storage/factory/storageServiceFactory_test.go b/storage/factory/storageServiceFactory_test.go index e45308f48d2..bdbda02d496 100644 --- a/storage/factory/storageServiceFactory_test.go +++ b/storage/factory/storageServiceFactory_test.go @@ -40,6 +40,7 @@ func createMockArgument(t *testing.T) StorageServiceFactoryArgs { BootstrapStorage: createMockStorageConfig("BootstrapStorage"), MiniBlocksStorage: createMockStorageConfig("MiniBlocksStorage"), MetaBlockStorage: createMockStorageConfig("MetaBlockStorage"), + ReceiptsDataStorage: createMockStorageConfig("ReceiptDataStorage"), MetaHdrNonceHashStorage: createMockStorageConfig("MetaHdrNonceHashStorage"), BlockHeaderStorage: createMockStorageConfig("BlockHeaderStorage"), AccountsTrieStorage: createMockStorageConfig("AccountsTrieStorage"), @@ -408,7 +409,7 @@ func TestStorageServiceFactory_CreateForShard(t *testing.T) { assert.Nil(t, err) assert.False(t, check.IfNil(storageService)) allStorers := storageService.GetAllStorers() - expectedStorers := 23 + expectedStorers := 24 assert.Equal(t, expectedStorers, len(allStorers)) storer, _ := storageService.GetStorer(dataRetriever.UserAccountsUnit) @@ -430,7 +431,7 @@ func TestStorageServiceFactory_CreateForShard(t *testing.T) { assert.False(t, check.IfNil(storageService)) allStorers := storageService.GetAllStorers() numDBLookupExtensionUnits := 6 - expectedStorers := 23 - numDBLookupExtensionUnits + expectedStorers := 24 - numDBLookupExtensionUnits assert.Equal(t, expectedStorers, len(allStorers)) _ = storageService.CloseAll() }) @@ -444,7 +445,7 @@ func TestStorageServiceFactory_CreateForShard(t *testing.T) { assert.Nil(t, err) assert.False(t, check.IfNil(storageService)) allStorers := storageService.GetAllStorers() - expectedStorers := 23 // we still have a storer for trie epoch root hash + expectedStorers := 24 // we still have a storer for trie epoch root hash assert.Equal(t, expectedStorers, len(allStorers)) _ = storageService.CloseAll() }) @@ -458,7 +459,7 @@ func TestStorageServiceFactory_CreateForShard(t *testing.T) { assert.Nil(t, err) assert.False(t, check.IfNil(storageService)) allStorers := storageService.GetAllStorers() - expectedStorers := 23 + expectedStorers := 24 assert.Equal(t, expectedStorers, len(allStorers)) storer, _ := storageService.GetStorer(dataRetriever.UserAccountsUnit) @@ -527,7 +528,7 @@ func TestStorageServiceFactory_CreateForMeta(t *testing.T) { allStorers := storageService.GetAllStorers() missingStorers := 2 // PeerChangesUnit and ShardHdrNonceHashDataUnit numShardHdrStorage := 3 - expectedStorers := 23 - missingStorers + numShardHdrStorage + expectedStorers := 24 - missingStorers + numShardHdrStorage assert.Equal(t, expectedStorers, len(allStorers)) storer, _ := storageService.GetStorer(dataRetriever.UserAccountsUnit) @@ -550,7 +551,7 @@ func TestStorageServiceFactory_CreateForMeta(t *testing.T) { allStorers := storageService.GetAllStorers() missingStorers := 2 // PeerChangesUnit and ShardHdrNonceHashDataUnit numShardHdrStorage := 3 - expectedStorers := 23 - missingStorers + numShardHdrStorage + expectedStorers := 24 - missingStorers + numShardHdrStorage assert.Equal(t, expectedStorers, len(allStorers)) storer, _ := storageService.GetStorer(dataRetriever.UserAccountsUnit) diff --git a/testscommon/dataRetriever/poolFactory.go b/testscommon/dataRetriever/poolFactory.go index a8f4374e800..d479b1fe374 100644 --- a/testscommon/dataRetriever/poolFactory.go +++ b/testscommon/dataRetriever/poolFactory.go @@ -80,6 +80,10 @@ func CreatePoolsHolder(numShards uint32, selfShard uint32) dataRetriever.PoolsHo txBlockBody, err := storageunit.NewCache(cacherConfig) panicIfError("CreatePoolsHolder", err) + cacherConfig = storageunit.CacheConfig{Capacity: 100000, Type: storageunit.LRUCache, Shards: 1} + receipts, err := storageunit.NewCache(cacherConfig) + panicIfError("CreatePoolsHolder", err) + cacherConfig = storageunit.CacheConfig{Capacity: 100000, Type: storageunit.LRUCache, Shards: 1} peerChangeBlockBody, err := storageunit.NewCache(cacherConfig) panicIfError("CreatePoolsHolder", err) @@ -154,6 +158,7 @@ func CreatePoolsHolder(numShards uint32, selfShard uint32) dataRetriever.PoolsHo PeerAuthentications: peerAuthPool, Heartbeats: heartbeatPool, ValidatorsInfo: validatorsInfo, + Receipts: receipts, } holder, err := dataPool.NewDataPool(dataPoolArgs) panicIfError("CreatePoolsHolder", err) diff --git a/testscommon/dataRetriever/poolsHolderMock.go b/testscommon/dataRetriever/poolsHolderMock.go index d3d30562954..06ac3231c62 100644 --- a/testscommon/dataRetriever/poolsHolderMock.go +++ b/testscommon/dataRetriever/poolsHolderMock.go @@ -33,6 +33,7 @@ type PoolsHolderMock struct { peerAuthentications storage.Cacher heartbeats storage.Cacher validatorsInfo dataRetriever.ShardedDataCacherNotifier + receipts storage.Cacher } // NewPoolsHolderMock - @@ -79,6 +80,9 @@ func NewPoolsHolderMock() *PoolsHolderMock { holder.miniBlocks, err = storageunit.NewCache(storageunit.CacheConfig{Type: storageunit.LRUCache, Capacity: 10000, Shards: 1, SizeInBytes: 0}) panicIfError("NewPoolsHolderMock", err) + holder.receipts, err = storageunit.NewCache(storageunit.CacheConfig{Type: storageunit.LRUCache, Capacity: 10000, Shards: 1, SizeInBytes: 0}) + panicIfError("NewPoolsHolderMock", err) + holder.peerChangesBlocks, err = storageunit.NewCache(storageunit.CacheConfig{Type: storageunit.LRUCache, Capacity: 10000, Shards: 1, SizeInBytes: 0}) panicIfError("NewPoolsHolderMock", err) @@ -113,6 +117,11 @@ func NewPoolsHolderMock() *PoolsHolderMock { return holder } +// Receipts - +func (holder *PoolsHolderMock) Receipts() storage.Cacher { + return holder.receipts +} + // CurrentBlockTxs - func (holder *PoolsHolderMock) CurrentBlockTxs() dataRetriever.TransactionCacher { return holder.currBlockTxs diff --git a/testscommon/dataRetriever/poolsHolderStub.go b/testscommon/dataRetriever/poolsHolderStub.go index 106c8b96bb5..4431f569bf6 100644 --- a/testscommon/dataRetriever/poolsHolderStub.go +++ b/testscommon/dataRetriever/poolsHolderStub.go @@ -23,9 +23,19 @@ type PoolsHolderStub struct { PeerAuthenticationsCalled func() storage.Cacher HeartbeatsCalled func() storage.Cacher ValidatorsInfoCalled func() dataRetriever.ShardedDataCacherNotifier + ReceiptsCalled func() storage.Cacher CloseCalled func() error } +// Receipts - +func (holder *PoolsHolderStub) Receipts() storage.Cacher { + if holder.ReceiptsCalled != nil { + return holder.ReceiptsCalled() + } + + return nil +} + // NewPoolsHolderStub - func NewPoolsHolderStub() *PoolsHolderStub { return &PoolsHolderStub{} diff --git a/testscommon/generalConfig.go b/testscommon/generalConfig.go index 53299443ebe..9414ab1a6c4 100644 --- a/testscommon/generalConfig.go +++ b/testscommon/generalConfig.go @@ -123,6 +123,7 @@ func GetGeneralConfig() config.Config { SizeInBytesPerSender: 10000000, Shards: 1, }, + ReceiptsDataPool: getLRUCacheConfig(), UnsignedTransactionDataPool: config.CacheConfig{ Capacity: 10000, SizeInBytes: 1000000000, @@ -218,6 +219,16 @@ func GetGeneralConfig() config.Config { MaxOpenFiles: 10, }, }, + ReceiptsDataStorage: config.StorageConfig{ + Cache: getLRUCacheConfig(), + DB: config.DBConfig{ + FilePath: AddTimestampSuffix("ReceiptsData"), + Type: string(storageunit.MemoryDB), + BatchDelaySeconds: 30, + MaxBatchSize: 6, + MaxOpenFiles: 10, + }, + }, RewardTxStorage: config.StorageConfig{ Cache: getLRUCacheConfig(), DB: config.DBConfig{