From 434481f858a235ad1e3d8880921e0c5b70e3fcb2 Mon Sep 17 00:00:00 2001 From: Mehdi AOUADI Date: Wed, 21 Feb 2024 11:12:49 +0100 Subject: [PATCH] add blob sidecars store test (#7986) * add blob sidecars store test --- .../teku/storage/store/AbstractStoreTest.java | 51 ++++- .../pegasys/teku/storage/store/StoreTest.java | 177 ++++++++++++++++++ .../teku/storage/client/ChainUpdater.java | 14 +- 3 files changed, 233 insertions(+), 9 deletions(-) diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java index acb240be129..a84eaa28587 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java @@ -53,7 +53,7 @@ public abstract class AbstractStoreTest { protected final ForkChoiceStrategy dummyForkChoiceStrategy = mock(ForkChoiceStrategy.class); protected void processChainWithLimitedCache( - BiConsumer chainProcessor) { + final BiConsumer chainProcessor) { final int cacheSize = 10; final int cacheMultiplier = 3; @@ -84,7 +84,7 @@ protected void processChainWithLimitedCache( } protected void processCheckpointsWithLimitedCache( - BiConsumer chainProcessor) { + final BiConsumer chainProcessor) { final int cacheSize = 3; final int epochsToProcess = cacheSize * 3; @@ -122,7 +122,7 @@ protected void processCheckpointsWithLimitedCache( } protected void processChainHeadWithMockForkChoiceStrategy( - BiConsumer chainProcessor) { + final BiConsumer chainProcessor) { final StoreConfig pruningOptions = StoreConfig.builder().build(); final UpdatableStore store = createGenesisStoreWithMockForkChoiceStrategy(pruningOptions); @@ -150,15 +150,54 @@ protected UpdatableStore createGenesisStore() { return createGenesisStore(defaultStoreConfig); } + protected UpdatableStore createGenesisStore( + final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) { + return createStoreBuilder(defaultStoreConfig, chainBuilder, earliestBlobSidecarSlotProvider) + .build(); + } + + protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) { + return createStoreBuilder(pruningOptions).build(); + } + + protected StoreBuilder createStoreBuilder( + final StoreConfig pruningOptions, + final ChainBuilder chainBuilder, + final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) { + final SignedBlockAndState genesis = chainBuilder.generateGenesis(); + final Checkpoint genesisCheckpoint = chainBuilder.getCurrentCheckpointForEpoch(0); + return createStoreBuilder( + pruningOptions, genesis, genesisCheckpoint, earliestBlobSidecarSlotProvider); + } + protected StoreBuilder createStoreBuilder(final StoreConfig pruningOptions) { final SignedBlockAndState genesis = chainBuilder.generateGenesis(); final Checkpoint genesisCheckpoint = chainBuilder.getCurrentCheckpointForEpoch(0); + return createStoreBuilder(pruningOptions, genesis, genesisCheckpoint); + } + + private StoreBuilder createStoreBuilder( + final StoreConfig pruningOptions, + final SignedBlockAndState genesis, + final Checkpoint genesisCheckpoint) { + return createStoreBuilder( + pruningOptions, + genesis, + genesisCheckpoint, + earliestBlobSidecarSlotProviderFromChainBuilder()); + } + + private StoreBuilder createStoreBuilder( + final StoreConfig pruningOptions, + final SignedBlockAndState genesis, + final Checkpoint genesisCheckpoint, + final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) { return StoreBuilder.create() .asyncRunner(SYNC_RUNNER) .metricsSystem(new StubMetricsSystem()) .specProvider(spec) .blockProvider(blockProviderFromChainBuilder()) - .earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProviderFromChainBuilder()) + .earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProvider) .stateProvider(StateAndBlockSummaryProvider.NOOP) .anchor(Optional.empty()) .genesisTime(genesis.getState().getGenesisTime()) @@ -181,10 +220,6 @@ protected StoreBuilder createStoreBuilder(final StoreConfig pruningOptions) { .votes(emptyMap()); } - protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) { - return createStoreBuilder(pruningOptions).build(); - } - protected UpdatableStore createGenesisStoreWithMockForkChoiceStrategy( final StoreConfig pruningOptions) { return createStoreBuilder(pruningOptions).forkChoiceStrategy(dummyForkChoiceStrategy).build(); diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java index a2a98d91803..c3371f0c7d4 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java @@ -33,6 +33,7 @@ import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem; import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock; import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; @@ -46,9 +47,12 @@ import tech.pegasys.teku.spec.datastructures.state.Checkpoint; import tech.pegasys.teku.spec.datastructures.state.CheckpointState; import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState; +import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment; import tech.pegasys.teku.spec.generator.ChainBuilder; import tech.pegasys.teku.storage.api.StubStorageUpdateChannel; import tech.pegasys.teku.storage.api.StubStorageUpdateChannelWithDelays; +import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder; +import tech.pegasys.teku.storage.storageSystem.StorageSystem; import tech.pegasys.teku.storage.store.UpdatableStore.StoreTransaction; class StoreTest extends AbstractStoreTest { @@ -224,6 +228,179 @@ public void isFfgCompetitive_missingProtoNodeEntries() { }); } + @Test + public void retrieveSignedBlock_withBlobs() { + final UpdatableStore store = createGenesisStore(); + final UInt64 slot = UInt64.ONE; + final SignedBlockAndState signedBlockAndState = + chainBuilder.generateBlockAtSlot( + slot, + ChainBuilder.BlockOptions.create() + .setGenerateRandomBlobs(true) + .setStoreBlobSidecars(true)); + + addBlock(store, signedBlockAndState); + + final StoreTransaction tx = store.startTransaction(new StubStorageUpdateChannel()); + chainBuilder + .streamBlocksAndStates() + .forEach( + blockAndState -> + tx.putBlockAndState( + blockAndState, + chainBuilder.getBlobSidecars(blockAndState.getBlock().getRoot()), + spec.calculateBlockCheckpoints(blockAndState.getState()))); + tx.commit().join(); + + final SlotAndBlockRoot slotAndBlockRoot = + new SlotAndBlockRoot(slot, signedBlockAndState.getBlock().getRoot()); + + final Optional> blobSidecarsFromStore = + store.getBlobSidecarsIfAvailable(slotAndBlockRoot); + + assertThat(blobSidecarsFromStore) + .hasValueSatisfying(blobSidecars -> assertThat(blobSidecars).isNotEmpty()); + + final SafeFuture> signedBlock = + store.retrieveSignedBlock(signedBlockAndState.getRoot()); + + assertThat(signedBlock) + .isCompletedWithValueMatching(Optional::isPresent, "Result must be present") + .isCompletedWithValueMatching( + signedBeaconBlock -> + signedBeaconBlock + .orElseThrow() + .getSignedBeaconBlock() + .orElseThrow() + .equals(signedBlockAndState.getBlock()), + " Block must match") + .isCompletedWithValueMatching( + signedBeaconBlock -> + signedBeaconBlock + .orElseThrow() + .getBeaconBlock() + .orElseThrow() + .getBody() + .getOptionalBlobKzgCommitments() + .orElseThrow() + .asList() + .containsAll( + blobSidecarsFromStore.get().stream() + .map(blob -> new SszKZGCommitment(blob.getKZGCommitment())) + .toList()), + "Blob sidecars must match"); + } + + @Test + public void retrieveSignedBlock_withEmptyBlobs() { + final UpdatableStore store = createGenesisStore(); + final UInt64 slot = UInt64.ONE; + final SignedBlockAndState signedBlockAndState = + chainBuilder.generateBlockAtSlot( + slot, + ChainBuilder.BlockOptions.create() + .setGenerateRandomBlobs(false) + .setStoreBlobSidecars(false)); + + addBlock(store, signedBlockAndState); + + final StoreTransaction tx = store.startTransaction(new StubStorageUpdateChannel()); + chainBuilder + .streamBlocksAndStates() + .forEach( + blockAndState -> + tx.putBlockAndState( + blockAndState, + chainBuilder.getBlobSidecars(blockAndState.getBlock().getRoot()), + spec.calculateBlockCheckpoints(blockAndState.getState()))); + tx.commit().join(); + + final SlotAndBlockRoot slotAndBlockRoot = + new SlotAndBlockRoot(slot, signedBlockAndState.getBlock().getRoot()); + final Optional> blobSidecarsFromStore = + store.getBlobSidecarsIfAvailable(slotAndBlockRoot); + + assertThat(blobSidecarsFromStore) + .hasValueSatisfying(blobSidecars -> assertThat(blobSidecars).isEmpty()); + + final SafeFuture> signedBlock = + store.retrieveSignedBlock(signedBlockAndState.getRoot()); + + assertThat(signedBlock) + .isCompletedWithValueMatching(Optional::isPresent, "Result must be present") + .isCompletedWithValueMatching( + signedBeaconBlock -> + signedBeaconBlock + .orElseThrow() + .getSignedBeaconBlock() + .orElseThrow() + .equals(signedBlockAndState.getBlock()), + " Block must match") + .isCompletedWithValueMatching( + signedBeaconBlockAndBlobsSidecar -> + signedBeaconBlockAndBlobsSidecar + .orElseThrow() + .getBeaconBlock() + .orElseThrow() + .getBody() + .getOptionalBlobKzgCommitments() + .orElseThrow() + .isEmpty(), + "Blob sidecars must be empty"); + } + + @Test + public void retrieveSignedBlock_shouldReturnEmptyIfBlockNotPresent() { + final UpdatableStore store = createGenesisStore(); + final SignedBlockAndState blockAndState = + chainBuilder.generateBlockAtSlot( + 1, ChainBuilder.BlockOptions.create().setGenerateRandomBlobs(true)); + + final SafeFuture> signedBeaconBlock = + store.retrieveSignedBlock(blockAndState.getRoot()); + + assertThat(signedBeaconBlock) + .isCompletedWithValueMatching(Optional::isEmpty, "Result must be empty"); + } + + @Test + public void retrieveEarliestBlobSidecarSlot_shouldReturnUpdatedValue() { + final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(spec); + storageSystem.chainUpdater().initializeGenesis(); + final UpdatableStore store = + createGenesisStore( + () -> + SafeFuture.completedFuture(storageSystem.database().getEarliestBlobSidecarSlot())); + + assertThat(store.retrieveEarliestBlobSidecarSlot()) + .isCompletedWithValueMatching( + maybeEarliestBlobSidecarSlot -> + maybeEarliestBlobSidecarSlot.isPresent() + && maybeEarliestBlobSidecarSlot.get().equals(UInt64.ZERO)); + + storageSystem + .chainUpdater() + .advanceChainUntil( + 10, + ChainBuilder.BlockOptions.create() + .setGenerateRandomBlobs(true) + .setStoreBlobSidecars(true)); + + assertThat(store.retrieveEarliestBlobSidecarSlot()) + .isCompletedWithValueMatching( + maybeEarliestBlobSidecarSlot -> + maybeEarliestBlobSidecarSlot.isPresent() + && maybeEarliestBlobSidecarSlot.get().equals(UInt64.ZERO)); + + storageSystem.database().pruneOldestBlobSidecars(UInt64.valueOf(5), 5); + + assertThat(store.retrieveEarliestBlobSidecarSlot()) + .isCompletedWithValueMatching( + maybeEarliestBlobSidecarSlot -> + maybeEarliestBlobSidecarSlot.isPresent() + && maybeEarliestBlobSidecarSlot.get().equals(UInt64.valueOf(4))); + } + private void setProtoNodeDataForBlock( SignedBlockAndState blockAndState, BlockCheckpoints headCheckpoint, diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java index 7ed8b1c92fe..c870c3ae96c 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java @@ -225,16 +225,28 @@ public SignedBlockAndState advanceChainUntil(final long slot) { } public SignedBlockAndState advanceChainUntil(final UInt64 slot) { + return advanceChainUntil(slot, this.blockOptions); + } + + public SignedBlockAndState advanceChainUntil(final long slot, final BlockOptions blockOptions) { + return advanceChainUntil(UInt64.valueOf(slot), blockOptions); + } + + public SignedBlockAndState advanceChainUntil(final UInt64 slot, final BlockOptions blockOptions) { UInt64 currentSlot = chainBuilder.getLatestSlot(); SignedBlockAndState latestSigneBlockAndState = chainBuilder.getLatestBlockAndState(); while (currentSlot.isLessThan(slot)) { currentSlot = currentSlot.increment(); - latestSigneBlockAndState = advanceChain(currentSlot); + latestSigneBlockAndState = advanceChain(currentSlot, blockOptions); } return latestSigneBlockAndState; } public SignedBlockAndState advanceChain(final UInt64 slot) { + return advanceChain(slot, this.blockOptions); + } + + public SignedBlockAndState advanceChain(final UInt64 slot, final BlockOptions blockOptions) { final SignedBlockAndState block = chainBuilder.generateBlockAtSlot(slot, blockOptions); final List blobSidecars = chainBuilder.getBlobSidecars(block.getRoot()); if (blobSidecars.isEmpty()) {