Skip to content

Commit

Permalink
add blob sidecars store test (Consensys#7986)
Browse files Browse the repository at this point in the history 
* add blob sidecars store test
  • Loading branch information
@mehdi-aouadi
mehdi-aouadi authored Feb 21, 2024
1 parent 4d6265c commit 434481f
Show file tree
Hide file tree
Showing 3 changed files with 233 additions and 9 deletions.
51 changes: 43 additions & 8 deletions storage/src/test/java/tech/pegasys/teku/storage/store/AbstractStoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public abstract class AbstractStoreTest {
protected final ForkChoiceStrategy dummyForkChoiceStrategy = mock(ForkChoiceStrategy.class);

protected void processChainWithLimitedCache(
BiConsumer<UpdatableStore, SignedBlockAndState> chainProcessor) {
final BiConsumer<UpdatableStore, SignedBlockAndState> chainProcessor) {
final int cacheSize = 10;
final int cacheMultiplier = 3;

Expand Down Expand Up @@ -84,7 +84,7 @@ protected void processChainWithLimitedCache(
}

protected void processCheckpointsWithLimitedCache(
BiConsumer<UpdatableStore, CheckpointState> chainProcessor) {
final BiConsumer<UpdatableStore, CheckpointState> chainProcessor) {
final int cacheSize = 3;
final int epochsToProcess = cacheSize * 3;

Expand Down Expand Up @@ -122,7 +122,7 @@ protected void processCheckpointsWithLimitedCache(
}

protected void processChainHeadWithMockForkChoiceStrategy(
BiConsumer<UpdatableStore, SignedBlockAndState> chainProcessor) {
final BiConsumer<UpdatableStore, SignedBlockAndState> chainProcessor) {
final StoreConfig pruningOptions = StoreConfig.builder().build();

final UpdatableStore store = createGenesisStoreWithMockForkChoiceStrategy(pruningOptions);
Expand Down Expand Up @@ -150,15 +150,54 @@ protected UpdatableStore createGenesisStore() {
return createGenesisStore(defaultStoreConfig);
}

protected UpdatableStore createGenesisStore(
final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) {
return createStoreBuilder(defaultStoreConfig, chainBuilder, earliestBlobSidecarSlotProvider)
.build();
}

protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) {
return createStoreBuilder(pruningOptions).build();
}

protected StoreBuilder createStoreBuilder(
final StoreConfig pruningOptions,
final ChainBuilder chainBuilder,
final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) {
final SignedBlockAndState genesis = chainBuilder.generateGenesis();
final Checkpoint genesisCheckpoint = chainBuilder.getCurrentCheckpointForEpoch(0);
return createStoreBuilder(
pruningOptions, genesis, genesisCheckpoint, earliestBlobSidecarSlotProvider);
}

protected StoreBuilder createStoreBuilder(final StoreConfig pruningOptions) {
final SignedBlockAndState genesis = chainBuilder.generateGenesis();
final Checkpoint genesisCheckpoint = chainBuilder.getCurrentCheckpointForEpoch(0);
return createStoreBuilder(pruningOptions, genesis, genesisCheckpoint);
}

private StoreBuilder createStoreBuilder(
final StoreConfig pruningOptions,
final SignedBlockAndState genesis,
final Checkpoint genesisCheckpoint) {
return createStoreBuilder(
pruningOptions,
genesis,
genesisCheckpoint,
earliestBlobSidecarSlotProviderFromChainBuilder());
}

private StoreBuilder createStoreBuilder(
final StoreConfig pruningOptions,
final SignedBlockAndState genesis,
final Checkpoint genesisCheckpoint,
final EarliestBlobSidecarSlotProvider earliestBlobSidecarSlotProvider) {
return StoreBuilder.create()
.asyncRunner(SYNC_RUNNER)
.metricsSystem(new StubMetricsSystem())
.specProvider(spec)
.blockProvider(blockProviderFromChainBuilder())
.earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProviderFromChainBuilder())
.earliestBlobSidecarSlotProvider(earliestBlobSidecarSlotProvider)
.stateProvider(StateAndBlockSummaryProvider.NOOP)
.anchor(Optional.empty())
.genesisTime(genesis.getState().getGenesisTime())
Expand All @@ -181,10 +220,6 @@ protected StoreBuilder createStoreBuilder(final StoreConfig pruningOptions) {
.votes(emptyMap());
}

protected UpdatableStore createGenesisStore(final StoreConfig pruningOptions) {
return createStoreBuilder(pruningOptions).build();
}

protected UpdatableStore createGenesisStoreWithMockForkChoiceStrategy(
final StoreConfig pruningOptions) {
return createStoreBuilder(pruningOptions).forkChoiceStrategy(dummyForkChoiceStrategy).build();
Expand Down
177 changes: 177 additions & 0 deletions storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import tech.pegasys.teku.infrastructure.async.SafeFuture;
import tech.pegasys.teku.infrastructure.metrics.StubMetricsSystem;
import tech.pegasys.teku.infrastructure.unsigned.UInt64;
import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
import tech.pegasys.teku.spec.datastructures.blocks.BlockCheckpoints;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
Expand All @@ -46,9 +47,12 @@
import tech.pegasys.teku.spec.datastructures.state.Checkpoint;
import tech.pegasys.teku.spec.datastructures.state.CheckpointState;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
import tech.pegasys.teku.spec.generator.ChainBuilder;
import tech.pegasys.teku.storage.api.StubStorageUpdateChannel;
import tech.pegasys.teku.storage.api.StubStorageUpdateChannelWithDelays;
import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder;
import tech.pegasys.teku.storage.storageSystem.StorageSystem;
import tech.pegasys.teku.storage.store.UpdatableStore.StoreTransaction;

class StoreTest extends AbstractStoreTest {
Expand Down Expand Up @@ -224,6 +228,179 @@ public void isFfgCompetitive_missingProtoNodeEntries() {
});
}

@Test
public void retrieveSignedBlock_withBlobs() {
final UpdatableStore store = createGenesisStore();
final UInt64 slot = UInt64.ONE;
final SignedBlockAndState signedBlockAndState =
chainBuilder.generateBlockAtSlot(
slot,
ChainBuilder.BlockOptions.create()
.setGenerateRandomBlobs(true)
.setStoreBlobSidecars(true));

addBlock(store, signedBlockAndState);

final StoreTransaction tx = store.startTransaction(new StubStorageUpdateChannel());
chainBuilder
.streamBlocksAndStates()
.forEach(
blockAndState ->
tx.putBlockAndState(
blockAndState,
chainBuilder.getBlobSidecars(blockAndState.getBlock().getRoot()),
spec.calculateBlockCheckpoints(blockAndState.getState())));
tx.commit().join();

final SlotAndBlockRoot slotAndBlockRoot =
new SlotAndBlockRoot(slot, signedBlockAndState.getBlock().getRoot());

final Optional<List<BlobSidecar>> blobSidecarsFromStore =
store.getBlobSidecarsIfAvailable(slotAndBlockRoot);

assertThat(blobSidecarsFromStore)
.hasValueSatisfying(blobSidecars -> assertThat(blobSidecars).isNotEmpty());

final SafeFuture<Optional<SignedBeaconBlock>> signedBlock =
store.retrieveSignedBlock(signedBlockAndState.getRoot());

assertThat(signedBlock)
.isCompletedWithValueMatching(Optional::isPresent, "Result must be present")
.isCompletedWithValueMatching(
signedBeaconBlock ->
signedBeaconBlock
.orElseThrow()
.getSignedBeaconBlock()
.orElseThrow()
.equals(signedBlockAndState.getBlock()),
" Block must match")
.isCompletedWithValueMatching(
signedBeaconBlock ->
signedBeaconBlock
.orElseThrow()
.getBeaconBlock()
.orElseThrow()
.getBody()
.getOptionalBlobKzgCommitments()
.orElseThrow()
.asList()
.containsAll(
blobSidecarsFromStore.get().stream()
.map(blob -> new SszKZGCommitment(blob.getKZGCommitment()))
.toList()),
"Blob sidecars must match");
}

@Test
public void retrieveSignedBlock_withEmptyBlobs() {
final UpdatableStore store = createGenesisStore();
final UInt64 slot = UInt64.ONE;
final SignedBlockAndState signedBlockAndState =
chainBuilder.generateBlockAtSlot(
slot,
ChainBuilder.BlockOptions.create()
.setGenerateRandomBlobs(false)
.setStoreBlobSidecars(false));

addBlock(store, signedBlockAndState);

final StoreTransaction tx = store.startTransaction(new StubStorageUpdateChannel());
chainBuilder
.streamBlocksAndStates()
.forEach(
blockAndState ->
tx.putBlockAndState(
blockAndState,
chainBuilder.getBlobSidecars(blockAndState.getBlock().getRoot()),
spec.calculateBlockCheckpoints(blockAndState.getState())));
tx.commit().join();

final SlotAndBlockRoot slotAndBlockRoot =
new SlotAndBlockRoot(slot, signedBlockAndState.getBlock().getRoot());
final Optional<List<BlobSidecar>> blobSidecarsFromStore =
store.getBlobSidecarsIfAvailable(slotAndBlockRoot);

assertThat(blobSidecarsFromStore)
.hasValueSatisfying(blobSidecars -> assertThat(blobSidecars).isEmpty());

final SafeFuture<Optional<SignedBeaconBlock>> signedBlock =
store.retrieveSignedBlock(signedBlockAndState.getRoot());

assertThat(signedBlock)
.isCompletedWithValueMatching(Optional::isPresent, "Result must be present")
.isCompletedWithValueMatching(
signedBeaconBlock ->
signedBeaconBlock
.orElseThrow()
.getSignedBeaconBlock()
.orElseThrow()
.equals(signedBlockAndState.getBlock()),
" Block must match")
.isCompletedWithValueMatching(
signedBeaconBlockAndBlobsSidecar ->
signedBeaconBlockAndBlobsSidecar
.orElseThrow()
.getBeaconBlock()
.orElseThrow()
.getBody()
.getOptionalBlobKzgCommitments()
.orElseThrow()
.isEmpty(),
"Blob sidecars must be empty");
}

@Test
public void retrieveSignedBlock_shouldReturnEmptyIfBlockNotPresent() {
final UpdatableStore store = createGenesisStore();
final SignedBlockAndState blockAndState =
chainBuilder.generateBlockAtSlot(
1, ChainBuilder.BlockOptions.create().setGenerateRandomBlobs(true));

final SafeFuture<Optional<SignedBeaconBlock>> signedBeaconBlock =
store.retrieveSignedBlock(blockAndState.getRoot());

assertThat(signedBeaconBlock)
.isCompletedWithValueMatching(Optional::isEmpty, "Result must be empty");
}

@Test
public void retrieveEarliestBlobSidecarSlot_shouldReturnUpdatedValue() {
final StorageSystem storageSystem = InMemoryStorageSystemBuilder.buildDefault(spec);
storageSystem.chainUpdater().initializeGenesis();
final UpdatableStore store =
createGenesisStore(
() ->
SafeFuture.completedFuture(storageSystem.database().getEarliestBlobSidecarSlot()));

assertThat(store.retrieveEarliestBlobSidecarSlot())
.isCompletedWithValueMatching(
maybeEarliestBlobSidecarSlot ->
maybeEarliestBlobSidecarSlot.isPresent()
&& maybeEarliestBlobSidecarSlot.get().equals(UInt64.ZERO));

storageSystem
.chainUpdater()
.advanceChainUntil(
10,
ChainBuilder.BlockOptions.create()
.setGenerateRandomBlobs(true)
.setStoreBlobSidecars(true));

assertThat(store.retrieveEarliestBlobSidecarSlot())
.isCompletedWithValueMatching(
maybeEarliestBlobSidecarSlot ->
maybeEarliestBlobSidecarSlot.isPresent()
&& maybeEarliestBlobSidecarSlot.get().equals(UInt64.ZERO));

storageSystem.database().pruneOldestBlobSidecars(UInt64.valueOf(5), 5);

assertThat(store.retrieveEarliestBlobSidecarSlot())
.isCompletedWithValueMatching(
maybeEarliestBlobSidecarSlot ->
maybeEarliestBlobSidecarSlot.isPresent()
&& maybeEarliestBlobSidecarSlot.get().equals(UInt64.valueOf(4)));
}

private void setProtoNodeDataForBlock(
SignedBlockAndState blockAndState,
BlockCheckpoints headCheckpoint,
Expand Down
14 changes: 13 additions & 1 deletion storage/src/testFixtures/java/tech/pegasys/teku/storage/client/ChainUpdater.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,16 +225,28 @@ public SignedBlockAndState advanceChainUntil(final long slot) {
}

public SignedBlockAndState advanceChainUntil(final UInt64 slot) {
return advanceChainUntil(slot, this.blockOptions);
}

public SignedBlockAndState advanceChainUntil(final long slot, final BlockOptions blockOptions) {
return advanceChainUntil(UInt64.valueOf(slot), blockOptions);
}

public SignedBlockAndState advanceChainUntil(final UInt64 slot, final BlockOptions blockOptions) {
UInt64 currentSlot = chainBuilder.getLatestSlot();
SignedBlockAndState latestSigneBlockAndState = chainBuilder.getLatestBlockAndState();
while (currentSlot.isLessThan(slot)) {
currentSlot = currentSlot.increment();
latestSigneBlockAndState = advanceChain(currentSlot);
latestSigneBlockAndState = advanceChain(currentSlot, blockOptions);
}
return latestSigneBlockAndState;
}

public SignedBlockAndState advanceChain(final UInt64 slot) {
return advanceChain(slot, this.blockOptions);
}

public SignedBlockAndState advanceChain(final UInt64 slot, final BlockOptions blockOptions) {
final SignedBlockAndState block = chainBuilder.generateBlockAtSlot(slot, blockOptions);
final List<BlobSidecar> blobSidecars = chainBuilder.getBlobSidecars(block.getRoot());
if (blobSidecars.isEmpty()) {
Expand Down

0 comments on commit 434481f

Please sign in to comment.