Merge remote-tracking branch 'upstream/master' into no-keys

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/Constants.java

@@ -31,6 +31,9 @@ public class Constants {
   // sync committee size.
   public static final int VALID_CONTRIBUTION_AND_PROOF_SET_SIZE = 512;
   public static final int VALID_SYNC_COMMITTEE_MESSAGE_SET_SIZE = 512;
+  // When finalization is at its best case with 100% of votes we could have up to 3 full
+  // epochs of non-finalized blocks
+  public static final int BEST_CASE_NON_FINALIZED_EPOCHS = 3;
 
   public static final Duration ETH1_INDIVIDUAL_BLOCK_RETRY_TIMEOUT = Duration.ofMillis(500);
   public static final Duration ETH1_DEPOSIT_REQUEST_RETRY_TIMEOUT = Duration.ofSeconds(2);

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java

@@ -134,4 +134,13 @@ default SafeFuture<Optional<BeaconBlock>> retrieveBlock(Bytes32 blockRoot) {
 
   SafeFuture<Optional<BeaconState>> retrieveCheckpointState(
       Checkpoint checkpoint, BeaconState latestStateAtEpoch);
+
+  // implements is_head_weak from fork-choice Consensus Spec
+  SafeFuture<Optional<Boolean>> isHeadWeak(final Bytes32 root);
+
+  // implements is_parent_strong from fork-choice Consensus Spec
+  SafeFuture<Optional<Boolean>> isParentStrong(final Bytes32 parentRoot);
+
+  // implements is_ffg_competitive from Consensus Spec
+  Optional<Boolean> isFfgCompetitive(final Bytes32 headRoot, final Bytes32 parentRoot);
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/logic/common/helpers/BeaconStateAccessors.java

@@ -170,7 +170,7 @@ public Bytes32 getSeed(BeaconState state, UInt64 epoch, Bytes4 domainType)
    * @return
    */
   public UInt64 calculateCommitteeFraction(
-      final BeaconState beaconState, final UInt64 committeePercent) {
+      final BeaconState beaconState, final int committeePercent) {
     final UInt64 committeeWeight =
         getTotalActiveBalance(beaconState).dividedBy(config.getSlotsPerEpoch());
     return committeeWeight.times(committeePercent).dividedBy(100);

ethereum/spec/src/test/java/tech/pegasys/teku/spec/logic/common/helpers/BeaconStateAccessorsTest.java

@@ -137,8 +137,7 @@ void calculateCommitteeFraction_full() {
         spec.atSlot(state.getSlot()).beaconStateAccessors().getTotalActiveBalance(state);
     final UInt64 totalActiveBalancePerSlot =
         totalActiveBalance.dividedBy(spec.getGenesisSpec().getSlotsPerEpoch());
-    final UInt64 fraction =
-        beaconStateAccessors.calculateCommitteeFraction(state, UInt64.valueOf(100));
+    final UInt64 fraction = beaconStateAccessors.calculateCommitteeFraction(state, 100);
     // at its simplest, if we've divided by slots in the function, this should be
     // totalActiveBalance/slots (because fraction is 100%)
     assertThat(fraction).isEqualTo(totalActiveBalancePerSlot);
@@ -151,7 +150,7 @@ void calculateCommitteeFraction_minimal() {
         spec.atSlot(state.getSlot()).beaconStateAccessors().getTotalActiveBalance(state);
     final UInt64 totalActiveBalancePerSlot =
         totalActiveBalance.dividedBy(spec.getGenesisSpec().getSlotsPerEpoch());
-    final UInt64 fraction = beaconStateAccessors.calculateCommitteeFraction(state, UInt64.ONE);
+    final UInt64 fraction = beaconStateAccessors.calculateCommitteeFraction(state, 1);
     // should be 1% of balance per slot...
     assertThat(fraction).isEqualTo(totalActiveBalancePerSlot.dividedBy(100));
   }

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java

@@ -246,6 +246,21 @@ public SafeFuture<Optional<BeaconState>> retrieveCheckpointState(
     return SafeFuture.completedFuture(Optional.of(latestStateAtEpoch));
   }
 
+  @Override
+  public SafeFuture<Optional<Boolean>> isHeadWeak(Bytes32 root) {
+    return SafeFuture.completedFuture(Optional.empty());
+  }
+
+  @Override
+  public SafeFuture<Optional<Boolean>> isParentStrong(Bytes32 parentRoot) {
+    return SafeFuture.completedFuture(Optional.empty());
+  }
+
+  @Override
+  public Optional<Boolean> isFfgCompetitive(Bytes32 headRoot, Bytes32 parentRoot) {
+    return Optional.empty();
+  }
+
   @Override
   public Optional<List<BlobSidecar>> getBlobSidecarsIfAvailable(
       final SlotAndBlockRoot slotAndBlockRoot) {

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/validation/BlobSidecarGossipValidator.java

@@ -14,7 +14,9 @@
 package tech.pegasys.teku.statetransition.validation;
 
 import static tech.pegasys.teku.infrastructure.async.SafeFuture.completedFuture;
+import static tech.pegasys.teku.spec.config.Constants.BEST_CASE_NON_FINALIZED_EPOCHS;
 import static tech.pegasys.teku.spec.config.Constants.VALID_BLOCK_SET_SIZE;
+import static tech.pegasys.teku.statetransition.validation.InternalValidationResult.ACCEPT;
 import static tech.pegasys.teku.statetransition.validation.InternalValidationResult.ignore;
 import static tech.pegasys.teku.statetransition.validation.InternalValidationResult.reject;
 
@@ -48,6 +50,7 @@ public class BlobSidecarGossipValidator {
 
   private final Spec spec;
   private final Set<SlotProposerIndexAndBlobIndex> receivedValidBlobSidecarInfoSet;
+  private final Set<Bytes32> validSignedBlockHeaders;
   private final GossipValidationHelper gossipValidationHelper;
   private final Map<Bytes32, BlockImportResult> invalidBlockRoots;
   private final MiscHelpersDeneb miscHelpersDeneb;
@@ -63,14 +66,18 @@ public static BlobSidecarGossipValidator create(
     final Optional<Integer> maybeMaxBlobsPerBlock = spec.getMaxBlobsPerBlock();
 
     final int validInfoSize = VALID_BLOCK_SET_SIZE * maybeMaxBlobsPerBlock.orElse(1);
+    // It's not fatal if we miss something and we don't need finalized data
+    final int validSignedBlockHeadersSize =
+        spec.getGenesisSpec().getSlotsPerEpoch() * BEST_CASE_NON_FINALIZED_EPOCHS;
 
     return new BlobSidecarGossipValidator(
         spec,
         invalidBlockRoots,
         validationHelper,
         miscHelpersDeneb,
         kzg,
-        LimitedSet.createSynchronized(validInfoSize));
+        LimitedSet.createSynchronized(validInfoSize),
+        LimitedSet.createSynchronized(validSignedBlockHeadersSize));
   }
 
   @VisibleForTesting
@@ -84,13 +91,15 @@ private BlobSidecarGossipValidator(
       final GossipValidationHelper gossipValidationHelper,
       final MiscHelpersDeneb miscHelpersDeneb,
       final KZG kzg,
-      final Set<SlotProposerIndexAndBlobIndex> receivedValidBlobSidecarInfoSet) {
+      final Set<SlotProposerIndexAndBlobIndex> receivedValidBlobSidecarInfoSet,
+      final Set<Bytes32> validSignedBlockHeaders) {
     this.spec = spec;
     this.invalidBlockRoots = invalidBlockRoots;
     this.gossipValidationHelper = gossipValidationHelper;
     this.miscHelpersDeneb = miscHelpersDeneb;
     this.kzg = kzg;
     this.receivedValidBlobSidecarInfoSet = receivedValidBlobSidecarInfoSet;
+    this.validSignedBlockHeaders = validSignedBlockHeaders;
   }
 
   public SafeFuture<InternalValidationResult> validate(final BlobSidecar blobSidecar) {
@@ -134,6 +143,12 @@ public SafeFuture<InternalValidationResult> validate(final BlobSidecar blobSidec
       return completedFuture(InternalValidationResult.IGNORE);
     }
 
+    // Optimization: If we have already completely verified BlobSidecar with the same
+    // SignedBlockHeader, we can skip most steps and jump to shortened validation
+    if (validSignedBlockHeaders.contains(blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot())) {
+      return validateBlobSidecarWithKnownValidHeader(blobSidecar, blockHeader);
+    }
+
     /*
      * [REJECT] The proposer signature of `blob_sidecar.signed_block_header`, is valid with respect
      * to the `block_header.proposer_index` pubkey.
@@ -241,6 +256,12 @@ public SafeFuture<InternalValidationResult> validate(final BlobSidecar blobSidec
                 return reject("BlobSidecar block header signature is invalid.");
               }
 
+              /*
+               * Checking it again at the very end because whole method is not synchronized
+               *
+               * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, blob_sidecar.index)
+               *  with valid header signature, sidecar inclusion proof, and kzg proof.
+               */
               if (!receivedValidBlobSidecarInfoSet.add(
                   new SlotProposerIndexAndBlobIndex(
                       blockHeader.getSlot(),
@@ -250,10 +271,55 @@ public SafeFuture<InternalValidationResult> validate(final BlobSidecar blobSidec
                     "BlobSidecar is not the first valid for its slot and index. It will be dropped.");
               }
 
-              return InternalValidationResult.ACCEPT;
+              validSignedBlockHeaders.add(blobSidecar.getSignedBeaconBlockHeader().hashTreeRoot());
+
+              return ACCEPT;
             });
   }
 
+  private SafeFuture<InternalValidationResult> validateBlobSidecarWithKnownValidHeader(
+      final BlobSidecar blobSidecar, final BeaconBlockHeader blockHeader) {
+
+    /*
+     * [REJECT] The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof(blob_sidecar)`.
+     */
+    if (!miscHelpersDeneb.verifyBlobSidecarMerkleProof(blobSidecar)) {
+      return completedFuture(reject("BlobSidecar inclusion proof validation failed"));
+    }
+
+    /*
+     * [REJECT] The sidecar's blob is valid as verified by
+     * `verify_blob_kzg_proof(blob_sidecar.blob, blob_sidecar.kzg_commitment, blob_sidecar.kzg_proof)`.
+     */
+    if (!miscHelpersDeneb.verifyBlobKzgProof(kzg, blobSidecar)) {
+      return completedFuture(reject("BlobSidecar does not pass kzg validation"));
+    }
+
+    // This can be changed between two received BlobSidecars from one block, so checking
+    /*
+     * [REJECT] The current finalized_checkpoint is an ancestor of the sidecar's block -- i.e.
+     * `get_checkpoint_block(store, block_header.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root`.
+     */
+    if (!gossipValidationHelper.currentFinalizedCheckpointIsAncestorOfBlock(
+        blockHeader.getSlot(), blockHeader.getParentRoot())) {
+      return completedFuture(
+          reject("BlobSidecar block header does not descend from finalized checkpoint"));
+    }
+
+    /*
+     * [IGNORE] The sidecar is the first sidecar for the tuple (block_header.slot, block_header.proposer_index, blob_sidecar.index)
+     *  with valid header signature, sidecar inclusion proof, and kzg proof.
+     */
+    if (!receivedValidBlobSidecarInfoSet.add(
+        new SlotProposerIndexAndBlobIndex(
+            blockHeader.getSlot(), blockHeader.getProposerIndex(), blobSidecar.getIndex()))) {
+      return SafeFuture.completedFuture(
+          ignore("BlobSidecar is not the first valid for its slot and index. It will be dropped."));
+    }
+
+    return SafeFuture.completedFuture(ACCEPT);
+  }
+
   private boolean verifyBlockHeaderSignature(
       final BeaconState state, final SignedBeaconBlockHeader signedBlockHeader) {
     final Bytes32 domain =