Merge remote-tracking branch 'upstream/master' into no-keys

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/VoluntaryExitAcceptanceTest.java

@@ -21,6 +21,7 @@
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
 import tech.pegasys.teku.test.acceptance.dsl.GenesisGenerator.InitialStateData;
+import tech.pegasys.teku.test.acceptance.dsl.TekuDepositSender;
 import tech.pegasys.teku.test.acceptance.dsl.TekuNode;
 import tech.pegasys.teku.test.acceptance.dsl.TekuValidatorNode;
 import tech.pegasys.teku.test.acceptance.dsl.TekuVoluntaryExit;
@@ -31,25 +32,29 @@ public class VoluntaryExitAcceptanceTest extends AcceptanceTestBase {
   @Test
   void shouldChangeValidatorStatusAfterSubmittingVoluntaryExit() throws Exception {
     final String networkName = "swift";
-    final ValidatorKeystores validatorKeystores =
-        createTekuDepositSender(networkName).generateValidatorKeys(4);
-    final ValidatorKeystores extraKeys =
-        createTekuDepositSender(networkName).generateValidatorKeys(1);
+
+    final TekuDepositSender depositSender = createTekuDepositSender(networkName);
+    final ValidatorKeystores validatorKeysToExit = depositSender.generateValidatorKeys(4);
+    // network of 8 validators (4 of them will exit)
+    final ValidatorKeystores validatorKeys =
+        ValidatorKeystores.add(validatorKeysToExit, depositSender.generateValidatorKeys(4));
+    // 1 unknown key to the network
+    final ValidatorKeystores unknownKeys = depositSender.generateValidatorKeys(1);
 
     final InitialStateData genesis =
-        createGenesisGenerator().network(networkName).validatorKeys(validatorKeystores).generate();
+        createGenesisGenerator().network(networkName).validatorKeys(validatorKeys).generate();
 
     final TekuNode beaconNode =
         createTekuNode(config -> config.withNetwork(networkName).withInitialState(genesis));
 
     final TekuVoluntaryExit voluntaryExitProcessFailing =
         createVoluntaryExit(config -> config.withBeaconNode(beaconNode))
-            .withValidatorKeystores(validatorKeystores);
+            .withValidatorKeystores(validatorKeysToExit);
 
     final TekuVoluntaryExit voluntaryExitProcessSuccessful =
         createVoluntaryExit(config -> config.withBeaconNode(beaconNode))
-            .withValidatorKeystores(validatorKeystores)
-            .withValidatorKeystores(extraKeys);
+            .withValidatorKeystores(validatorKeysToExit)
+            .withValidatorKeystores(unknownKeys);
 
     final TekuValidatorNode validatorClient =
         createValidatorNode(
@@ -58,7 +63,7 @@ void shouldChangeValidatorStatusAfterSubmittingVoluntaryExit() throws Exception
                         .withNetwork(networkName)
                         .withInteropModeDisabled()
                         .withBeaconNode(beaconNode))
-            .withValidatorKeystores(validatorKeystores);
+            .withValidatorKeystores(validatorKeys);
 
     beaconNode.start();
     validatorClient.start();
@@ -83,6 +88,6 @@ void shouldChangeValidatorStatusAfterSubmittingVoluntaryExit() throws Exception
     assertThat(validatorIds.size()).isEqualTo(4);
     assertThat(validatorIds).containsExactlyInAnyOrder(0, 1, 2, 3);
     assertThat(voluntaryExitProcessSuccessful.getLoggedErrors())
-        .contains("Validator not found: " + extraKeys.getPublicKeys().get(0).toString());
+        .contains("Validator not found: " + unknownKeys.getPublicKeys().get(0).toString());
   }
 }

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuDepositSender.java

@@ -74,9 +74,7 @@ public void sendValidatorDeposits(
         new DepositSenderService(
             spec, eth1Node.getExternalJsonRpcUrl(), eth1Credentials, eth1Address, amount)) {
       final List<SafeFuture<TransactionReceipt>> transactionReceipts =
-          validatorKeys.getValidatorKeys().stream()
-              .map(depositSenderService::sendDeposit)
-              .collect(Collectors.toList());
+          validatorKeys.getValidatorKeys().stream().map(depositSenderService::sendDeposit).toList();
       final SafeFuture<Void> future =
           SafeFuture.allOf(transactionReceipts.toArray(SafeFuture[]::new));
       Waiter.waitFor(future, Duration.ofMinutes(2));

build.gradle

@@ -499,9 +499,10 @@ def dockerImage = "consensys/teku"
 def dockerJdkVariants = [ "jdk17" ]
 def dockerBuildDir = "build/docker-teku/"
 
+def executableAndArg = System.getProperty('os.name').toLowerCase().contains('windows') ? ["cmd", "/c"] : ["sh", "-c"]
+
 task distDocker  {
   dependsOn dockerDistUntar
-
   def dockerBuildVersion = 'develop'
   doLast {
     for (def variant in dockerJdkVariants) {
@@ -512,14 +513,14 @@ task distDocker  {
       exec {
         def image = "${dockerImage}:${dockerBuildVersion}-${variant}"
         workingDir dockerBuildDir
-        executable "sh"
-        args "-c", "docker build --pull --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} -t ${image} ."
+        executable executableAndArg[0]
+        args executableAndArg[1], "docker build --pull --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} -t ${image} ."
       }
     }
     // tag the "default" (which is the variant in the zero position)
     exec {
-      executable "sh"
-      args "-c", "docker tag '${dockerImage}:${dockerBuildVersion}-${dockerJdkVariants[0]}' '${dockerImage}:${dockerBuildVersion}'"
+      executable executableAndArg[0]
+      args executableAndArg[1], "docker tag ${dockerImage}:${dockerBuildVersion}-${dockerJdkVariants[0]} ${dockerImage}:${dockerBuildVersion}"
     }
   }
 }
@@ -556,17 +557,17 @@ task uploadDocker {
 
       exec {
         workingDir dockerBuildDir
-        executable "sh"
-        args "-c", "docker build --pull --platform ${platform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} ${tags} ."
+        executable executableAndArg[0]
+        args executableAndArg[1], "docker build --pull --platform ${platform} --build-arg BUILD_DATE=${buildTime()} --build-arg VERSION=${dockerBuildVersion} --build-arg VCS_REF=${getCheckedOutGitCommitHash()} ${tags} ."
       }
 
       //docker trust sign runs one image at a time, so we have to remove the '-t' in the string and split into a list we can use
       def trustTags = tags.replaceAll( '-t ', '' ).trim().split(' ')
       for (def t in trustTags) {
         exec {
           workingDir dockerBuildDir
-          executable "sh"
-          args "-c", "docker trust sign ${t} && docker push ${t} "
+          executable executableAndArg[0]
+          args executableAndArg[1], "docker trust sign ${t} && docker push ${t} "
         }
       }
     }
@@ -600,8 +601,8 @@ task manifestDocker {
       for (def tag in tags) {
         platforms.forEach { platform -> cmd += "${tag}-${platform} " }
         exec {
-          executable "sh"
-          args "-c", "docker manifest create ${tag} ${cmd} && docker manifest push ${tag}"
+          executable executableAndArg[0]
+          args executableAndArg[1], "docker manifest create ${tag} ${cmd} && docker manifest push ${tag}"
         }
       }
     }

eth-reference-tests/src/referenceTest/java/tech/pegasys/teku/reference/phase0/forkchoice/ForkChoiceTestExecutor.java

@@ -464,14 +464,10 @@ private void applyChecks(
 
           case "get_proposer_head" -> {
             final Bytes32 expectedProposerHead = getBytes32(checks, checkType);
-            final Optional<Bytes32> boostedRoot = recentChainData.getBestBlockRoot();
-            if (expectedProposerHead.isZero()) {
-              assertThat(boostedRoot).describedAs("get_proposer_head").isEmpty();
-            } else {
-              assertThat(boostedRoot)
-                  .describedAs("get_proposer_head")
-                  .contains(expectedProposerHead);
-            }
+            final Bytes32 root =
+                recentChainData.getProposerHead(
+                    expectedProposerHead, recentChainData.getHeadSlot().increment());
+            assertThat(root).describedAs("get_proposer_head").isEqualTo(expectedProposerHead);
           }
 
           case "should_override_forkchoice_update" -> {

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/forkchoice/ReadOnlyStore.java

@@ -136,10 +136,10 @@ SafeFuture<Optional<BeaconState>> retrieveCheckpointState(
       Checkpoint checkpoint, BeaconState latestStateAtEpoch);
 
   // implements is_head_weak from fork-choice Consensus Spec
-  SafeFuture<Optional<Boolean>> isHeadWeak(final Bytes32 root);
+  boolean isHeadWeak(BeaconState justifiedState, Bytes32 root);
 
   // implements is_parent_strong from fork-choice Consensus Spec
-  SafeFuture<Optional<Boolean>> isParentStrong(final Bytes32 parentRoot);
+  boolean isParentStrong(BeaconState justifiedState, Bytes32 parentRoot);
 
   // implements is_ffg_competitive from Consensus Spec
   Optional<Boolean> isFfgCompetitive(final Bytes32 headRoot, final Bytes32 parentRoot);

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/datastructures/forkchoice/TestStoreImpl.java

@@ -247,13 +247,13 @@ public SafeFuture<Optional<BeaconState>> retrieveCheckpointState(
   }
 
   @Override
-  public SafeFuture<Optional<Boolean>> isHeadWeak(Bytes32 root) {
-    return SafeFuture.completedFuture(Optional.empty());
+  public boolean isHeadWeak(BeaconState justifiedState, Bytes32 root) {
+    return false;
   }
 
   @Override
-  public SafeFuture<Optional<Boolean>> isParentStrong(Bytes32 parentRoot) {
-    return SafeFuture.completedFuture(Optional.empty());
+  public boolean isParentStrong(BeaconState justifiedState, Bytes32 parentRoot) {
+    return false;
   }
 
   @Override

storage/src/main/java/tech/pegasys/teku/storage/client/RecentChainData.java

@@ -21,6 +21,7 @@
 import java.util.NavigableMap;
 import java.util.Optional;
 import java.util.TreeMap;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.logging.log4j.LogManager;
@@ -619,6 +620,74 @@ public List<Bytes32> getAllBlockRootsAtSlot(final UInt64 slot) {
         .orElse(Collections.emptyList());
   }
 
+  public Bytes32 getProposerHead(final Bytes32 headRoot, final UInt64 slot) {
+    // if proposer boost is still active, don't attempt to override head
+    final boolean isProposerBoostActive =
+        store.getProposerBoostRoot().map(root -> !root.equals(headRoot)).orElse(false);
+    final boolean isShufflingStable = spec.atSlot(slot).getForkChoiceUtil().isShufflingStable(slot);
+    final boolean isFinalizationOk =
+        spec.atSlot(slot).getForkChoiceUtil().isFinalizationOk(store, slot);
+    final boolean isProposingOnTime = isProposingOnTime(slot);
+    final boolean isHeadLate = isBlockLate(headRoot);
+    final Optional<SignedBeaconBlock> maybeHead = store.getBlockIfAvailable(headRoot);
+
+    // to return parent root, we need all of (isHeadLate, isShufflingStable, isFfgCompetetive,
+    // isFinalizationOk, isProposingOnTime, isSingleSlotReorg, isHeadWeak, isParentStrong)
+
+    // cheap checks of that list:
+    // (isHeadLate, isShufflingStable, isFinalizationOk, isProposingOnTime);
+    // and  isProposerBoostActive (assert condition);
+    // finally need head block to make further checks
+    if (!isHeadLate
+        || !isShufflingStable
+        || !isFinalizationOk
+        || !isProposingOnTime
+        || isProposerBoostActive
+        || maybeHead.isEmpty()) {
+      return headRoot;
+    }
+
+    final SignedBeaconBlock head = maybeHead.get();
+    final boolean isFfgCompetitive =
+        store.isFfgCompetitive(headRoot, head.getParentRoot()).orElse(false);
+
+    final Optional<UInt64> maybeParentSlot = getSlotForBlockRoot(head.getParentRoot());
+    final boolean isParentSlotOk =
+        maybeParentSlot.map(uInt64 -> uInt64.increment().equals(head.getSlot())).orElse(false);
+    final boolean isCurrentTimeOk = head.getSlot().increment().equals(slot);
+    final boolean isSingleSlotReorg = isParentSlotOk && isCurrentTimeOk;
+
+    // from the initial list, check
+    // isFfgCompetitive, isSingleSlotReorg
+    if (!isFfgCompetitive || !isSingleSlotReorg) {
+      return headRoot;
+    }
+
+    final SafeFuture<Optional<BeaconState>> future =
+        store.retrieveCheckpointState(store.getJustifiedCheckpoint());
+    try {
+      final Optional<BeaconState> maybeJustifiedState = future.join();
+      // to make further checks, we would need the justified state, return headRoot if we don't have
+      // it.
+      if (maybeJustifiedState.isEmpty()) {
+        return headRoot;
+      }
+      final boolean isHeadWeak = store.isHeadWeak(maybeJustifiedState.get(), headRoot);
+      final boolean isParentStrong =
+          store.isParentStrong(maybeJustifiedState.get(), head.getParentRoot());
+      // finally, the parent must be strong, and the current head must be weak.
+      if (isHeadWeak && isParentStrong) {
+        return head.getParentRoot();
+      }
+    } catch (Exception exception) {
+      if (!(exception instanceof CancellationException)) {
+        LOG.error("Failed to get justified checkpoint", exception);
+      }
+    }
+
+    return headRoot;
+  }
+
   public void setBlockTimelinessFromArrivalTime(
       final SignedBeaconBlock block, final UInt64 arrivalTime) {
     blockTimelinessTracker.setBlockTimelinessFromArrivalTime(block, arrivalTime);