Enable VC crashing on 0 keys loaded (Consensys#7751)

CHANGELOG.md

@@ -19,5 +19,6 @@ the [releases page](https://github.com/Consensys/teku/releases).
 - Added POST `/eth/v1/beacon/states/{state_id}/validators` beacon API.
 - Added POST `/eth/v1/beacon/states/{state_id}/validator_balances` beacon API.
 - Third party library updates.
+- Added `--exit-when-no-validator-keys-enabled` command line option.
 
 ### Bug Fixes

acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Consensys Software Inc., 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.test.acceptance;
+
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase;
+import tech.pegasys.teku.test.acceptance.dsl.TekuNode;
+import tech.pegasys.teku.test.acceptance.dsl.TekuValidatorNode;
+
+public class ValidatorClientServiceAcceptanceTest extends AcceptanceTestBase {
+
+  @Test
+  void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnBeaconNode() throws Exception {
+    TekuNode beaconNode = createTekuNode(config -> config.withExitWhenNoValidatorKeysEnabled(true));
+    beaconNode.startWithFailure(
+        "No loaded validators when --exit-when-no-validator-keys-enabled option is false");
+  }
+
+  @Test
+  void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnValidatorClient() throws Exception {
+    TekuNode beaconNode = createTekuNode();
+    TekuValidatorNode validatorClient =
+        createValidatorNode(
+            config -> config.withBeaconNode(beaconNode).withExitWhenNoValidatorKeysEnabled(true));
+    beaconNode.start();
+    validatorClient.startWithFailure(
+        "No loaded validators when --exit-when-no-validator-keys-enabled option is false");
+  }
+}

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java

@@ -61,6 +61,7 @@
 import org.apache.tuweni.units.bigints.UInt256;
 import org.testcontainers.containers.Network;
 import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
 import org.testcontainers.utility.MountableFile;
 import tech.pegasys.teku.api.response.v1.EventType;
 import tech.pegasys.teku.api.response.v1.HeadEvent;
@@ -143,6 +144,20 @@ public static TekuNode create(
   }
 
   public void start() throws Exception {
+    setUpStart();
+    container.start();
+  }
+
+  public void startWithFailure(final String expectedError) throws Exception {
+    setUpStart();
+    container.waitingFor(
+        new LogMessageWaitStrategy()
+            .withRegEx(".*" + expectedError + ".*")
+            .withStartupTimeout(Duration.ofSeconds(10)));
+    container.start();
+  }
+
+  private void setUpStart() throws Exception {
     assertThat(started).isFalse();
     LOG.debug("Start node {}", nodeAlias);
     started = true;
@@ -153,7 +168,6 @@ public void start() throws Exception {
             container.withCopyFileToContainer(
                 MountableFile.forHostPath(localFile.getAbsolutePath()), targetPath));
     config.getTarballsToCopy().forEach(this::copyContentsToWorkingDirectory);
-    container.start();
   }
 
   public void startEventListener(final EventType... eventTypes) {
@@ -861,6 +875,11 @@ public Config withDoppelgangerDetectionEnabled() {
       return this;
     }
 
+    public Config withExitWhenNoValidatorKeysEnabled(boolean exitWhenNoValidatorKeysEnabled) {
+      configMap.put("exit-when-no-validator-keys-enabled", exitWhenNoValidatorKeysEnabled);
+      return this;
+    }
+
     public Config withInteropNumberOfValidators(final int validatorCount) {
       configMap.put("Xinterop-number-of-validators", validatorCount);
       return this;

acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java

@@ -28,6 +28,7 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -38,6 +39,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.testcontainers.containers.Network;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
 import org.testcontainers.shaded.org.apache.commons.io.IOUtils;
 import org.testcontainers.utility.MountableFile;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
@@ -108,6 +110,20 @@ public TekuValidatorNode withValidatorKeystores(ValidatorKeystores validatorKeys
   }
 
   public void start() throws Exception {
+    setUpStart();
+    container.start();
+  }
+
+  public void startWithFailure(final String expectedError) throws Exception {
+    setUpStart();
+    container.waitingFor(
+        new LogMessageWaitStrategy()
+            .withRegEx(".*" + expectedError + ".*")
+            .withStartupTimeout(Duration.ofSeconds(30)));
+    container.start();
+  }
+
+  private void setUpStart() throws Exception {
     assertThat(started).isFalse();
     LOG.debug("Start validator node {}", nodeAlias);
     started = true;
@@ -118,7 +134,6 @@ public void start() throws Exception {
         (localFile, targetPath) ->
             container.withCopyFileToContainer(
                 MountableFile.forHostPath(localFile.getAbsolutePath()), targetPath));
-    container.start();
   }
 
   @Override
@@ -239,6 +254,12 @@ public TekuValidatorNode.Config withExternalSignerUrl(final String externalSigne
       return this;
     }
 
+    public TekuValidatorNode.Config withExitWhenNoValidatorKeysEnabled(
+        boolean exitWhenNoValidatorKeysEnabled) {
+      configMap.put("exit-when-no-validator-keys-enabled", exitWhenNoValidatorKeysEnabled);
+      return this;
+    }
+
     public TekuValidatorNode.Config withBeaconNode(final TekuNode beaconNode) {
       return withBeaconNodes(beaconNode);
     }

teku/src/main/java/tech/pegasys/teku/cli/BeaconNodeCommand.java

@@ -73,6 +73,7 @@
 import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.storage.server.DatabaseStorageException;
+import tech.pegasys.teku.validator.client.NoValidatorKeysStateException;
 
 @SuppressWarnings("unused")
 @Command(
@@ -352,12 +353,25 @@ public int handleExceptionAndReturnExitCode(final Throwable e) {
       return 2;
     } else {
       reportUnexpectedError(e);
+      if (ExceptionUtil.hasCause(e, NoValidatorKeysStateException.class)) {
+        return 2;
+      }
       return 1;
     }
   }
 
   public void reportUnexpectedError(final Throwable t) {
-    getLogger().fatal("Teku failed to start", t);
+    if (ExceptionUtil.hasCause(t, NoValidatorKeysStateException.class)) {
+      getLogger().fatal("Teku failed to start: " + t.getMessage());
+    } else {
+      getLogger().fatal("Teku failed to start", t);
+    }
+    errorWriter.println("Teku failed to start: " + t.getMessage());
+    printUsage(errorWriter);
+  }
+
+  public void reportUnexpectedErrorNoStacktrace(final Throwable t) {
+    getLogger().fatal("Teku failed to start", t.getMessage());
     errorWriter.println("Teku failed to start: " + t.getMessage());
     printUsage(errorWriter);
   }

teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorOptions.java

@@ -130,6 +130,16 @@ public class ValidatorOptions {
       fallbackValue = "true")
   private boolean blockV3Enabled = ValidatorConfig.DEFAULT_BLOCK_V3_ENABLED;
 
+  @Option(
+      names = {"--exit-when-no-validator-keys-enabled"},
+      paramLabel = "<BOOLEAN>",
+      description = "Enable terminating the process if no validator keys are found during startup",
+      showDefaultValue = CommandLine.Help.Visibility.ALWAYS,
+      arity = "0..1",
+      fallbackValue = "true")
+  private boolean exitWhenNoValidatorKeysEnabled =
+      ValidatorConfig.DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED;
+
   public void configure(TekuConfiguration.Builder builder) {
     builder.validator(
         config ->
@@ -145,7 +155,8 @@ public void configure(TekuConfiguration.Builder builder) {
                 .executorMaxQueueSize(executorMaxQueueSize)
                 .doppelgangerDetectionEnabled(doppelgangerDetectionEnabled)
                 .executorThreads(executorThreads)
-                .blockV3enabled(blockV3Enabled));
+                .blockV3enabled(blockV3Enabled)
+                .exitWhenNoValidatorKeysEnabled(exitWhenNoValidatorKeysEnabled));
     validatorProposerOptions.configure(builder);
     validatorKeysOptions.configure(builder);
   }

teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java

@@ -61,6 +61,7 @@
 import tech.pegasys.teku.spec.datastructures.state.Fork;
 import tech.pegasys.teku.spec.datastructures.state.ForkInfo;
 import tech.pegasys.teku.spec.signatures.RejectingSlashingProtector;
+import tech.pegasys.teku.validator.api.ValidatorConfig;
 import tech.pegasys.teku.validator.client.Validator;
 import tech.pegasys.teku.validator.client.loader.HttpClientExternalSignerFactory;
 import tech.pegasys.teku.validator.client.loader.PublicKeyLoader;
@@ -318,20 +319,20 @@ private void initialise() {
       dataDirLayout = Optional.of(DataDirLayout.createFrom(dataOptions.getDataConfig()));
     }
 
+    final ValidatorConfig validatorConfig = config.validatorClient().getValidatorConfig();
     final Supplier<HttpClient> externalSignerHttpClientFactory =
-        HttpClientExternalSignerFactory.create(config.validatorClient().getValidatorConfig());
+        HttpClientExternalSignerFactory.create(validatorConfig);
 
     final ValidatorLoader validatorLoader =
         ValidatorLoader.create(
             spec,
-            config.validatorClient().getValidatorConfig(),
+            validatorConfig,
             config.validatorClient().getInteropConfig(),
             externalSignerHttpClientFactory,
             new RejectingSlashingProtector(),
             slashingProtectionLogger,
             new PublicKeyLoader(
-                externalSignerHttpClientFactory,
-                config.validatorClient().getValidatorConfig().getValidatorExternalSignerUrl()),
+                externalSignerHttpClientFactory, validatorConfig.getValidatorExternalSignerUrl()),
             asyncRunner,
             metricsSystem,
             dataDirLayout);

teku/src/test/java/tech/pegasys/teku/cli/options/ValidatorOptionsTest.java

@@ -202,4 +202,20 @@ public void shouldSetDefaultGasLimitIfRegistrationDefaultGasLimitIsSpecified() {
             config.validatorClient().getValidatorConfig().getBuilderRegistrationDefaultGasLimit())
         .isEqualTo(UInt64.valueOf(1000));
   }
+
+  @Test
+  public void shouldDefaultFalseExitWhenNoValidatorKeysEnabled() {
+    final ValidatorConfig config =
+        getTekuConfigurationFromArguments().validatorClient().getValidatorConfig();
+    assertThat(config.isExitWhenNoValidatorKeysEnabled()).isFalse();
+  }
+
+  @Test
+  public void shouldSetExitWhenNoValidatorKeysEnabled() {
+    final ValidatorConfig config =
+        getTekuConfigurationFromArguments("--exit-when-no-validator-keys-enabled=true")
+            .validatorClient()
+            .getValidatorConfig();
+    assertThat(config.isExitWhenNoValidatorKeysEnabled()).isTrue();
+  }
 }

validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java

@@ -45,6 +45,7 @@ public class ValidatorConfig {
   public static final boolean DEFAULT_FAILOVERS_SEND_SUBNET_SUBSCRIPTIONS_ENABLED = true;
   public static final boolean DEFAULT_FAILOVERS_PUBLISH_SIGNED_DUTIES_ENABLED = true;
   public static final boolean DEFAULT_BLOCK_V3_ENABLED = false;
+  public static final boolean DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED = false;
   public static final boolean DEFAULT_VALIDATOR_CLIENT_SSZ_BLOCKS_ENABLED = true;
   public static final boolean DEFAULT_DOPPELGANGER_DETECTION_ENABLED = false;
   public static final int DEFAULT_EXECUTOR_MAX_QUEUE_SIZE = 20_000;
@@ -85,6 +86,7 @@ public class ValidatorConfig {
   private final boolean failoversSendSubnetSubscriptionsEnabled;
   private final boolean failoversPublishSignedDutiesEnabled;
   private final boolean blockV3Enabled;
+  private final boolean exitWhenNoValidatorKeysEnabled;
   private final UInt64 builderRegistrationDefaultGasLimit;
   private final int builderRegistrationSendingBatchSize;
   private final Optional<UInt64> builderRegistrationTimestampOverride;
@@ -120,6 +122,7 @@ private ValidatorConfig(
       final boolean failoversSendSubnetSubscriptionsEnabled,
       final boolean failoversPublishSignedDutiesEnabled,
       final boolean blockV3Enabled,
+      final boolean exitWhenNoValidatorKeysEnabled,
       final UInt64 builderRegistrationDefaultGasLimit,
       final int builderRegistrationSendingBatchSize,
       final Optional<UInt64> builderRegistrationTimestampOverride,
@@ -155,6 +158,7 @@ private ValidatorConfig(
     this.failoversSendSubnetSubscriptionsEnabled = failoversSendSubnetSubscriptionsEnabled;
     this.failoversPublishSignedDutiesEnabled = failoversPublishSignedDutiesEnabled;
     this.blockV3Enabled = blockV3Enabled;
+    this.exitWhenNoValidatorKeysEnabled = exitWhenNoValidatorKeysEnabled;
     this.builderRegistrationDefaultGasLimit = builderRegistrationDefaultGasLimit;
     this.builderRegistrationSendingBatchSize = builderRegistrationSendingBatchSize;
     this.builderRegistrationTimestampOverride = builderRegistrationTimestampOverride;
@@ -279,6 +283,10 @@ public boolean isBlockV3Enabled() {
     return blockV3Enabled;
   }
 
+  public boolean isExitWhenNoValidatorKeysEnabled() {
+    return exitWhenNoValidatorKeysEnabled;
+  }
+
   public boolean isBuilderRegistrationDefaultEnabled() {
     return builderRegistrationDefaultEnabled;
   }
@@ -338,6 +346,7 @@ public static final class Builder {
     private boolean failoversPublishSignedDutiesEnabled =
         DEFAULT_FAILOVERS_PUBLISH_SIGNED_DUTIES_ENABLED;
     private boolean blockV3Enabled = DEFAULT_BLOCK_V3_ENABLED;
+    private boolean exitWhenNoValidatorKeysEnabled = DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED;
     private UInt64 builderRegistrationDefaultGasLimit = DEFAULT_BUILDER_REGISTRATION_GAS_LIMIT;
     private int builderRegistrationSendingBatchSize =
         DEFAULT_VALIDATOR_REGISTRATION_SENDING_BATCH_SIZE;
@@ -521,6 +530,11 @@ public Builder blockV3enabled(final boolean useBlockV3) {
       return this;
     }
 
+    public Builder exitWhenNoValidatorKeysEnabled(final boolean exitWhenNoValidatorKeysEnabled) {
+      this.exitWhenNoValidatorKeysEnabled = exitWhenNoValidatorKeysEnabled;
+      return this;
+    }
+
     public Builder builderRegistrationDefaultGasLimit(
         final UInt64 builderRegistrationDefaultGasLimit) {
       this.builderRegistrationDefaultGasLimit = builderRegistrationDefaultGasLimit;
@@ -590,6 +604,7 @@ public ValidatorConfig build() {
           failoversSendSubnetSubscriptionsEnabled,
           failoversPublishSignedDutiesEnabled,
           blockV3Enabled,
+          exitWhenNoValidatorKeysEnabled,
           builderRegistrationDefaultGasLimit,
           builderRegistrationSendingBatchSize,
           builderRegistrationTimestampOverride,

validator/client/src/main/java/tech/pegasys/teku/validator/client/NoValidatorKeysStateException.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright Consensys Software Inc., 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.validator.client;
+
+public class NoValidatorKeysStateException extends IllegalStateException {
+  public NoValidatorKeysStateException(String message, Throwable cause) {
+    super(message, cause);
+  }
+
+  public NoValidatorKeysStateException(String message) {
+    super(message);
+  }
+}

validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java

@@ -270,6 +270,12 @@ public static ValidatorClientService create(
             })
         .always(() -> LOG.trace("Finished starting validator client service."));
 
+    if (validatorConfig.isExitWhenNoValidatorKeysEnabled()
+        && validatorLoader.getOwnedValidators().getActiveValidators().size() == 0) {
+      throw new NoValidatorKeysStateException(
+          "No loaded validators when --exit-when-no-validator-keys-enabled option is false");
+    }
+
     return validatorClientService;
   }