From c7b8734fc4faf7efaf8f01a205cddf3b9dca4403 Mon Sep 17 00:00:00 2001 From: Courtney <45641759+courtneyeh@users.noreply.github.com> Date: Mon, 4 Dec 2023 16:39:30 +1300 Subject: [PATCH] Enable VC crashing on 0 keys loaded (#7751) --- CHANGELOG.md | 1 + .../ValidatorClientServiceAcceptanceTest.java | 40 +++++++++++++++++++ .../teku/test/acceptance/dsl/TekuNode.java | 21 +++++++++- .../acceptance/dsl/TekuValidatorNode.java | 23 ++++++++++- .../pegasys/teku/cli/BeaconNodeCommand.java | 16 +++++++- .../teku/cli/options/ValidatorOptions.java | 13 +++++- .../cli/subcommand/VoluntaryExitCommand.java | 9 +++-- .../cli/options/ValidatorOptionsTest.java | 16 ++++++++ .../teku/validator/api/ValidatorConfig.java | 15 +++++++ .../client/NoValidatorKeysStateException.java | 24 +++++++++++ .../client/ValidatorClientService.java | 6 +++ 11 files changed, 176 insertions(+), 8 deletions(-) create mode 100644 acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java create mode 100644 validator/client/src/main/java/tech/pegasys/teku/validator/client/NoValidatorKeysStateException.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b83524b4e4..11bd66a36a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,5 +19,6 @@ the [releases page](https://github.com/Consensys/teku/releases). - Added POST `/eth/v1/beacon/states/{state_id}/validators` beacon API. - Added POST `/eth/v1/beacon/states/{state_id}/validator_balances` beacon API. - Third party library updates. +- Added `--exit-when-no-validator-keys-enabled` command line option. ### Bug Fixes diff --git a/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java b/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java new file mode 100644 index 00000000000..ceaf39d8384 --- /dev/null +++ b/acceptance-tests/src/acceptance-test/java/tech/pegasys/teku/test/acceptance/ValidatorClientServiceAcceptanceTest.java @@ -0,0 +1,40 @@ +/* + * Copyright Consensys Software Inc., 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.test.acceptance; + +import org.junit.jupiter.api.Test; +import tech.pegasys.teku.test.acceptance.dsl.AcceptanceTestBase; +import tech.pegasys.teku.test.acceptance.dsl.TekuNode; +import tech.pegasys.teku.test.acceptance.dsl.TekuValidatorNode; + +public class ValidatorClientServiceAcceptanceTest extends AcceptanceTestBase { + + @Test + void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnBeaconNode() throws Exception { + TekuNode beaconNode = createTekuNode(config -> config.withExitWhenNoValidatorKeysEnabled(true)); + beaconNode.startWithFailure( + "No loaded validators when --exit-when-no-validator-keys-enabled option is false"); + } + + @Test + void shouldFailWithNoValidatorKeysWhenExitOptionEnabledOnValidatorClient() throws Exception { + TekuNode beaconNode = createTekuNode(); + TekuValidatorNode validatorClient = + createValidatorNode( + config -> config.withBeaconNode(beaconNode).withExitWhenNoValidatorKeysEnabled(true)); + beaconNode.start(); + validatorClient.startWithFailure( + "No loaded validators when --exit-when-no-validator-keys-enabled option is false"); + } +} diff --git a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java index fd487b09e60..0c175b8106d 100644 --- a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java +++ b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuNode.java @@ -61,6 +61,7 @@ import org.apache.tuweni.units.bigints.UInt256; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.MountableFile; import tech.pegasys.teku.api.response.v1.EventType; import tech.pegasys.teku.api.response.v1.HeadEvent; @@ -143,6 +144,20 @@ public static TekuNode create( } public void start() throws Exception { + setUpStart(); + container.start(); + } + + public void startWithFailure(final String expectedError) throws Exception { + setUpStart(); + container.waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*" + expectedError + ".*") + .withStartupTimeout(Duration.ofSeconds(10))); + container.start(); + } + + private void setUpStart() throws Exception { assertThat(started).isFalse(); LOG.debug("Start node {}", nodeAlias); started = true; @@ -153,7 +168,6 @@ public void start() throws Exception { container.withCopyFileToContainer( MountableFile.forHostPath(localFile.getAbsolutePath()), targetPath)); config.getTarballsToCopy().forEach(this::copyContentsToWorkingDirectory); - container.start(); } public void startEventListener(final EventType... eventTypes) { @@ -861,6 +875,11 @@ public Config withDoppelgangerDetectionEnabled() { return this; } + public Config withExitWhenNoValidatorKeysEnabled(boolean exitWhenNoValidatorKeysEnabled) { + configMap.put("exit-when-no-validator-keys-enabled", exitWhenNoValidatorKeysEnabled); + return this; + } + public Config withInteropNumberOfValidators(final int validatorCount) { configMap.put("Xinterop-number-of-validators", validatorCount); return this; diff --git a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java index 82d5226948c..bcde7c745fe 100644 --- a/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java +++ b/acceptance-tests/src/testFixtures/java/tech/pegasys/teku/test/acceptance/dsl/TekuValidatorNode.java @@ -28,6 +28,7 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.time.Duration; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -38,6 +39,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.testcontainers.containers.Network; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.shaded.org.apache.commons.io.IOUtils; import org.testcontainers.utility.MountableFile; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -108,6 +110,20 @@ public TekuValidatorNode withValidatorKeystores(ValidatorKeystores validatorKeys } public void start() throws Exception { + setUpStart(); + container.start(); + } + + public void startWithFailure(final String expectedError) throws Exception { + setUpStart(); + container.waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*" + expectedError + ".*") + .withStartupTimeout(Duration.ofSeconds(30))); + container.start(); + } + + private void setUpStart() throws Exception { assertThat(started).isFalse(); LOG.debug("Start validator node {}", nodeAlias); started = true; @@ -118,7 +134,6 @@ public void start() throws Exception { (localFile, targetPath) -> container.withCopyFileToContainer( MountableFile.forHostPath(localFile.getAbsolutePath()), targetPath)); - container.start(); } @Override @@ -239,6 +254,12 @@ public TekuValidatorNode.Config withExternalSignerUrl(final String externalSigne return this; } + public TekuValidatorNode.Config withExitWhenNoValidatorKeysEnabled( + boolean exitWhenNoValidatorKeysEnabled) { + configMap.put("exit-when-no-validator-keys-enabled", exitWhenNoValidatorKeysEnabled); + return this; + } + public TekuValidatorNode.Config withBeaconNode(final TekuNode beaconNode) { return withBeaconNodes(beaconNode); } diff --git a/teku/src/main/java/tech/pegasys/teku/cli/BeaconNodeCommand.java b/teku/src/main/java/tech/pegasys/teku/cli/BeaconNodeCommand.java index 5cc19ea8801..eb2c2b66a58 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/BeaconNodeCommand.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/BeaconNodeCommand.java @@ -73,6 +73,7 @@ import tech.pegasys.teku.infrastructure.metrics.TekuMetricCategory; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.storage.server.DatabaseStorageException; +import tech.pegasys.teku.validator.client.NoValidatorKeysStateException; @SuppressWarnings("unused") @Command( @@ -352,12 +353,25 @@ public int handleExceptionAndReturnExitCode(final Throwable e) { return 2; } else { reportUnexpectedError(e); + if (ExceptionUtil.hasCause(e, NoValidatorKeysStateException.class)) { + return 2; + } return 1; } } public void reportUnexpectedError(final Throwable t) { - getLogger().fatal("Teku failed to start", t); + if (ExceptionUtil.hasCause(t, NoValidatorKeysStateException.class)) { + getLogger().fatal("Teku failed to start: " + t.getMessage()); + } else { + getLogger().fatal("Teku failed to start", t); + } + errorWriter.println("Teku failed to start: " + t.getMessage()); + printUsage(errorWriter); + } + + public void reportUnexpectedErrorNoStacktrace(final Throwable t) { + getLogger().fatal("Teku failed to start", t.getMessage()); errorWriter.println("Teku failed to start: " + t.getMessage()); printUsage(errorWriter); } diff --git a/teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorOptions.java b/teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorOptions.java index 463760f882a..0de25a42e35 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorOptions.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/options/ValidatorOptions.java @@ -130,6 +130,16 @@ public class ValidatorOptions { fallbackValue = "true") private boolean blockV3Enabled = ValidatorConfig.DEFAULT_BLOCK_V3_ENABLED; + @Option( + names = {"--exit-when-no-validator-keys-enabled"}, + paramLabel = "", + description = "Enable terminating the process if no validator keys are found during startup", + showDefaultValue = CommandLine.Help.Visibility.ALWAYS, + arity = "0..1", + fallbackValue = "true") + private boolean exitWhenNoValidatorKeysEnabled = + ValidatorConfig.DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED; + public void configure(TekuConfiguration.Builder builder) { builder.validator( config -> @@ -145,7 +155,8 @@ public void configure(TekuConfiguration.Builder builder) { .executorMaxQueueSize(executorMaxQueueSize) .doppelgangerDetectionEnabled(doppelgangerDetectionEnabled) .executorThreads(executorThreads) - .blockV3enabled(blockV3Enabled)); + .blockV3enabled(blockV3Enabled) + .exitWhenNoValidatorKeysEnabled(exitWhenNoValidatorKeysEnabled)); validatorProposerOptions.configure(builder); validatorKeysOptions.configure(builder); } diff --git a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java index 618956f0fc6..742d4412fe5 100644 --- a/teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java +++ b/teku/src/main/java/tech/pegasys/teku/cli/subcommand/VoluntaryExitCommand.java @@ -61,6 +61,7 @@ import tech.pegasys.teku.spec.datastructures.state.Fork; import tech.pegasys.teku.spec.datastructures.state.ForkInfo; import tech.pegasys.teku.spec.signatures.RejectingSlashingProtector; +import tech.pegasys.teku.validator.api.ValidatorConfig; import tech.pegasys.teku.validator.client.Validator; import tech.pegasys.teku.validator.client.loader.HttpClientExternalSignerFactory; import tech.pegasys.teku.validator.client.loader.PublicKeyLoader; @@ -318,20 +319,20 @@ private void initialise() { dataDirLayout = Optional.of(DataDirLayout.createFrom(dataOptions.getDataConfig())); } + final ValidatorConfig validatorConfig = config.validatorClient().getValidatorConfig(); final Supplier externalSignerHttpClientFactory = - HttpClientExternalSignerFactory.create(config.validatorClient().getValidatorConfig()); + HttpClientExternalSignerFactory.create(validatorConfig); final ValidatorLoader validatorLoader = ValidatorLoader.create( spec, - config.validatorClient().getValidatorConfig(), + validatorConfig, config.validatorClient().getInteropConfig(), externalSignerHttpClientFactory, new RejectingSlashingProtector(), slashingProtectionLogger, new PublicKeyLoader( - externalSignerHttpClientFactory, - config.validatorClient().getValidatorConfig().getValidatorExternalSignerUrl()), + externalSignerHttpClientFactory, validatorConfig.getValidatorExternalSignerUrl()), asyncRunner, metricsSystem, dataDirLayout); diff --git a/teku/src/test/java/tech/pegasys/teku/cli/options/ValidatorOptionsTest.java b/teku/src/test/java/tech/pegasys/teku/cli/options/ValidatorOptionsTest.java index 105a3bbb857..3bce582e57c 100644 --- a/teku/src/test/java/tech/pegasys/teku/cli/options/ValidatorOptionsTest.java +++ b/teku/src/test/java/tech/pegasys/teku/cli/options/ValidatorOptionsTest.java @@ -202,4 +202,20 @@ public void shouldSetDefaultGasLimitIfRegistrationDefaultGasLimitIsSpecified() { config.validatorClient().getValidatorConfig().getBuilderRegistrationDefaultGasLimit()) .isEqualTo(UInt64.valueOf(1000)); } + + @Test + public void shouldDefaultFalseExitWhenNoValidatorKeysEnabled() { + final ValidatorConfig config = + getTekuConfigurationFromArguments().validatorClient().getValidatorConfig(); + assertThat(config.isExitWhenNoValidatorKeysEnabled()).isFalse(); + } + + @Test + public void shouldSetExitWhenNoValidatorKeysEnabled() { + final ValidatorConfig config = + getTekuConfigurationFromArguments("--exit-when-no-validator-keys-enabled=true") + .validatorClient() + .getValidatorConfig(); + assertThat(config.isExitWhenNoValidatorKeysEnabled()).isTrue(); + } } diff --git a/validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java b/validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java index a64bccd8795..fb502163f6e 100644 --- a/validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java +++ b/validator/api/src/main/java/tech/pegasys/teku/validator/api/ValidatorConfig.java @@ -45,6 +45,7 @@ public class ValidatorConfig { public static final boolean DEFAULT_FAILOVERS_SEND_SUBNET_SUBSCRIPTIONS_ENABLED = true; public static final boolean DEFAULT_FAILOVERS_PUBLISH_SIGNED_DUTIES_ENABLED = true; public static final boolean DEFAULT_BLOCK_V3_ENABLED = false; + public static final boolean DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED = false; public static final boolean DEFAULT_VALIDATOR_CLIENT_SSZ_BLOCKS_ENABLED = true; public static final boolean DEFAULT_DOPPELGANGER_DETECTION_ENABLED = false; public static final int DEFAULT_EXECUTOR_MAX_QUEUE_SIZE = 20_000; @@ -85,6 +86,7 @@ public class ValidatorConfig { private final boolean failoversSendSubnetSubscriptionsEnabled; private final boolean failoversPublishSignedDutiesEnabled; private final boolean blockV3Enabled; + private final boolean exitWhenNoValidatorKeysEnabled; private final UInt64 builderRegistrationDefaultGasLimit; private final int builderRegistrationSendingBatchSize; private final Optional builderRegistrationTimestampOverride; @@ -120,6 +122,7 @@ private ValidatorConfig( final boolean failoversSendSubnetSubscriptionsEnabled, final boolean failoversPublishSignedDutiesEnabled, final boolean blockV3Enabled, + final boolean exitWhenNoValidatorKeysEnabled, final UInt64 builderRegistrationDefaultGasLimit, final int builderRegistrationSendingBatchSize, final Optional builderRegistrationTimestampOverride, @@ -155,6 +158,7 @@ private ValidatorConfig( this.failoversSendSubnetSubscriptionsEnabled = failoversSendSubnetSubscriptionsEnabled; this.failoversPublishSignedDutiesEnabled = failoversPublishSignedDutiesEnabled; this.blockV3Enabled = blockV3Enabled; + this.exitWhenNoValidatorKeysEnabled = exitWhenNoValidatorKeysEnabled; this.builderRegistrationDefaultGasLimit = builderRegistrationDefaultGasLimit; this.builderRegistrationSendingBatchSize = builderRegistrationSendingBatchSize; this.builderRegistrationTimestampOverride = builderRegistrationTimestampOverride; @@ -279,6 +283,10 @@ public boolean isBlockV3Enabled() { return blockV3Enabled; } + public boolean isExitWhenNoValidatorKeysEnabled() { + return exitWhenNoValidatorKeysEnabled; + } + public boolean isBuilderRegistrationDefaultEnabled() { return builderRegistrationDefaultEnabled; } @@ -338,6 +346,7 @@ public static final class Builder { private boolean failoversPublishSignedDutiesEnabled = DEFAULT_FAILOVERS_PUBLISH_SIGNED_DUTIES_ENABLED; private boolean blockV3Enabled = DEFAULT_BLOCK_V3_ENABLED; + private boolean exitWhenNoValidatorKeysEnabled = DEFAULT_EXIT_WHEN_NO_VALIDATOR_KEYS_ENABLED; private UInt64 builderRegistrationDefaultGasLimit = DEFAULT_BUILDER_REGISTRATION_GAS_LIMIT; private int builderRegistrationSendingBatchSize = DEFAULT_VALIDATOR_REGISTRATION_SENDING_BATCH_SIZE; @@ -521,6 +530,11 @@ public Builder blockV3enabled(final boolean useBlockV3) { return this; } + public Builder exitWhenNoValidatorKeysEnabled(final boolean exitWhenNoValidatorKeysEnabled) { + this.exitWhenNoValidatorKeysEnabled = exitWhenNoValidatorKeysEnabled; + return this; + } + public Builder builderRegistrationDefaultGasLimit( final UInt64 builderRegistrationDefaultGasLimit) { this.builderRegistrationDefaultGasLimit = builderRegistrationDefaultGasLimit; @@ -590,6 +604,7 @@ public ValidatorConfig build() { failoversSendSubnetSubscriptionsEnabled, failoversPublishSignedDutiesEnabled, blockV3Enabled, + exitWhenNoValidatorKeysEnabled, builderRegistrationDefaultGasLimit, builderRegistrationSendingBatchSize, builderRegistrationTimestampOverride, diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/NoValidatorKeysStateException.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/NoValidatorKeysStateException.java new file mode 100644 index 00000000000..c22709976a4 --- /dev/null +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/NoValidatorKeysStateException.java @@ -0,0 +1,24 @@ +/* + * Copyright Consensys Software Inc., 2023 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.validator.client; + +public class NoValidatorKeysStateException extends IllegalStateException { + public NoValidatorKeysStateException(String message, Throwable cause) { + super(message, cause); + } + + public NoValidatorKeysStateException(String message) { + super(message); + } +} diff --git a/validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java b/validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java index b0b5288a910..aeaec99f7cc 100644 --- a/validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java +++ b/validator/client/src/main/java/tech/pegasys/teku/validator/client/ValidatorClientService.java @@ -270,6 +270,12 @@ public static ValidatorClientService create( }) .always(() -> LOG.trace("Finished starting validator client service.")); + if (validatorConfig.isExitWhenNoValidatorKeysEnabled() + && validatorLoader.getOwnedValidators().getActiveValidators().size() == 0) { + throw new NoValidatorKeysStateException( + "No loaded validators when --exit-when-no-validator-keys-enabled option is false"); + } + return validatorClientService; }