Fix issue where remote config is missing data (Consensys#7510)

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigLoader.java

@@ -22,6 +22,8 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Consumer;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import tech.pegasys.teku.infrastructure.io.resource.ResourceLoader;
 import tech.pegasys.teku.spec.config.builder.SpecConfigBuilder;
 import tech.pegasys.teku.spec.networks.Eth2Network;
@@ -33,6 +35,8 @@ public class SpecConfigLoader {
   private static final String CONFIG_PATH = "configs/";
   private static final String PRESET_PATH = "presets/";
 
+  private static final Logger LOG = LogManager.getLogger();
+
   public static SpecConfig loadConfigStrict(final String configName) {
     return loadConfig(configName, false, __ -> {});
   }
@@ -57,6 +61,17 @@ public static SpecConfig loadConfig(
 
   public static SpecConfig loadRemoteConfig(final Map<String, String> config) {
     final SpecConfigReader reader = new SpecConfigReader();
+    if (config.containsKey(SpecConfigReader.CONFIG_NAME_KEY)) {
+      final String configNameKey = config.get(SpecConfigReader.CONFIG_NAME_KEY);
+      try {
+        processConfig(configNameKey, reader, true);
+      } catch (IllegalArgumentException exception) {
+        LOG.debug(
+            "Failed to load base configuration from {}, {}",
+            () -> configNameKey,
+            exception::getMessage);
+      }
+    }
     if (config.containsKey(SpecConfigReader.PRESET_KEY)) {
       try {
         applyPreset("remote", reader, true, config.get(SpecConfigReader.PRESET_KEY));

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/SpecConfigReader.java

@@ -53,7 +53,7 @@
 public class SpecConfigReader {
   private static final Logger LOG = LogManager.getLogger();
   public static final String PRESET_KEY = "PRESET_BASE";
-  private static final String CONFIG_NAME_KEY = "CONFIG_NAME";
+  public static final String CONFIG_NAME_KEY = "CONFIG_NAME";
   private static final ImmutableSet<String> KEYS_TO_IGNORE =
       ImmutableSet.of(
           PRESET_KEY,

ethereum/spec/src/main/java/tech/pegasys/teku/spec/config/builder/SpecConfigBuilder.java

@@ -132,7 +132,7 @@ public SpecConfig build() {
           }
         });
     validate();
-    SpecConfig config =
+    final SpecConfig config =
         new SpecConfigPhase0(
             rawConfig,
             eth1FollowDistance,

teku/src/test/java/tech/pegasys/teku/cli/subcommand/RemoteSpecLoaderTest.java

@@ -14,18 +14,25 @@
 package tech.pegasys.teku.cli.subcommand;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.io.Resources;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Map;
 import java.util.Optional;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.api.ConfigProvider;
 import tech.pegasys.teku.api.response.v1.config.GetSpecResponse;
-import tech.pegasys.teku.infrastructure.exceptions.InvalidConfigurationException;
+import tech.pegasys.teku.infrastructure.bytes.Bytes4;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.config.SpecConfig;
+import tech.pegasys.teku.spec.config.SpecConfigLoader;
 import tech.pegasys.teku.validator.remote.apiclient.OkHttpValidatorRestApiClient;
 
 class RemoteSpecLoaderTest {
@@ -44,15 +51,37 @@ void shouldIgnoreUnknownConfigItems() {
   }
 
   @Test
-  void shouldFailWhenRequiredItemsAreMissing() {
+  void shouldFillWhenRequiredItemsAreMissing() {
     final Map<String, String> rawConfig = getRawConfigForSpec(spec);
     assertThat(rawConfig.remove("GENESIS_FORK_VERSION")).isNotNull();
 
     when(apiClient.getConfigSpec()).thenReturn(Optional.of(new GetSpecResponse(rawConfig)));
 
-    assertThatThrownBy(() -> RemoteSpecLoader.getSpec(apiClient))
-        .isInstanceOf(InvalidConfigurationException.class)
-        .hasMessageContaining("GENESIS_FORK_VERSION");
+    final SpecConfig config = RemoteSpecLoader.getSpec(apiClient).getSpecConfig(UInt64.ONE);
+    assertThat(config.getGenesisForkVersion()).isEqualTo(Bytes4.fromHexString("0x00000001"));
+  }
+
+  @Test
+  void shouldProvideValidSpecConfigWithIncompleteRemoteConfig() throws IOException {
+    final String jsonConfig =
+        Resources.toString(
+            Resources.getResource(RemoteSpecLoaderTest.class, "config_missing_network_fields.json"),
+            StandardCharsets.UTF_8);
+    final ObjectMapper objectMapper = new ObjectMapper();
+    TypeReference<Map<String, String>> typeReference = new TypeReference<>() {};
+    Map<String, String> data = objectMapper.readValue(jsonConfig, typeReference);
+    final SpecConfig specConfig = SpecConfigLoader.loadRemoteConfig(data);
+
+    // Check values not assigned, using default values
+    assertThat(specConfig.getGossipMaxSize()).isEqualTo(10485760);
+    assertThat(specConfig.getMaxChunkSize()).isEqualTo(10485760);
+    assertThat(specConfig.getMaxRequestBlocks()).isEqualTo(1024);
+    assertThat(specConfig.getEpochsPerSubnetSubscription()).isEqualTo(256);
+    assertThat(specConfig.getMinEpochsForBlockRequests()).isEqualTo(33024);
+    assertThat(specConfig.getTtfbTimeout()).isEqualTo(5);
+    assertThat(specConfig.getRespTimeout()).isEqualTo(10);
+    assertThat(specConfig.getAttestationPropagationSlotRange()).isEqualTo(32);
+    assertThat(specConfig.getMaximumGossipClockDisparity()).isEqualTo(500);
   }
 
   private Map<String, String> getRawConfigForSpec(final Spec spec) {

teku/src/test/resources/tech/pegasys/teku/cli/subcommand/config_missing_network_fields.json

@@ -0,0 +1,97 @@
+{
+  "CONFIG_NAME": "mainnet",
+  "PRESET_BASE": "mainnet",
+  "TERMINAL_TOTAL_DIFFICULTY": "58750000000000000000000",
+  "TERMINAL_BLOCK_HASH": "0x0000000000000000000000000000000000000000000000000000000000000000",
+  "TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH": "18446744073709551615",
+  "SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY": "128",
+  "MIN_GENESIS_ACTIVE_VALIDATOR_COUNT": "16384",
+  "MIN_GENESIS_TIME": "1606824000",
+  "GENESIS_FORK_VERSION": "0x00000000",
+  "GENESIS_DELAY": "604800",
+  "ALTAIR_FORK_VERSION": "0x01000000",
+  "ALTAIR_FORK_EPOCH": "74240",
+  "BELLATRIX_FORK_VERSION": "0x02000000",
+  "BELLATRIX_FORK_EPOCH": "144896",
+  "CAPELLA_FORK_VERSION": "0x03000000",
+  "CAPELLA_FORK_EPOCH": "194048",
+  "SECONDS_PER_SLOT": "12",
+  "SECONDS_PER_ETH1_BLOCK": "14",
+  "MIN_VALIDATOR_WITHDRAWABILITY_DELAY": "256",
+  "SHARD_COMMITTEE_PERIOD": "256",
+  "ETH1_FOLLOW_DISTANCE": "2048",
+  "SUBNETS_PER_NODE": "2",
+  "INACTIVITY_SCORE_BIAS": "4",
+  "INACTIVITY_SCORE_RECOVERY_RATE": "16",
+  "EJECTION_BALANCE": "16000000000",
+  "MIN_PER_EPOCH_CHURN_LIMIT": "4",
+  "CHURN_LIMIT_QUOTIENT": "65536",
+  "PROPOSER_SCORE_BOOST": "40",
+  "DEPOSIT_CHAIN_ID": "1",
+  "DEPOSIT_NETWORK_ID": "1",
+  "DEPOSIT_CONTRACT_ADDRESS": "0x00000000219ab540356cbb839cbe05303d7705fa",
+  "MAX_COMMITTEES_PER_SLOT": "64",
+  "TARGET_COMMITTEE_SIZE": "128",
+  "MAX_VALIDATORS_PER_COMMITTEE": "2048",
+  "SHUFFLE_ROUND_COUNT": "90",
+  "HYSTERESIS_QUOTIENT": "4",
+  "HYSTERESIS_DOWNWARD_MULTIPLIER": "1",
+  "HYSTERESIS_UPWARD_MULTIPLIER": "5",
+  "SAFE_SLOTS_TO_UPDATE_JUSTIFIED": "8",
+  "MIN_DEPOSIT_AMOUNT": "1000000000",
+  "MAX_EFFECTIVE_BALANCE": "32000000000",
+  "EFFECTIVE_BALANCE_INCREMENT": "1000000000",
+  "MIN_ATTESTATION_INCLUSION_DELAY": "1",
+  "SLOTS_PER_EPOCH": "32",
+  "MIN_SEED_LOOKAHEAD": "1",
+  "MAX_SEED_LOOKAHEAD": "4",
+  "EPOCHS_PER_ETH1_VOTING_PERIOD": "64",
+  "SLOTS_PER_HISTORICAL_ROOT": "8192",
+  "MIN_EPOCHS_TO_INACTIVITY_PENALTY": "4",
+  "EPOCHS_PER_HISTORICAL_VECTOR": "65536",
+  "EPOCHS_PER_SLASHINGS_VECTOR": "8192",
+  "HISTORICAL_ROOTS_LIMIT": "16777216",
+  "VALIDATOR_REGISTRY_LIMIT": "1099511627776",
+  "BASE_REWARD_FACTOR": "64",
+  "WHISTLEBLOWER_REWARD_QUOTIENT": "512",
+  "PROPOSER_REWARD_QUOTIENT": "8",
+  "INACTIVITY_PENALTY_QUOTIENT": "67108864",
+  "MIN_SLASHING_PENALTY_QUOTIENT": "128",
+  "PROPORTIONAL_SLASHING_MULTIPLIER": "1",
+  "MAX_PROPOSER_SLASHINGS": "16",
+  "MAX_ATTESTER_SLASHINGS": "2",
+  "MAX_ATTESTATIONS": "128",
+  "MAX_DEPOSITS": "16",
+  "MAX_VOLUNTARY_EXITS": "16",
+  "INACTIVITY_PENALTY_QUOTIENT_ALTAIR": "50331648",
+  "MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR": "64",
+  "PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR": "2",
+  "SYNC_COMMITTEE_SIZE": "512",
+  "EPOCHS_PER_SYNC_COMMITTEE_PERIOD": "256",
+  "MIN_SYNC_COMMITTEE_PARTICIPANTS": "1",
+  "INACTIVITY_PENALTY_QUOTIENT_BELLATRIX": "16777216",
+  "MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX": "32",
+  "PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX": "3",
+  "MAX_BYTES_PER_TRANSACTION": "1073741824",
+  "MAX_TRANSACTIONS_PER_PAYLOAD": "1048576",
+  "BYTES_PER_LOGS_BLOOM": "256",
+  "MAX_EXTRA_DATA_BYTES": "32",
+  "MAX_BLS_TO_EXECUTION_CHANGES": "16",
+  "MAX_WITHDRAWALS_PER_PAYLOAD": "16",
+  "MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP": "16384",
+  "SYNC_COMMITTEE_SUBNET_COUNT": "4",
+  "BLS_WITHDRAWAL_PREFIX": "0x00",
+  "DOMAIN_RANDAO": "0x02000000",
+  "DOMAIN_DEPOSIT": "0x03000000",
+  "DOMAIN_SELECTION_PROOF": "0x05000000",
+  "DOMAIN_SYNC_COMMITTEE": "0x07000000",
+  "DOMAIN_BEACON_ATTESTER": "0x01000000",
+  "DOMAIN_APPLICATION_MASK": "0x00000001",
+  "DOMAIN_VOLUNTARY_EXIT": "0x04000000",
+  "TARGET_AGGREGATORS_PER_COMMITTEE": "16",
+  "TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE": "16",
+  "DOMAIN_AGGREGATE_AND_PROOF": "0x06000000",
+  "DOMAIN_CONTRIBUTION_AND_PROOF": "0x09000000",
+  "DOMAIN_BEACON_PROPOSER": "0x00000000",
+  "DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF": "0x08000000"
+}