Skip to content

Commit

Permalink
Data storage option refactoring (#7807)
Browse files Browse the repository at this point in the history 
Renaming and refactoring the classes common to both Bonsai and Verkle to facilitate the future integration of Verkle.

---------

Signed-off-by: Karim Taam <[email protected]>
  • Loading branch information
@matkt
matkt authored Oct 25, 2024
1 parent a27dccd commit a3b8863
Show file tree
Hide file tree
Showing 28 changed files with 1,136 additions and 710 deletions.
2 changes: 1 addition & 1 deletion ...l/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
import static com.google.common.base.Preconditions.checkState;
import static java.nio.charset.StandardCharsets.UTF_8;

import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration;
import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
Expand Down
144 changes: 83 additions & 61 deletions besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import org.hyperledger.besu.cli.custom.JsonRPCAllowlistHostsProperty;
import org.hyperledger.besu.cli.error.BesuExecutionExceptionHandler;
import org.hyperledger.besu.cli.error.BesuParameterExceptionHandler;
import org.hyperledger.besu.cli.options.DataStorageOptions;
import org.hyperledger.besu.cli.options.MiningOptions;
import org.hyperledger.besu.cli.options.TransactionPoolOptions;
import org.hyperledger.besu.cli.options.stable.ApiConfigurationOptions;
Expand All @@ -57,6 +56,8 @@
import org.hyperledger.besu.cli.options.stable.PermissionsOptions;
import org.hyperledger.besu.cli.options.stable.PluginsConfigurationOptions;
import org.hyperledger.besu.cli.options.stable.RpcWebsocketOptions;
import org.hyperledger.besu.cli.options.storage.DataStorageOptions;
import org.hyperledger.besu.cli.options.storage.DiffBasedSubStorageOptions;
import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions;
import org.hyperledger.besu.cli.options.unstable.DnsOptions;
import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
Expand Down Expand Up @@ -139,7 +140,9 @@
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStorageProviderBuilder;
import org.hyperledger.besu.ethereum.transaction.TransactionSimulator;
import org.hyperledger.besu.ethereum.worldstate.DataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.DiffBasedSubStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDataStorageConfiguration;
import org.hyperledger.besu.ethereum.worldstate.ImmutableDiffBasedSubStorageConfiguration;
import org.hyperledger.besu.evm.precompile.AbstractAltBnPrecompiledContract;
import org.hyperledger.besu.evm.precompile.BigIntegerModularExponentiationPrecompiledContract;
import org.hyperledger.besu.evm.precompile.KZGPointEvalPrecompiledContract;
Expand Down Expand Up @@ -1643,7 +1646,11 @@ && isOptionSet(commandLine, "--sync-min-peers")) {
CommandLineUtils.failIfOptionDoesntMeetRequirement(
commandLine,
"--Xsnapsync-synchronizer-flat option can only be used when --Xbonsai-full-flat-db-enabled is true",
dataStorageOptions.toDomainObject().getUnstable().getBonsaiFullFlatDbEnabled(),
dataStorageOptions
.toDomainObject()
.getDiffBasedSubStorageConfiguration()
.getUnstable()
.getFullFlatDbEnabled(),
asList(
"--Xsnapsync-synchronizer-flat-account-healed-count-per-request",
"--Xsnapsync-synchronizer-flat-slot-healed-count-per-request"));
Expand Down Expand Up @@ -1775,38 +1782,46 @@ public BesuControllerBuilder setupControllerBuilder() {
.withMiningParameters(miningParametersSupplier.get())
.withJsonRpcHttpOptions(jsonRpcHttpOptions);
final KeyValueStorageProvider storageProvider = keyValueStorageProvider(keyValueStorageName);
return controllerBuilder
.fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet())
.synchronizerConfiguration(buildSyncConfig())
.ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject())
.networkConfiguration(unstableNetworkingOptions.toDomainObject())
.dataDirectory(dataDir())
.dataStorageConfiguration(getDataStorageConfiguration())
.miningParameters(miningParametersSupplier.get())
.transactionPoolConfiguration(buildTransactionPoolConfiguration())
.nodeKey(new NodeKey(securityModule()))
.metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem())
.messagePermissioningProviders(permissioningService.getMessagePermissioningProviders())
.privacyParameters(privacyParameters())
.clock(Clock.systemUTC())
.isRevertReasonEnabled(isRevertReasonEnabled)
.isParallelTxProcessingEnabled(
dataStorageConfiguration.getUnstable().isParallelTxProcessingEnabled())
.storageProvider(storageProvider)
.gasLimitCalculator(
miningParametersSupplier.get().getTargetGasLimit().isPresent()
? new FrontierTargetingGasLimitCalculator()
: GasLimitCalculator.constant())
.requiredBlocks(requiredBlocks)
.reorgLoggingThreshold(reorgLoggingThreshold)
.evmConfiguration(unstableEvmOptions.toDomainObject())
.maxPeers(p2PDiscoveryOptions.maxPeers)
.maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers)
.randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority)
.chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
.cacheLastBlocks(numberOfblocksToCache)
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
.besuComponent(besuComponent);
BesuControllerBuilder besuControllerBuilder =
controllerBuilder
.fromEthNetworkConfig(updateNetworkConfig(network), getDefaultSyncModeIfNotSet())
.synchronizerConfiguration(buildSyncConfig())
.ethProtocolConfiguration(unstableEthProtocolOptions.toDomainObject())
.networkConfiguration(unstableNetworkingOptions.toDomainObject())
.dataDirectory(dataDir())
.dataStorageConfiguration(getDataStorageConfiguration())
.miningParameters(miningParametersSupplier.get())
.transactionPoolConfiguration(buildTransactionPoolConfiguration())
.nodeKey(new NodeKey(securityModule()))
.metricsSystem((ObservableMetricsSystem) besuComponent.getMetricsSystem())
.messagePermissioningProviders(permissioningService.getMessagePermissioningProviders())
.privacyParameters(privacyParameters())
.clock(Clock.systemUTC())
.isRevertReasonEnabled(isRevertReasonEnabled)
.storageProvider(storageProvider)
.gasLimitCalculator(
miningParametersSupplier.get().getTargetGasLimit().isPresent()
? new FrontierTargetingGasLimitCalculator()
: GasLimitCalculator.constant())
.requiredBlocks(requiredBlocks)
.reorgLoggingThreshold(reorgLoggingThreshold)
.evmConfiguration(unstableEvmOptions.toDomainObject())
.maxPeers(p2PDiscoveryOptions.maxPeers)
.maxRemotelyInitiatedPeers(maxRemoteInitiatedPeers)
.randomPeerPriority(p2PDiscoveryOptions.randomPeerPriority)
.chainPruningConfiguration(unstableChainPruningOptions.toDomainObject())
.cacheLastBlocks(numberOfblocksToCache)
.genesisStateHashCacheEnabled(genesisStateHashCacheEnabled)
.besuComponent(besuComponent);
if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
getDataStorageConfiguration().getDiffBasedSubStorageConfiguration();
if (subStorageConfiguration.getLimitTrieLogsEnabled()) {
besuControllerBuilder.isParallelTxProcessingEnabled(
subStorageConfiguration.getUnstable().isParallelTxProcessingEnabled());
}
}
return besuControllerBuilder;
}

private JsonRpcConfiguration createEngineJsonRpcConfiguration() {
Expand Down Expand Up @@ -2126,29 +2141,34 @@ public DataStorageConfiguration getDataStorageConfiguration() {
}

if (SyncMode.FULL.equals(getDefaultSyncModeIfNotSet())
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())
&& dataStorageConfiguration.getBonsaiLimitTrieLogsEnabled()) {
&& DataStorageFormat.BONSAI.equals(dataStorageConfiguration.getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration diffBasedSubStorageConfiguration =
dataStorageConfiguration.getDiffBasedSubStorageConfiguration();
if (diffBasedSubStorageConfiguration.getLimitTrieLogsEnabled()) {
if (CommandLineUtils.isOptionSet(
commandLine, DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED)) {
throw new ParameterException(
commandLine,
String.format(
"Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode",
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED,
SyncMode.FULL,
DataStorageFormat.BONSAI,
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false"));
}

if (CommandLineUtils.isOptionSet(
commandLine, DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED)) {
throw new ParameterException(
commandLine,
String.format(
"Cannot enable %s with --sync-mode=%s and --data-storage-format=%s. You must set %s or use a different sync-mode",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED,
SyncMode.FULL,
DataStorageFormat.BONSAI,
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false"));
dataStorageConfiguration =
ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration)
.withDiffBasedSubStorageConfiguration(
ImmutableDiffBasedSubStorageConfiguration.copyOf(
dataStorageConfiguration.getDiffBasedSubStorageConfiguration())
.withLimitTrieLogsEnabled(false));
logger.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
DiffBasedSubStorageOptions.LIMIT_TRIE_LOGS_ENABLED + "=false",
SyncMode.FULL,
DataStorageFormat.BONSAI);
}

dataStorageConfiguration =
ImmutableDataStorageConfiguration.copyOf(dataStorageConfiguration)
.withBonsaiLimitTrieLogsEnabled(false);
logger.warn(
"Forcing {}, since it cannot be enabled with --sync-mode={} and --data-storage-format={}.",
DataStorageOptions.BONSAI_LIMIT_TRIE_LOGS_ENABLED + "=false",
SyncMode.FULL,
DataStorageFormat.BONSAI);
}
return dataStorageConfiguration;
}
Expand Down Expand Up @@ -2715,12 +2735,14 @@ private String generateConfigurationOverview() {
builder.setHighSpecEnabled();
}

if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())
&& getDataStorageConfiguration().getBonsaiLimitTrieLogsEnabled()) {
builder.setLimitTrieLogsEnabled();
builder.setTrieLogRetentionLimit(getDataStorageConfiguration().getBonsaiMaxLayersToLoad());
builder.setTrieLogsPruningWindowSize(
getDataStorageConfiguration().getBonsaiTrieLogPruningWindowSize());
if (DataStorageFormat.BONSAI.equals(getDataStorageConfiguration().getDataStorageFormat())) {
final DiffBasedSubStorageConfiguration subStorageConfiguration =
getDataStorageConfiguration().getDiffBasedSubStorageConfiguration();
if (subStorageConfiguration.getLimitTrieLogsEnabled()) {
builder.setLimitTrieLogsEnabled();
builder.setTrieLogRetentionLimit(subStorageConfiguration.getMaxLayersToLoad());
builder.setTrieLogsPruningWindowSize(subStorageConfiguration.getTrieLogPruningWindowSize());
}
}

builder.setSnapServerEnabled(this.unstableSynchronizerOptions.isSnapsyncServerEnabled());
Expand Down
6 changes: 3 additions & 3 deletions besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public class ConfigurationOverviewBuilder {
private Collection<String> engineApis;
private String engineJwtFilePath;
private boolean isHighSpec = false;
private boolean isBonsaiLimitTrieLogsEnabled = false;
private boolean isLimitTrieLogsEnabled = false;
private long trieLogRetentionLimit = 0;
private Integer trieLogsPruningWindowSize = null;
private boolean isSnapServerEnabled = false;
Expand Down Expand Up @@ -220,7 +220,7 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() {
* @return the builder
*/
public ConfigurationOverviewBuilder setLimitTrieLogsEnabled() {
isBonsaiLimitTrieLogsEnabled = true;
isLimitTrieLogsEnabled = true;
return this;
}

Expand Down Expand Up @@ -389,7 +389,7 @@ public String build() {
lines.add("Experimental Snap Sync for BFT enabled");
}

if (isBonsaiLimitTrieLogsEnabled) {
if (isLimitTrieLogsEnabled) {
final StringBuilder trieLogPruningString = new StringBuilder();
trieLogPruningString
.append("Limit trie logs enabled: retention: ")
Expand Down
Loading

0 comments on commit a3b8863

Please sign in to comment.