From 501a3bf10e1beb96d0fa5a505342514fd091d42e Mon Sep 17 00:00:00 2001 From: Lukasz Gasior Date: Tue, 27 Feb 2024 15:31:37 +0100 Subject: [PATCH 1/4] Add vertex store size limit; cleanup vertex store events --- .../java/com/radixdlt/monitoring/Metrics.java | 7 +- .../com/radixdlt/utils/WrappedByteArray.java | 4 + .../DivergentExecutionLivenessBreakTest.java | 356 ++++++++++++++++++ ...PacemakerRoundUpdateRaceConditionTest.java | 4 +- .../java/com/radixdlt/RadixNodeModule.java | 14 + .../consensus/bft/BFTHighQCUpdate.java | 55 +-- .../consensus/bft/BFTInsertUpdate.java | 70 +--- .../consensus/bft/BFTRebuildUpdate.java | 42 +-- .../bft/processor/SyncUpPreprocessor.java | 4 +- .../consensus/epoch/EpochManager.java | 2 +- .../epoch/EpochsConsensusModule.java | 15 +- .../consensus/liveness/Pacemaker.java | 4 +- .../vertexstore/PersistentVertexStore.java | 2 +- .../consensus/vertexstore/VertexStore.java | 35 +- .../vertexstore/VertexStoreAdapter.java | 61 +-- .../VertexStoreConfig.java} | 24 +- .../vertexstore/VertexStoreJavaImpl.java | 331 ++++++++++------ .../vertexstore/VertexStoreState.java | 54 +-- .../radixdlt/ledger/StateComputerLedger.java | 106 +++--- .../radixdlt/modules/DispatcherModule.java | 38 +- .../com/radixdlt/modules/LedgerModule.java | 6 +- .../radixdlt/modules/SystemInfoModule.java | 2 - .../com/radixdlt/rev2/REv2StateComputer.java | 17 +- .../rev2/modules/MockedVertexStoreModule.java | 3 +- .../rev2/modules/REv2StateManagerModule.java | 29 +- .../environment/NoEpochsConsensusModule.java | 28 +- .../deterministic/DeterministicNodes.java | 11 +- .../deterministic/DeterministicTest.java | 11 + .../deterministic/SafetyCheckerModule.java | 4 +- .../invariants/DeterministicMonitors.java | 12 +- .../harness/invariants/SafetyChecker.java | 10 +- .../harness/predicates/NodePredicate.java | 9 + .../monitors/SimulationNodeEventsModule.java | 8 - .../monitors/consensus/ConsensusMonitors.java | 5 +- .../monitors/consensus/LivenessInvariant.java | 9 +- .../consensus/ProposerTimestampChecker.java | 9 +- .../monitors/consensus/SafetyInvariant.java | 7 +- .../monitors/epochs/EpochRoundInvariant.java | 20 +- .../ConsensusToLedgerCommittedInvariant.java | 12 +- .../modules/FunctionalRadixNodeModule.java | 20 +- .../MockedMempoolStateComputer.java | 7 +- .../statecomputer/MockedStateComputer.java | 2 +- .../MockedStateComputerWithEpochs.java | 7 +- .../statecomputer/StatelessComputer.java | 14 +- .../consensus/bft/BFTHighQCUpdateTest.java | 79 ---- .../consensus/bft/BFTInsertUpdateTest.java | 79 ---- .../consensus/epoch/EpochManagerTest.java | 6 +- .../consensus/liveness/PacemakerTest.java | 4 +- .../vertexstore/VertexStoreTest.java | 73 +++- .../radixdlt/modules/ConsensusModuleTest.java | 4 +- docker/config/default.config.envsubst | 2 + 51 files changed, 1004 insertions(+), 733 deletions(-) create mode 100644 core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java rename core/src/main/java/com/radixdlt/consensus/{bft/BFTCommittedUpdate.java => vertexstore/VertexStoreConfig.java} (79%) delete mode 100644 core/src/test/java/com/radixdlt/consensus/bft/BFTHighQCUpdateTest.java delete mode 100644 core/src/test/java/com/radixdlt/consensus/bft/BFTInsertUpdateTest.java diff --git a/common/src/main/java/com/radixdlt/monitoring/Metrics.java b/common/src/main/java/com/radixdlt/monitoring/Metrics.java index 6ef6dba06c..736c10de9c 100644 --- a/common/src/main/java/com/radixdlt/monitoring/Metrics.java +++ b/common/src/main/java/com/radixdlt/monitoring/Metrics.java @@ -233,7 +233,12 @@ public record Sync( Counter invalidEpochInitialQcSyncStates) {} public record VertexStore( - Gauge size, Counter forks, Counter rebuilds, Counter indirectParents) {} + Gauge size, + Gauge byteSize, + Counter forks, + Counter rebuilds, + Counter indirectParents, + Counter errorsDueToSizeLimit) {} public record DivergentVertexExecution(int numDistinctExecutionResults) {} } diff --git a/common/src/main/java/com/radixdlt/utils/WrappedByteArray.java b/common/src/main/java/com/radixdlt/utils/WrappedByteArray.java index 9d1906fb14..f22e916dbb 100644 --- a/common/src/main/java/com/radixdlt/utils/WrappedByteArray.java +++ b/common/src/main/java/com/radixdlt/utils/WrappedByteArray.java @@ -83,6 +83,10 @@ public String toHexString() { return Hex.toHexString(value); } + public int size() { + return value.length; + } + @Override public byte[] hashableBytes() { return value; diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java new file mode 100644 index 0000000000..68490431ee --- /dev/null +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java @@ -0,0 +1,356 @@ +/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). + * + * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this + * file except in compliance with the License. You may obtain a copy of the License at: + * + * radixfoundation.org/licenses/LICENSE-v1 + * + * The Licensor hereby grants permission for the Canonical version of the Work to be + * published, distributed and used under or by reference to the Licensor’s trademark + * Radix ® and use of any unregistered trade names, logos or get-up. + * + * The Licensor provides the Work (and each Contributor provides its Contributions) on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, + * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, + * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. + * + * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create + * a distributed ledger it is your responsibility to test and validate the code, together + * with all logic and performance of that code under all foreseeable scenarios. + * + * The Licensor does not make or purport to make and hereby excludes liability for all + * and any representation, warranty or undertaking in any form whatsoever, whether express + * or implied, to any entity or person, including any representation, warranty or + * undertaking, as to the functionality security use, value or other characteristics of + * any distributed ledger nor in respect the functioning or value of any tokens which may + * be created stored or transferred using the Work. The Licensor does not warrant that the + * Work or any use of the Work complies with any law or regulation in any territory where + * it may be implemented or used or that it will be appropriate for any specific purpose. + * + * Neither the licensor nor any current or former employees, officers, directors, partners, + * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor + * shall be liable for any direct or indirect, special, incidental, consequential or other + * losses of any kind, in tort, contract or otherwise (including but not limited to loss + * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss + * of any economic or other opportunity of whatsoever nature or howsoever arising), arising + * out of or in connection with (without limitation of any use, misuse, of any ledger system + * or use made or its functionality or any performance or operation of any code or protocol + * caused by bugs or programming or logic errors or otherwise); + * + * A. any offer, purchase, holding, use, sale, exchange or transmission of any + * cryptographic keys, tokens or assets created, exchanged, stored or arising from any + * interaction with the Work; + * + * B. any failure in a transmission or loss of any token or assets keys or other digital + * artefacts due to errors in transmission; + * + * C. bugs, hacks, logic errors or faults in the Work or any communication; + * + * D. system software or apparatus including but not limited to losses caused by errors + * in holding or transmitting tokens by any third-party; + * + * E. breaches or failure of security including hacker attacks, loss or disclosure of + * password, loss of private key, unauthorised use or misuse of such passwords or keys; + * + * F. any losses including loss of anticipated savings or other benefits resulting from + * use of the Work or any changes to the Work (however implemented). + * + * You are solely responsible for; testing, validating and evaluation of all operation + * logic, functionality, security and appropriateness of using the Work for any commercial + * or non-commercial purpose and for any reproduction or redistribution by You of the + * Work. You assume all risks associated with Your use of the Work and the exercise of + * permissions under this License. + */ + +package com.radixdlt.integration.steady_state.deterministic.consensus; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.inject.*; +import com.google.inject.Module; +import com.radixdlt.consensus.LedgerHashes; +import com.radixdlt.consensus.NextEpoch; +import com.radixdlt.consensus.bft.Round; +import com.radixdlt.consensus.vertexstore.ExecutedVertex; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; +import com.radixdlt.crypto.HashUtils; +import com.radixdlt.environment.deterministic.network.MessageSelector; +import com.radixdlt.genesis.GenesisBuilder; +import com.radixdlt.genesis.GenesisConsensusManagerConfig; +import com.radixdlt.harness.deterministic.DeterministicTest; +import com.radixdlt.harness.deterministic.PhysicalNodeConfig; +import com.radixdlt.harness.predicates.NodePredicate; +import com.radixdlt.harness.predicates.NodesPredicate; +import com.radixdlt.lang.Option; +import com.radixdlt.ledger.*; +import com.radixdlt.mempool.MempoolAdd; +import com.radixdlt.modules.FunctionalRadixNodeModule; +import com.radixdlt.modules.FunctionalRadixNodeModule.ConsensusConfig; +import com.radixdlt.modules.FunctionalRadixNodeModule.NodeStorageConfig; +import com.radixdlt.modules.StateComputerConfig; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.networks.Network; +import com.radixdlt.p2p.NodeId; +import com.radixdlt.rev2.Decimal; +import com.radixdlt.rev2.REV2TransactionGenerator; +import com.radixdlt.rev2.REv2StateComputer; +import com.radixdlt.rev2.REv2TransactionsAndProofReader; +import com.radixdlt.transactions.RawNotarizedTransaction; +import java.util.List; +import java.util.Random; +import java.util.function.Consumer; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +// spotless:off +/** + * Test a scenario when there's a prolonged liveness break (for the purpose of this test caused by a + * forced/faked divergent vertex execution). We verify that: + * 1) The situation is kept under control, vertex store size-wise. I.e. that it doesn't grow uncontrollably, + * which would ultimately result in node crash due to SBOR overflow. + * 2) Liveness can be restored by: + * a) fixing the underlying issue (here: reverting the mocked divergence) and + * b) increasing the vertex store size limit (so that it can accept a few more vertices, + * while staying below the critical SBOR limit) + * + * The test consists of 3 phases: + * - Phase 1 (rounds 0 to LIVENESS_BREAK_START_ROUND): initial liveness (to verify that the test setup is correct) + * - Phase 2 (from LIVENESS_BREAK_START_ROUND): liveness break due to divergent vertex execution (different hashes) + * - Phase 3 Recovery. The nodes reboot with a new configuration (slightly increased vertex store size limit) +* and the forced divergent vertex execution in `prepare` is reverted. + */ +// spotless:on +public final class DivergentExecutionLivenessBreakTest { + private static final Random random = new Random(123456); + + private static final int NUM_VALIDATORS = 4; + + // The test starts with 30 regular rounds, as a sanity check for our test setup + private static final Round LIVENESS_BREAK_START_ROUND = Round.of(30); + + // Vertex store config used in the first two phases of the test: initial liveness followed by a + // liveness break + private static final VertexStoreConfig INITIAL_VERTEX_STORE_CONFIG = + new VertexStoreConfig(100_000 /* 100 KiB */); + private static final ConsensusConfig INITIAL_CONSENSUS_CONFIG = + new ConsensusConfig(1000, 200L, 2.0, 0L, 0L, INITIAL_VERTEX_STORE_CONFIG); + + // Vertex store config used in the third phase of the test: liveness "fix" and recovery + private static final VertexStoreConfig RECOVERY_VERTEX_STORE_CONFIG = + new VertexStoreConfig(150_000 /* 150 KiB */); + private static final ConsensusConfig RECOVERY_CONSENSUS_CONFIG = + new ConsensusConfig(1000, 200L, 2.0, 0L, 0L, RECOVERY_VERTEX_STORE_CONFIG); + + // A module that overrides StateComputer in the first and second phase of a test: + // It runs normally until LIVENESS_BREAK_START_ROUND, at which point it switches + // its implementation to produce a liveness break by altering the resultant ledger hashes in + // `prepare`. + private static final Module INITIAL_OVERRIDE_MODULE = + new AbstractModule() { + @Provides + @Singleton + private StateComputerLedger.StateComputer stateComputer( + REv2StateComputer underlyingStateComputer) { + return new StateComputerLedger.StateComputer() { + @Override + public void addToMempool(MempoolAdd mempoolAdd, NodeId origin) { + underlyingStateComputer.addToMempool(mempoolAdd, origin); + } + + @Override + public List getTransactionsForProposal( + List executedTransactions) { + return underlyingStateComputer.getTransactionsForProposal(executedTransactions); + } + + @Override + public StateComputerLedger.StateComputerPrepareResult prepare( + LedgerHashes committedLedgerHashes, + List preparedUncommittedVertices, + LedgerHashes preparedUncommittedLedgerHashes, + List proposedTransactions, + RoundDetails roundDetails) { + final var baseResult = + underlyingStateComputer.prepare( + committedLedgerHashes, + preparedUncommittedVertices, + preparedUncommittedLedgerHashes, + proposedTransactions, + roundDetails); + + if (roundDetails.roundNumber() < LIVENESS_BREAK_START_ROUND.number()) { + return baseResult; + } else { + return new StateComputerLedger.StateComputerPrepareResult( + baseResult.getSuccessfullyExecutedTransactions(), + baseResult.getRejectedTransactionCount(), + baseResult.getNextEpoch().or((NextEpoch) null), + baseResult.getNextProtocolVersion().or((String) null), + // Random hashes to produce a liveness break + LedgerHashes.create( + HashUtils.random256(), HashUtils.random256(), HashUtils.random256())); + } + } + + @Override + public LedgerProofBundle commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + return underlyingStateComputer.commit(ledgerExtension, serializedVertexStoreState); + } + }; + } + }; + + // A module used in the third phase of the test that: + // - reverts StateComputer override from INITIAL_OVERRIDE_MODULE (bringing back liveness) + // - updates ConsensusConfig to RECOVERY_CONSENSUS_CONFIG (which increases vertex store size + // limit) + private static final Module RECOVERY_OVERRIDE_MODULE = + new AbstractModule() { + @Override + public void configure() { + install(RECOVERY_CONSENSUS_CONFIG.asModule()); + } + + @Provides + @Singleton + private StateComputerLedger.StateComputer stateComputer(REv2StateComputer underlying) { + return underlying; + } + }; + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private DeterministicTest createTest() { + return DeterministicTest.builder() + .addPhysicalNodes(PhysicalNodeConfig.createBatch(NUM_VALIDATORS, true)) + .messageSelector(MessageSelector.randomSelector(random)) + .overrideWithIncorrectModule(INITIAL_OVERRIDE_MODULE) + .functionalNodeModule( + new FunctionalRadixNodeModule( + NodeStorageConfig.tempFolder(folder), + true, + FunctionalRadixNodeModule.SafetyRecoveryConfig.BERKELEY_DB, + INITIAL_CONSENSUS_CONFIG, + FunctionalRadixNodeModule.LedgerConfig.stateComputerNoSync( + StateComputerConfig.rev2( + Network.INTEGRATIONTESTNET.getId(), + GenesisBuilder.createTestGenesisWithNumValidators( + NUM_VALIDATORS, + Decimal.ONE, + GenesisConsensusManagerConfig.Builder.testWithRoundsPerEpoch(100000)), + StateComputerConfig.REV2ProposerConfig.transactionGenerator( + new REV2TransactionGenerator(), 1))))); + } + + @Test + public void test_divergent_execution_liveness_break_and_recovery() { + try (final var test = createTest()) { + // Phase 1: Initial liveness + test.startAllNodes(); + test.runUntilState( + NodesPredicate.allNodesMatch( + NodePredicate.atOrOverRound(LIVENESS_BREAK_START_ROUND.previous()))); + + verifyMetricsOnAllNodes( + test, + metrics -> { + // No divergent executions + assertEquals(0, (int) metrics.bft().divergentVertexExecutions().getSum()); + // No timeouts quorums + assertEquals( + 0, + (int) + metrics + .bft() + .quorumResolutions() + .label(new Metrics.Bft.QuorumResolution(true)) + .get()); + }); + + // Phase 2: Liveness break + // Run until we observe that vertex store hits its size limit + // 100 occurrences is chosen arbitrarily, just to make sure that the issue is not transient + test.runUntilState( + NodesPredicate.allNodesMatch( + NodePredicate.metricsPredicate( + metrics -> metrics.bft().vertexStore().errorsDueToSizeLimit().get() > 100))); + + verifyMetricsOnAllNodes( + test, + metrics -> { + // Verify that the cause is what we expect: a divergent execution + assertTrue(metrics.bft().divergentVertexExecutions().getSum() > 1); + // Cross-check another metric to verify that vertex store + // indeed holds more vertices than expected in a healthy scenario. + assertTrue(metrics.bft().vertexStore().size().get() >= 20); + }); + + // Another verification that we're in a liveness break + final var stateVersionA = + test.getInstance(0, REv2TransactionsAndProofReader.class) + .getLatestProofBundle() + .orElseThrow() + .primaryProof() + .stateVersion(); + test.runForCount(1000); + final var stateVersionB = + test.getInstance(0, REv2TransactionsAndProofReader.class) + .getLatestProofBundle() + .orElseThrow() + .primaryProof() + .stateVersion(); + assertEquals(stateVersionA, stateVersionB); + + // Phase 3: Recovery + for (var i = 0; i < test.numNodes(); i++) { + // Restart all nodes with recovery settings (provided via RECOVERY_OVERRIDE_MODULE) + test.restartNodeWithOverrideModule(i, RECOVERY_OVERRIDE_MODULE); + } + + // Verify liveness: run until each node resolves at least 20 (chosen arbitrarily) non-timeout + // quorums + // Note: nodes start with fresh metrics post-restart + test.runUntilState( + NodesPredicate.allNodesMatch( + NodePredicate.metricsPredicate( + metrics -> + metrics + .bft() + .quorumResolutions() + .label(new Metrics.Bft.QuorumResolution(false)) + .get() + > 20))); + + verifyMetricsOnAllNodes( + test, + metrics -> { + // Assert that vertex store is well below its max size + assertTrue( + (int) metrics.bft().vertexStore().byteSize().get() + < INITIAL_VERTEX_STORE_CONFIG.maxSerializedSizeBytes() / 2); + // No more errors due to size limit + assertEquals(0, (int) metrics.bft().vertexStore().errorsDueToSizeLimit().get()); + // There are no more than 3 vertices (as expected in a healthy network) + assertTrue(metrics.bft().vertexStore().size().get() <= 3); + // No timeouts + assertEquals( + 0, + (int) + metrics + .bft() + .quorumResolutions() + .label(new Metrics.Bft.QuorumResolution(true)) + .get()); + }); + } + } + + private void verifyMetricsOnAllNodes(DeterministicTest test, Consumer testFn) { + for (final var i : test.getNodeInjectors()) { + testFn.accept(i.getInstance(Metrics.class)); + } + } +} diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/PacemakerRoundUpdateRaceConditionTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/PacemakerRoundUpdateRaceConditionTest.java index 47078e1fc1..e79e723581 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/PacemakerRoundUpdateRaceConditionTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/PacemakerRoundUpdateRaceConditionTest.java @@ -126,7 +126,7 @@ public void test_pacemaker_round_update_race_condition() { private EventProcessor bftInsertUpdateProcessor() { final Map insertedVertices = new HashMap<>(); return bftInsertUpdate -> { - final ExecutedVertex inserted = bftInsertUpdate.getInserted(); + final ExecutedVertex inserted = bftInsertUpdate.insertedVertex(); insertedVertices.putIfAbsent(inserted.getVertexHash(), inserted); final Optional maybeParent = Optional.ofNullable(insertedVertices.get(inserted.getParentId())); @@ -204,7 +204,7 @@ private static MessageMutator messUpMessagesForNodeUnderTest() { queue.add(message.withAdditionalDelay(additionalMessageDelay)); return true; } else if (msg instanceof BFTInsertUpdate - && ((BFTInsertUpdate) msg).getInserted().getRound().equals(Round.of(1))) { + && ((BFTInsertUpdate) msg).insertedVertex().getRound().equals(Round.of(1))) { queue.add(message.withAdditionalDelay(additionalMessageDelay)); return true; } else { diff --git a/core/src/main/java/com/radixdlt/RadixNodeModule.java b/core/src/main/java/com/radixdlt/RadixNodeModule.java index 5191971c32..db1ebfb404 100644 --- a/core/src/main/java/com/radixdlt/RadixNodeModule.java +++ b/core/src/main/java/com/radixdlt/RadixNodeModule.java @@ -75,6 +75,7 @@ import com.radixdlt.consensus.bft.*; import com.radixdlt.consensus.epoch.EpochsConsensusModule; import com.radixdlt.consensus.sync.BFTSyncPatienceMillis; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.environment.*; import com.radixdlt.environment.rx.RxEnvironmentModule; import com.radixdlt.genesis.GenesisProvider; @@ -156,6 +157,19 @@ protected void configure() { bindConstant().annotatedWith(AdditionalRoundTimeIfProposalReceivedMs.class).to(30_000L); bindConstant().annotatedWith(TimeoutQuorumResolutionDelayMs.class).to(0L); + final var vertexStoreConfig = + new VertexStoreConfig( + properties.get( + "bft.vertex_store.max_serialized_size_bytes", + VertexStoreConfig.DEFAULT_MAX_SERIALIZED_SIZE_BYTES)); + bind(VertexStoreConfig.class).toInstance(vertexStoreConfig); + + Preconditions.checkArgument( + vertexStoreConfig.maxSerializedSizeBytes() + >= VertexStoreConfig.MIN_MAX_SERIALIZED_SIZE_BYTES, + "Invalid configuration: bft.vertex_store.max_serialized_size_byte must be at least " + + VertexStoreConfig.MIN_MAX_SERIALIZED_SIZE_BYTES); + // System (e.g. time, random) install(new SystemModule()); diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java index c3d560146d..1029634a6e 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java @@ -64,48 +64,17 @@ package com.radixdlt.consensus.bft; +import com.google.common.collect.ImmutableList; import com.radixdlt.consensus.HighQC; -import com.radixdlt.consensus.vertexstore.VertexStoreState; -import java.util.Objects; +import com.radixdlt.consensus.vertexstore.ExecutedVertex; +import com.radixdlt.lang.Option; +import com.radixdlt.utils.WrappedByteArray; -/** An event emitted when the high qc has been updated */ -public final class BFTHighQCUpdate { - private final VertexStoreState vertexStoreState; - - private BFTHighQCUpdate(VertexStoreState vertexStoreState) { - this.vertexStoreState = vertexStoreState; - } - - public static BFTHighQCUpdate create(VertexStoreState vertexStoreState) { - return new BFTHighQCUpdate(vertexStoreState); - } - - public HighQC getHighQC() { - return vertexStoreState.getHighQC(); - } - - public VertexStoreState getVertexStoreState() { - return vertexStoreState; - } - - @Override - public String toString() { - return String.format( - "%s{highQC=%s}", this.getClass().getSimpleName(), vertexStoreState.getHighQC()); - } - - @Override - public int hashCode() { - return Objects.hash(vertexStoreState); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof BFTHighQCUpdate)) { - return false; - } - - BFTHighQCUpdate other = (BFTHighQCUpdate) o; - return Objects.equals(other.vertexStoreState, this.vertexStoreState); - } -} +/** + * An event emitted when vertex store updates its highQC, which possibly results in some vertices + * being committed. + */ +public record BFTHighQCUpdate( + HighQC newHighQc, + Option> committedVertices, + WrappedByteArray serializedVertexStoreState) {} diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java index 352a9aef68..69ee285669 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java @@ -64,71 +64,9 @@ package com.radixdlt.consensus.bft; -import com.radixdlt.consensus.BFTHeader; import com.radixdlt.consensus.vertexstore.ExecutedVertex; -import com.radixdlt.consensus.vertexstore.VertexStoreState; -import java.util.Objects; +import com.radixdlt.utils.WrappedByteArray; -/** An update emitted when the BFT has inserted a new vertex */ -public final class BFTInsertUpdate { - private final VertexStoreState vertexStoreState; - private final ExecutedVertex insertedVertex; - private final int siblingsCount; - - private BFTInsertUpdate( - ExecutedVertex insertedVertex, int siblingsCount, VertexStoreState vertexStoreState) { - this.insertedVertex = Objects.requireNonNull(insertedVertex); - this.siblingsCount = siblingsCount; - this.vertexStoreState = Objects.requireNonNull(vertexStoreState); - } - - public static BFTInsertUpdate insertedVertex( - ExecutedVertex insertedVertex, int siblingsCount, VertexStoreState vertexStoreState) { - return new BFTInsertUpdate(insertedVertex, siblingsCount, vertexStoreState); - } - - public VertexStoreState getVertexStoreState() { - return vertexStoreState; - } - - public int getSiblingsCount() { - return siblingsCount; - } - - public int getVertexStoreSize() { - return vertexStoreState.getVertices().size(); - } - - public BFTHeader getHeader() { - return new BFTHeader( - insertedVertex.getRound(), - insertedVertex.getVertexHash(), - insertedVertex.getLedgerHeader()); - } - - public ExecutedVertex getInserted() { - return insertedVertex; - } - - @Override - public int hashCode() { - return Objects.hash(vertexStoreState, insertedVertex, siblingsCount); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof BFTInsertUpdate)) { - return false; - } - - BFTInsertUpdate other = (BFTInsertUpdate) o; - return Objects.equals(this.vertexStoreState, other.vertexStoreState) - && Objects.equals(this.insertedVertex, other.insertedVertex) - && this.siblingsCount == other.siblingsCount; - } - - @Override - public String toString() { - return String.format("%s{inserted=%s}", getClass().getSimpleName(), insertedVertex); - } -} +/** An event emitted after a vertex has been inserted into the vertex store. */ +public record BFTInsertUpdate( + ExecutedVertex insertedVertex, WrappedByteArray serializedVertexStoreState) {} diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java index 1e78fc8e98..5d8b735299 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java @@ -65,42 +65,8 @@ package com.radixdlt.consensus.bft; import com.radixdlt.consensus.vertexstore.VertexStoreState; -import java.util.Objects; +import com.radixdlt.utils.WrappedByteArray; -/** An update emitted when the BFT has been rebuilt */ -public final class BFTRebuildUpdate { - private final VertexStoreState vertexStoreState; - - private BFTRebuildUpdate(VertexStoreState vertexStoreState) { - this.vertexStoreState = vertexStoreState; - } - - public static BFTRebuildUpdate create(VertexStoreState vertexStoreState) { - return new BFTRebuildUpdate(vertexStoreState); - } - - public VertexStoreState getVertexStoreState() { - return vertexStoreState; - } - - @Override - public String toString() { - return String.format( - "%s{root=%s}", this.getClass().getSimpleName(), vertexStoreState.getRoot()); - } - - @Override - public int hashCode() { - return Objects.hash(vertexStoreState); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof BFTRebuildUpdate)) { - return false; - } - - BFTRebuildUpdate other = (BFTRebuildUpdate) o; - return Objects.equals(other.vertexStoreState, this.vertexStoreState); - } -} +/** An even emitted when the vertex store has been rebuilt. */ +public record BFTRebuildUpdate( + VertexStoreState vertexStoreState, WrappedByteArray serializedVertexStoreState) {} diff --git a/core/src/main/java/com/radixdlt/consensus/bft/processor/SyncUpPreprocessor.java b/core/src/main/java/com/radixdlt/consensus/bft/processor/SyncUpPreprocessor.java index 5c3025c0d2..40ecd46ac2 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/processor/SyncUpPreprocessor.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/processor/SyncUpPreprocessor.java @@ -150,7 +150,7 @@ private void processRoundCachedEvent(QueuedConsensusEvent queuedEvent) { @Override public void processBFTUpdate(BFTInsertUpdate update) { - final var vertexId = update.getInserted().getVertexHash(); + final var vertexId = update.insertedVertex().getVertexHash(); log.trace("LOCAL_SYNC: {}", vertexId); syncingEvents.stream() @@ -167,7 +167,7 @@ public void processBFTUpdate(BFTInsertUpdate update) { @Override public void processBFTRebuildUpdate(BFTRebuildUpdate rebuildUpdate) { rebuildUpdate - .getVertexStoreState() + .vertexStoreState() .getVertices() .forEach( v -> { diff --git a/core/src/main/java/com/radixdlt/consensus/epoch/EpochManager.java b/core/src/main/java/com/radixdlt/consensus/epoch/EpochManager.java index 6f04dc5be3..034ef101a0 100644 --- a/core/src/main/java/com/radixdlt/consensus/epoch/EpochManager.java +++ b/core/src/main/java/com/radixdlt/consensus/epoch/EpochManager.java @@ -483,7 +483,7 @@ public void processBFTUpdate(BFTInsertUpdate update) { public EventProcessor bftRebuildUpdateEventProcessor() { return update -> { if (update - .getVertexStoreState() + .vertexStoreState() .getRoot() .vertex() .getParentHeader() diff --git a/core/src/main/java/com/radixdlt/consensus/epoch/EpochsConsensusModule.java b/core/src/main/java/com/radixdlt/consensus/epoch/EpochsConsensusModule.java index 0196b3f640..273f2e2fd2 100644 --- a/core/src/main/java/com/radixdlt/consensus/epoch/EpochsConsensusModule.java +++ b/core/src/main/java/com/radixdlt/consensus/epoch/EpochsConsensusModule.java @@ -76,6 +76,7 @@ import com.radixdlt.consensus.liveness.*; import com.radixdlt.consensus.sync.*; import com.radixdlt.consensus.vertexstore.VertexStoreAdapter; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.consensus.vertexstore.VertexStoreJavaImpl; import com.radixdlt.crypto.Hasher; import com.radixdlt.environment.*; @@ -84,6 +85,7 @@ import com.radixdlt.messaging.core.GetVerticesRequestRateLimit; import com.radixdlt.monitoring.Metrics; import com.radixdlt.p2p.NodeId; +import com.radixdlt.serialization.Serialization; import com.radixdlt.sync.messages.local.LocalSyncRequest; import com.radixdlt.utils.TimeSupplier; import java.util.Comparator; @@ -107,7 +109,6 @@ protected void configure() { eventBinder.addBinding().toInstance(BFTRebuildUpdate.class); eventBinder.addBinding().toInstance(BFTInsertUpdate.class); eventBinder.addBinding().toInstance(BFTHighQCUpdate.class); - eventBinder.addBinding().toInstance(BFTCommittedUpdate.class); eventBinder.addBinding().toInstance(Proposal.class); eventBinder.addBinding().toInstance(Vote.class); eventBinder.addBinding().toInstance(LedgerUpdate.class); @@ -458,15 +459,17 @@ private VertexStoreFactory vertexStoreFactory( EventDispatcher updateSender, EventDispatcher rebuildUpdateDispatcher, EventDispatcher highQCUpdateEventDispatcher, - EventDispatcher committedDispatcher, Ledger ledger, - Hasher hasher) { + Hasher hasher, + Serialization serialization, + Metrics metrics, + VertexStoreConfig vertexStoreConfig) { return vertexStoreState -> new VertexStoreAdapter( - VertexStoreJavaImpl.create(vertexStoreState, ledger, hasher), + new VertexStoreJavaImpl( + ledger, hasher, serialization, metrics, vertexStoreConfig, vertexStoreState), highQCUpdateEventDispatcher, updateSender, - rebuildUpdateDispatcher, - committedDispatcher); + rebuildUpdateDispatcher); } } diff --git a/core/src/main/java/com/radixdlt/consensus/liveness/Pacemaker.java b/core/src/main/java/com/radixdlt/consensus/liveness/Pacemaker.java index 42f1dac351..a32f4c654d 100644 --- a/core/src/main/java/com/radixdlt/consensus/liveness/Pacemaker.java +++ b/core/src/main/java/com/radixdlt/consensus/liveness/Pacemaker.java @@ -230,8 +230,8 @@ public void processProposal(Proposal proposal) { public void processBFTUpdate(BFTInsertUpdate update) { log.trace("BFTUpdate: Processing {}", update); - final var round = update.getHeader().getRound(); - final var vertex = update.getInserted(); + final var round = update.insertedVertex().getRound(); + final var vertex = update.insertedVertex(); if (round.equals(currentRound())) { // A vertex for the current round has been inserted diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/PersistentVertexStore.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/PersistentVertexStore.java index 6da06a4eea..66a2c52031 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/PersistentVertexStore.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/PersistentVertexStore.java @@ -69,5 +69,5 @@ * RadixEngine ((RPNV1-718) */ public interface PersistentVertexStore { - void save(VertexStoreState vertexStoreState); + void save(byte[] serializedVertexStoreState); } diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStore.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStore.java index 450f283141..b26a390966 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStore.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStore.java @@ -66,23 +66,21 @@ import com.google.common.collect.ImmutableList; import com.google.common.hash.HashCode; -import com.radixdlt.consensus.HighQC; -import com.radixdlt.consensus.QuorumCertificate; -import com.radixdlt.consensus.TimeoutCertificate; -import com.radixdlt.consensus.VertexWithHash; +import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.BFTInsertUpdate; import com.radixdlt.lang.Option; +import com.radixdlt.lang.Result; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; /** Manages the BFT Vertex chain. TODO: Move this logic into ledger package. */ public interface VertexStore { - record CommittedUpdate(ImmutableList committedVertices) {} + record CommittedUpdate(ImmutableList committedVertices, HighQC newHighQc) {} sealed interface InsertQcResult { record Inserted( HighQC newHighQc, - // TODO: remove me once vertex store persistence and commit on the java side are gone - VertexStoreState vertexStoreState, + WrappedByteArray serializedVertexStoreState, Option committedUpdate) implements InsertQcResult {} @@ -91,19 +89,32 @@ record Ignored() implements InsertQcResult {} record VertexIsMissing() implements InsertQcResult {} } + sealed interface InsertTcResult { + record Inserted(HighQC newHighQc, WrappedByteArray serializedVertexStoreState) + implements InsertTcResult {} + + record Ignored() implements InsertTcResult {} + } + record InsertVertexChainResult( List insertedQcs, List insertUpdates) {} + record RebuildSummary( + VertexStoreState resultantState, WrappedByteArray serializedVertexStoreState) {} + + enum RebuildError { + VERTEX_STORE_SIZE_EXCEEDED, + VERTEX_EXECUTION_ERROR + } + InsertQcResult insertQc(QuorumCertificate qc); /** * Inserts a timeout certificate into the store. * * @param timeoutCertificate the timeout certificate - * @return true if the timeout certificate was inserted, false if it was ignored because it's not - * the highest */ - boolean insertTimeoutCertificate(TimeoutCertificate timeoutCertificate); + InsertTcResult insertTimeoutCertificate(TimeoutCertificate timeoutCertificate); /** * Inserts a vertex and then attempts to create the next header. @@ -114,7 +125,7 @@ record InsertVertexChainResult( InsertVertexChainResult insertVertexChain(VertexChain vertexChain); - Option tryRebuild(VertexStoreState vertexStoreState); + Result tryRebuild(VertexStoreState vertexStoreState); boolean containsVertex(HashCode vertexId); @@ -143,4 +154,6 @@ record InsertVertexChainResult( * @return the list of vertices if all found, otherwise an empty list */ Option> getVertices(HashCode vertexHash, int count); + + int getCurrentSerializedSizeBytes(); } diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java index 9397b1acb0..e1a3482ab7 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java @@ -71,11 +71,11 @@ import com.radixdlt.consensus.QuorumCertificate; import com.radixdlt.consensus.TimeoutCertificate; import com.radixdlt.consensus.VertexWithHash; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.BFTInsertUpdate; import com.radixdlt.consensus.bft.BFTRebuildUpdate; import com.radixdlt.environment.EventDispatcher; +import com.radixdlt.lang.Option; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -92,34 +92,45 @@ public final class VertexStoreAdapter { private final EventDispatcher highQCUpdateDispatcher; private final EventDispatcher bftUpdateDispatcher; private final EventDispatcher bftRebuildDispatcher; - private final EventDispatcher bftCommittedDispatcher; @Inject public VertexStoreAdapter( VertexStore vertexStore, EventDispatcher highQCUpdateDispatcher, EventDispatcher bftUpdateDispatcher, - EventDispatcher bftRebuildDispatcher, - EventDispatcher bftCommittedDispatcher) { + EventDispatcher bftRebuildDispatcher) { this.vertexStore = Objects.requireNonNull(vertexStore); this.highQCUpdateDispatcher = Objects.requireNonNull(highQCUpdateDispatcher); this.bftUpdateDispatcher = Objects.requireNonNull(bftUpdateDispatcher); this.bftRebuildDispatcher = Objects.requireNonNull(bftRebuildDispatcher); - this.bftCommittedDispatcher = Objects.requireNonNull(bftCommittedDispatcher); } public boolean tryRebuild(VertexStoreState vertexStoreState) { final var result = vertexStore.tryRebuild(vertexStoreState); - result.onPresent( - newVertexStoreState -> - bftRebuildDispatcher.dispatch(BFTRebuildUpdate.create(newVertexStoreState))); + result.onSuccess( + rebuildSummary -> + bftRebuildDispatcher.dispatch( + new BFTRebuildUpdate( + rebuildSummary.resultantState(), rebuildSummary.serializedVertexStoreState()))); - return result.isPresent(); + return result.isSuccess(); } public boolean insertTimeoutCertificate(TimeoutCertificate timeoutCertificate) { - return vertexStore.insertTimeoutCertificate(timeoutCertificate); + final var result = vertexStore.insertTimeoutCertificate(timeoutCertificate); + + return switch (result) { + case VertexStore.InsertTcResult.Inserted inserted -> { + highQCUpdateDispatcher.dispatch( + new BFTHighQCUpdate( + inserted.newHighQc(), + Option.empty() /* TC can't commit anything */, + inserted.serializedVertexStoreState())); + yield true; + } + case VertexStore.InsertTcResult.Ignored ignored -> false; // No-op + }; } public VertexStore.InsertQcResult insertQc(QuorumCertificate qc) { @@ -145,33 +156,21 @@ public void insertVertexChain(VertexChain vertexChain) { } private void dispatchPostQcInsertionEvents(VertexStore.InsertQcResult.Inserted inserted) { - // This is a bit of an abstraction break. - // We don't want to persist the vertex store state (via PersistentVertexStore) - // if the state is already being persisted alongside commit. - // This implicitly assumes that: - // - BFTHighQCUpdate triggers vertex store state persistence - // (that's why this if statement is needed) - // - no other component needs this event if we're also committing - if (inserted.committedUpdate().isEmpty()) { - this.highQCUpdateDispatcher.dispatch(BFTHighQCUpdate.create(inserted.vertexStoreState())); - } - - inserted - .committedUpdate() - .onPresent( - committedUpdate -> - this.bftCommittedDispatcher.dispatch( - new BFTCommittedUpdate( - committedUpdate.committedVertices(), inserted.vertexStoreState()))); + this.highQCUpdateDispatcher.dispatch( + new BFTHighQCUpdate( + inserted.newHighQc(), + inserted.committedUpdate().map(VertexStore.CommittedUpdate::committedVertices), + inserted.serializedVertexStoreState())); } public Optional> getVertices(HashCode vertexId, int count) { return vertexStore.getVertices(vertexId, count).toOptional(); } - public void insertVertex(VertexWithHash vertex) { + public boolean insertVertex(VertexWithHash vertex) { final var result = vertexStore.insertVertex(vertex); result.onPresent(bftUpdateDispatcher::dispatch); + return result.isPresent(); } public boolean containsVertex(HashCode vertexId) { @@ -194,4 +193,8 @@ public HighQC highQC() { public VertexWithHash getRoot() { return vertexStore.getRoot(); } + + public int getCurrentSerializedSizeBytes() { + return vertexStore.getCurrentSerializedSizeBytes(); + } } diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTCommittedUpdate.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreConfig.java similarity index 79% rename from core/src/main/java/com/radixdlt/consensus/bft/BFTCommittedUpdate.java rename to core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreConfig.java index 4de497cc45..74ee05edc2 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTCommittedUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreConfig.java @@ -62,16 +62,22 @@ * permissions under this License. */ -package com.radixdlt.consensus.bft; +package com.radixdlt.consensus.vertexstore; -import com.google.common.collect.ImmutableList; -import com.radixdlt.consensus.vertexstore.ExecutedVertex; -import com.radixdlt.consensus.vertexstore.VertexStoreState; +public record VertexStoreConfig(int maxSerializedSizeBytes) { + /* This limit is mostly to prevent the node from crashing when we're + SBOR-encoding the vertex store (so that it can be persisted on the Rust side). + It could grow too large in a scenario of a prolonged liveness break. + The default limit is based on the SBOR limit of 0x0FFFFFFF (268_435_455). + 150 MB is a reasonable default: it's more than enough for regular node operation + and at the same time it leaves enough room for intervention (additional ~100 MB) + in case we run into issues. + The default limit can be changed with a configuration parameter (bft.vertex_store.max_serialized_size_bytes). */ + public static final int DEFAULT_MAX_SERIALIZED_SIZE_BYTES = 150 * 1024 * 1024 /* 150 MB */; -/** Vertex Store update of committed vertices */ -public record BFTCommittedUpdate( - ImmutableList committed, VertexStoreState vertexStoreState) { - public int vertexStoreSize() { - return vertexStoreState.getVertices().size(); + public static final int MIN_MAX_SERIALIZED_SIZE_BYTES = 10 * 1024 * 1024 /* 10 MB */; + + public static VertexStoreConfig testingDefault() { + return new VertexStoreConfig(DEFAULT_MAX_SERIALIZED_SIZE_BYTES); } } diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java index 235561418d..cd148baf9b 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java @@ -77,13 +77,12 @@ import com.radixdlt.consensus.bft.MissingParentException; import com.radixdlt.crypto.Hasher; import com.radixdlt.lang.Option; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Map; -import java.util.Objects; -import java.util.Set; +import com.radixdlt.lang.Result; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.serialization.DsonOutput; +import com.radixdlt.serialization.Serialization; +import com.radixdlt.utils.WrappedByteArray; +import java.util.*; import javax.annotation.concurrent.NotThreadSafe; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -95,6 +94,9 @@ public final class VertexStoreJavaImpl implements VertexStore { private final Hasher hasher; private final Ledger ledger; + private final Serialization serialization; + private final Metrics metrics; + private final VertexStoreConfig config; private final Map vertices = new HashMap<>(); private final Map> vertexChildren = new HashMap<>(); @@ -103,39 +105,56 @@ public final class VertexStoreJavaImpl implements VertexStore { // These should never be null private VertexWithHash rootVertex; private HighQC highQC; - - private VertexStoreJavaImpl( - Ledger ledger, Hasher hasher, VertexWithHash rootVertex, HighQC highQC) { + private int currentSerializedSizeBytes; + + public VertexStoreJavaImpl( + Ledger ledger, + Hasher hasher, + Serialization serialization, + Metrics metrics, + VertexStoreConfig config, + VertexStoreState initialState) { this.ledger = Objects.requireNonNull(ledger); this.hasher = Objects.requireNonNull(hasher); - this.rootVertex = Objects.requireNonNull(rootVertex); - this.highQC = Objects.requireNonNull(highQC); - this.vertexChildren.put(rootVertex.hash(), new HashSet<>()); + this.serialization = Objects.requireNonNull(serialization); + this.metrics = Objects.requireNonNull(metrics); + this.config = Objects.requireNonNull(config); + + resetToState(initialState, serializeState(initialState)); + } + + public int getCurrentSerializedSizeBytes() { + return currentSerializedSizeBytes; } - public static VertexStoreJavaImpl create( - VertexStoreState vertexStoreState, Ledger ledger, Hasher hasher) { - final var vertexStore = - new VertexStoreJavaImpl( - ledger, hasher, vertexStoreState.getRoot(), vertexStoreState.getHighQC()); + private void resetToState(VertexStoreState state, WrappedByteArray serializedState) { + this.rootVertex = state.getRoot(); + this.highQC = state.getHighQC(); + this.vertices.clear(); + this.executedVertices.clear(); + this.vertexChildren.clear(); + this.vertexChildren.put(rootVertex.hash(), new HashSet<>()); - for (var vertexWithHash : vertexStoreState.getVertices()) { - vertexStore.vertices.put(vertexWithHash.hash(), vertexWithHash); - vertexStore.vertexChildren.put(vertexWithHash.hash(), new HashSet<>()); - var siblings = vertexStore.vertexChildren.get(vertexWithHash.vertex().getParentVertexId()); + for (var vertexWithHash : state.getVertices()) { + this.vertices.put(vertexWithHash.hash(), vertexWithHash); + this.vertexChildren.put(vertexWithHash.hash(), new HashSet<>()); + var siblings = this.vertexChildren.get(vertexWithHash.vertex().getParentVertexId()); siblings.add(vertexWithHash.hash()); } - return vertexStore; + trackCurrentStateSize(serializedState); } @Override - public VertexWithHash getRoot() { - return rootVertex; - } + public Result tryRebuild(VertexStoreState vertexStoreState) { + final var serializedVertexStoreState = serializeState(vertexStoreState); + + if (serializedVertexStoreState.size() > config.maxSerializedSizeBytes()) { + // Can't rebuild, new state is too large + metrics.bft().vertexStore().errorsDueToSizeLimit().inc(); + return Result.error(RebuildError.VERTEX_STORE_SIZE_EXCEEDED); + } - @Override - public Option tryRebuild(VertexStoreState vertexStoreState) { // FIXME: Currently this assumes vertexStoreState is a chain with no forks which is our only use // case at the moment. var executedVertices = new LinkedList(); @@ -144,34 +163,25 @@ public Option tryRebuild(VertexStoreState vertexStoreState) { // If any vertex couldn't be executed successfully, our saved state is invalid if (executedVertexMaybe.isEmpty()) { - return Option.empty(); + return Result.error(RebuildError.VERTEX_EXECUTION_ERROR); } executedVertices.add(executedVertexMaybe.get()); } - this.rootVertex = vertexStoreState.getRoot(); - this.highQC = vertexStoreState.getHighQC(); - this.vertices.clear(); - this.executedVertices.clear(); - this.vertexChildren.clear(); - this.vertexChildren.put(rootVertex.hash(), new HashSet<>()); + // Reset the vertex store to the new state + resetToState(vertexStoreState, serializedVertexStoreState); + // Insert the executed vertices, since we've already executed them. // Note that the vertices aren't re-executed at boot. See comment at `getExecutedVertex` for (var executedVertex : executedVertices) { - this.vertices.put(executedVertex.getVertexHash(), executedVertex.getVertexWithHash()); this.executedVertices.put(executedVertex.getVertexHash(), executedVertex); - this.vertexChildren.put(executedVertex.getVertexHash(), new HashSet<>()); - Set siblings = vertexChildren.get(executedVertex.getParentId()); - siblings.add(executedVertex.getVertexHash()); } - return Option.present(vertexStoreState); - } + metrics.bft().vertexStore().size().set(vertexStoreState.getVertices().size()); + metrics.bft().vertexStore().rebuilds().inc(); - @Override - public boolean containsVertex(HashCode vertexId) { - return vertices.containsKey(vertexId) || rootVertex.hash().equals(vertexId); + return Result.success(new RebuildSummary(vertexStoreState, serializedVertexStoreState)); } @Override @@ -186,7 +196,7 @@ public InsertQcResult insertQc(QuorumCertificate qc) { return new VertexStore.InsertQcResult.Ignored(); } - // proposed vertex doesn't have any children + // Proposed vertex doesn't have any children boolean isHighQC = qc.getRound().gt(highQC.highestQC().getRound()); if (isHighQC) { this.highQC = this.highQC.withHighestQC(qc); @@ -207,44 +217,71 @@ public InsertQcResult insertQc(QuorumCertificate qc) { committedUpdate = Option.empty(); } + metrics.bft().vertexStore().size().set(vertices.size()); + if (isHighQC || committedUpdate.isPresent()) { // We have either received a new highQc, or some vertices // were committed, or both. - return new VertexStore.InsertQcResult.Inserted(highQC(), getState(), committedUpdate); + final var state = getState(); + final var serializedVertexStoreState = serializeState(state); + // Note that inserting a QC can increase the size of the vertex store above the limit (e.g. if + // it doesn't + // commit any vertices and carries more signatures than our previous highest QC), but + // its size is small enough (compared to vertices) that we're accepting this corner case. + this.trackCurrentStateSize(serializedVertexStoreState); + + return new VertexStore.InsertQcResult.Inserted( + state.getHighQC(), serializedVertexStoreState, committedUpdate); } else { // This wasn't our new high QC and nothing has been committed return new VertexStore.InsertQcResult.Ignored(); } } - private void getChildrenVerticesList( - VertexWithHash parent, ImmutableList.Builder builder) { - Set childrenIds = this.vertexChildren.get(parent.hash()); - if (childrenIds == null) { - return; - } + /** + * Commit a vertex. Executes the transactions and prunes the tree. + * + * @param header the header to be committed + * @param commitQC the proof of commit + */ + private CommittedUpdate commit(BFTHeader header, QuorumCertificate commitQC) { + final HashCode vertexId = header.getVertexId(); + final VertexWithHash tipVertex = vertices.get(vertexId); - for (HashCode childId : childrenIds) { - final var v = vertices.get(childId); - builder.add(v); - getChildrenVerticesList(v, builder); + /* removeVertexAndPruneInternal skips children removal for the rootVertex, so we need to + keep a reference to the previous root and prune it *after* new rootVertex is set. + This isn't particularly easy to reason about and should be refactored at some point + (i.e. the logic should be moved out of removeVertexAndPruneInternal). */ + final var prevRootVertex = this.rootVertex; + this.rootVertex = tipVertex; + this.highQC = this.highQC.withHighestCommittedQC(commitQC); + final var path = ImmutableList.copyOf(getPathFromRoot(tipVertex.hash())); + HashCode prev = null; + for (int i = path.size() - 1; i >= 0; i--) { + this.removeVertexAndPruneInternal(path.get(i).getVertexHash(), Optional.ofNullable(prev)); + prev = path.get(i).getVertexHash(); } - } + removeVertexAndPruneInternal(prevRootVertex.hash(), Optional.empty()); - private VertexStoreState getState() { - // TODO: store list dynamically rather than recomputing - ImmutableList.Builder verticesBuilder = ImmutableList.builder(); - getChildrenVerticesList(this.rootVertex, verticesBuilder); - return VertexStoreState.create(this.highQC(), this.rootVertex, verticesBuilder.build(), hasher); + return new CommittedUpdate(path, highQC); } @Override - public boolean insertTimeoutCertificate(TimeoutCertificate timeoutCertificate) { + public InsertTcResult insertTimeoutCertificate(TimeoutCertificate timeoutCertificate) { if (timeoutCertificate.getRound().gt(highQC().getHighestRound())) { this.highQC = this.highQC.withHighestTC(timeoutCertificate); - return true; + + final var state = getState(); + final var serializedVertexStoreState = serializeState(state); + // Note that inserting a TC can increase the size of the vertex store above the limit + // (e.g. if it carries more signatures than our previous TC), but + // its size is small enough (compared to vertices) that we're accepting this corner case. + trackCurrentStateSize(serializedVertexStoreState); + + return new InsertTcResult.Inserted(state.getHighQC(), serializedVertexStoreState); + } else { + return new InsertTcResult.Ignored(); } - return false; } // TODO: reimplement in async way @@ -289,6 +326,7 @@ public Option getExecutedVertex(HashCode vertexHash) { public InsertVertexChainResult insertVertexChain(VertexChain vertexChain) { final var bftInsertUpdates = new ArrayList(); final var insertedQcs = new ArrayList(); + vertices_loop: for (VertexWithHash v : vertexChain.getVertices()) { final var insertQcResult = insertQc(v.vertex().getQCToParent()); @@ -302,7 +340,20 @@ public InsertVertexChainResult insertVertexChain(VertexChain vertexChain) { } } - insertVertex(v).onPresent(bftInsertUpdates::add); + final var insertRes = insertVertexInternal(v); + if (insertRes.isSuccess()) { + bftInsertUpdates.add(insertRes.unwrap()); + } else { + switch (insertRes.unwrapError()) { + case ALREADY_PRESENT, PREPARE_FAILED -> { + // No-op, continue iterating the vertices + } + case VERTEX_STORE_SIZE_EXCEEDED -> { + // Stop if we hit the size limit + break vertices_loop; + } + } + } } return new InsertVertexChainResult(insertedQcs, bftInsertUpdates); @@ -310,8 +361,13 @@ public InsertVertexChainResult insertVertexChain(VertexChain vertexChain) { @Override public Option insertVertex(VertexWithHash vertexWithHash) { + return insertVertexInternal(vertexWithHash).toOption(); + } + + private Result insertVertexInternal( + VertexWithHash vertexWithHash) { if (vertices.containsKey(vertexWithHash.hash())) { - return Option.empty(); + return Result.error(VertexInsertError.ALREADY_PRESENT); } final var vertex = vertexWithHash.vertex(); @@ -319,28 +375,53 @@ public Option insertVertex(VertexWithHash vertexWithHash) { throw new MissingParentException(vertex.getParentVertexId()); } - return insertVertexInternal(vertexWithHash); - } + // Before we execute the vertex, let's check if we can fit it into the store... + final var postInsertState = getState().withVertex(vertexWithHash); + final var postInsertSerializedState = serializeState(postInsertState); + if (postInsertSerializedState.size() > config.maxSerializedSizeBytes()) { + // ...nope, it won't fit + metrics.bft().vertexStore().errorsDueToSizeLimit().inc(); + return Result.error(VertexInsertError.VERTEX_STORE_SIZE_EXCEEDED); + } + // ...all good (size-wise), let's continue the insertion process. - private Option insertVertexInternal(VertexWithHash vertexWithHash) { - LinkedList previous = - getPathFromRoot(vertexWithHash.vertex().getParentVertexId()); + final var previous = getPathFromRoot(vertexWithHash.vertex().getParentVertexId()); final var executedVertexOption = Option.from(ledger.prepare(previous, vertexWithHash)); - return executedVertexOption.map( - executedVertex -> { - vertices.put(executedVertex.getVertexHash(), executedVertex.getVertexWithHash()); - executedVertices.put(executedVertex.getVertexHash(), executedVertex); - vertexChildren.put(executedVertex.getVertexHash(), new HashSet<>()); - Set siblings = vertexChildren.get(executedVertex.getParentId()); - siblings.add(executedVertex.getVertexHash()); - - VertexStoreState vertexStoreState = getState(); - return BFTInsertUpdate.insertedVertex(executedVertex, siblings.size(), vertexStoreState); - }); + return executedVertexOption + .>map( + executedVertex -> { + // The vertex was executed successfully, so we're inserting it + vertices.put(executedVertex.getVertexHash(), executedVertex.getVertexWithHash()); + executedVertices.put(executedVertex.getVertexHash(), executedVertex); + vertexChildren.put(executedVertex.getVertexHash(), new HashSet<>()); + Set siblings = vertexChildren.get(executedVertex.getParentId()); + siblings.add(executedVertex.getVertexHash()); + + // We've already calculated the post-insert state (and verified + // its size against the limit), so we can just use it here. + trackCurrentStateSize(postInsertSerializedState); + + // Update the metrics + metrics.bft().vertexStore().size().set(vertices.size()); + if (siblings.size() > 1) { + metrics.bft().vertexStore().forks().inc(); + } + if (!vertexWithHash.vertex().hasDirectParent()) { + metrics.bft().vertexStore().indirectParents().inc(); + } + + return Result.success(new BFTInsertUpdate(executedVertex, postInsertSerializedState)); + }) + .orElse(Result.error(VertexInsertError.PREPARE_FAILED)); } - private void removeVertexAndPruneInternal(HashCode vertexId, HashCode skip) { - vertices.remove(vertexId); + private void removeVertexAndPruneInternal(HashCode vertexId, Optional skip) { + Optional.ofNullable(vertices.remove(vertexId)) + .flatMap( + removedVertex -> + Optional.ofNullable(vertexChildren.get(removedVertex.vertex().getParentVertexId()))) + .ifPresent(siblings -> siblings.remove(vertexId)); + executedVertices.remove(vertexId); if (this.rootVertex.hash().equals(vertexId)) { @@ -350,43 +431,15 @@ private void removeVertexAndPruneInternal(HashCode vertexId, HashCode skip) { var children = vertexChildren.remove(vertexId); if (children != null) { for (HashCode child : children) { - if (!child.equals(skip)) { - removeVertexAndPruneInternal(child, null); + if (!skip.map(child::equals).orElse(false)) { + removeVertexAndPruneInternal(child, Optional.empty()); } } } } - /** - * Commit a vertex. Executes the transactions and prunes the tree. - * - * @param header the header to be committed - * @param commitQC the proof of commit - */ - private CommittedUpdate commit(BFTHeader header, QuorumCertificate commitQC) { - final HashCode vertexId = header.getVertexId(); - final VertexWithHash tipVertex = vertices.get(vertexId); - - /* removeVertexAndPruneInternal skips children removal for the rootVertex, so we need to - keep a reference to the previous root and prune it *after* new rootVertex is set. - This isn't particularly easy to reason about and should be refactored at some point - (i.e. the logic should be moved out of removeVertexAndPruneInternal). */ - final var prevRootVertex = this.rootVertex; - this.rootVertex = tipVertex; - this.highQC = this.highQC.withHighestCommittedQC(commitQC); - final var path = ImmutableList.copyOf(getPathFromRoot(tipVertex.hash())); - HashCode prev = null; - for (int i = path.size() - 1; i >= 0; i--) { - this.removeVertexAndPruneInternal(path.get(i).getVertexHash(), prev); - prev = path.get(i).getVertexHash(); - } - removeVertexAndPruneInternal(prevRootVertex.hash(), null); - - return new CommittedUpdate(path); - } - - @Override /** Returns a path of vertices up to the root vertex (excluding the root itself) */ + @Override public LinkedList getPathFromRoot(HashCode vertexId) { final LinkedList path = new LinkedList<>(); @@ -408,6 +461,37 @@ actually allows to still be able to get a path (the issue was more likely when v return path; } + @Override + public VertexWithHash getRoot() { + return rootVertex; + } + + @Override + public boolean containsVertex(HashCode vertexId) { + return vertices.containsKey(vertexId) || rootVertex.hash().equals(vertexId); + } + + private VertexStoreState getState() { + // TODO: store list dynamically rather than recomputing + ImmutableList.Builder verticesBuilder = ImmutableList.builder(); + getChildrenVerticesList(this.rootVertex, verticesBuilder); + return VertexStoreState.create(this.highQC(), this.rootVertex, verticesBuilder.build(), hasher); + } + + private void getChildrenVerticesList( + VertexWithHash parent, ImmutableList.Builder builder) { + Set childrenIds = this.vertexChildren.get(parent.hash()); + if (childrenIds == null) { + return; + } + + for (HashCode childId : childrenIds) { + final var v = vertices.get(childId); + builder.add(v); + getChildrenVerticesList(v, builder); + } + } + @Override public HighQC highQC() { return this.highQC; @@ -434,8 +518,23 @@ public Option> getVertices(HashCode vertexHash, in return Option.present(builder.build()); } + private void trackCurrentStateSize(WrappedByteArray serializedVertexStoreState) { + this.currentSerializedSizeBytes = serializedVertexStoreState.size(); + metrics.bft().vertexStore().byteSize().set(this.currentSerializedSizeBytes); + } + + private WrappedByteArray serializeState(VertexStoreState state) { + return new WrappedByteArray(serialization.toDson(state.toSerialized(), DsonOutput.Output.ALL)); + } + @VisibleForTesting Set verticesForWhichChildrenAreBeingStored() { return this.vertexChildren.keySet(); } + + private enum VertexInsertError { + ALREADY_PRESENT, + PREPARE_FAILED, + VERTEX_STORE_SIZE_EXCEEDED + } } diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java index bd750bfe39..b7b61bd863 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java @@ -67,7 +67,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.Round; @@ -89,25 +88,18 @@ *

In future, we'd like to move to having a separate vertex store, responsible for maintaining * its own state. */ -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") @Immutable public final class VertexStoreState { private static final Logger logger = LogManager.getLogger(); private final VertexWithHash root; private final HighQC highQC; - // TODO: collapse the following two private final ImmutableList vertices; - private final ImmutableMap idToVertex; private VertexStoreState( - HighQC highQC, - VertexWithHash root, - ImmutableMap idToVertex, - ImmutableList vertices) { + HighQC highQC, VertexWithHash root, ImmutableList vertices) { this.highQC = highQC; this.root = root; - this.idToVertex = idToVertex; this.vertices = vertices; } @@ -217,7 +209,14 @@ public static VertexStoreState create( */ } - return new VertexStoreState(highQC, root, ImmutableMap.copyOf(seen), vertices); + return new VertexStoreState(highQC, root, vertices); + } + + public VertexStoreState withVertex(VertexWithHash vertex) { + return new VertexStoreState( + this.highQC, + this.root, + ImmutableList.builder().addAll(this.vertices).add(vertex).build()); } public SerializedVertexStoreState toSerialized() { @@ -243,7 +242,7 @@ public ImmutableList getVertices() { @Override public int hashCode() { - return Objects.hash(root, highQC, idToVertex, vertices); + return Objects.hash(root, highQC, vertices); } @Override @@ -255,8 +254,7 @@ public boolean equals(Object o) { return o instanceof VertexStoreState other && Objects.equals(this.root, other.root) && Objects.equals(this.highQC, other.highQC) - && Objects.equals(this.vertices, other.vertices) - && Objects.equals(this.idToVertex, other.idToVertex); + && Objects.equals(this.vertices, other.vertices); } @Override @@ -303,16 +301,17 @@ public SerializedVertexStoreState( this.highQC = Objects.requireNonNull(highQC); } - public Vertex getRoot() { - return root; + public boolean isForEpoch(long epoch) { + return highQC.highestQC().getEpoch() == epoch; } - public ImmutableList getVertices() { - return vertices; - } + public VertexStoreState toVertexStoreState(Hasher hasher) { + var rootWithHash = root.withId(hasher); - public HighQC getHighQC() { - return highQC; + var verticesWithHash = + vertices.stream().map(v -> v.withId(hasher)).collect(ImmutableList.toImmutableList()); + + return VertexStoreState.create(highQC, rootWithHash, verticesWithHash, hasher); } @Override @@ -320,21 +319,6 @@ public int hashCode() { return Objects.hash(root, vertices, highQC); } - public boolean isForEpoch(long epoch) { - return getHighQC().highestQC().getEpoch() == epoch; - } - - public VertexStoreState toVertexStoreState(Hasher hasher) { - var rootVertex = getRoot().withId(hasher); - - var vertices = - getVertices().stream() - .map(v -> v.withId(hasher)) - .collect(ImmutableList.toImmutableList()); - - return VertexStoreState.create(getHighQC(), rootVertex, vertices, hasher); - } - @Override public boolean equals(Object o) { if (o == this) { diff --git a/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java b/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java index 27f2c2e1fb..043357f8e9 100644 --- a/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java +++ b/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java @@ -86,7 +86,6 @@ import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.TimeSupplier; import java.util.*; -import javax.annotation.Nullable; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -162,7 +161,8 @@ StateComputerPrepareResult prepare( List proposedTransactions, RoundDetails roundDetails); - LedgerProofBundle commit(LedgerExtension ledgerExtension, VertexStoreState vertexStore); + LedgerProofBundle commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState); } private final StateComputer stateComputer; @@ -294,53 +294,61 @@ private Optional prepareInternal( this.timeSupplier.currentTime())); } - public EventProcessor bftCommittedUpdateEventProcessor() { - return committedUpdate -> { - updateCommittedVerticesMetrics(committedUpdate); - - final ImmutableList transactions = - committedUpdate.committed().stream() - .flatMap(ExecutedVertex::successfulTransactions) - .map(ExecutedTransaction::transaction) - .collect(ImmutableList.toImmutableList()); - - final var maybeProcessedQcCommit = - committedUpdate - .vertexStoreState() - .getHighQC() - .highestCommittedQC() - .getProcessedCommit(hasher); - - maybeProcessedQcCommit.ifPresentOrElse( - processedQcCommit -> { - switch (processedQcCommit) { - case ProcessedQcCommit.OfConensusQc ofConensusQc -> { - final var ledgerExtension = - LedgerExtension.create(transactions, ofConensusQc.ledgerProof()); - metrics - .ledger() - .commit() - .measure( - () -> this.commit(ledgerExtension, committedUpdate.vertexStoreState())); - } - case ProcessedQcCommit.OfInitialEpochQc ofInitialEpochQc -> { - // no-op, ignore - this.metrics.ledger().ignoredBftCommittedUpdates().inc(); - } - } - }, - () -> { - // no-op, ignore - this.metrics.ledger().ignoredBftCommittedUpdates().inc(); - }); + public EventProcessor bftHighQcUpdateEventProcessor() { + return highQcUpdate -> { + highQcUpdate + .committedVertices() + .ifPresent( + committedVertices -> { + updateCommittedVerticesMetrics(committedVertices); + + final ImmutableList transactions = + committedVertices.stream() + .flatMap(ExecutedVertex::successfulTransactions) + .map(ExecutedTransaction::transaction) + .collect(ImmutableList.toImmutableList()); + + highQcUpdate + .newHighQc() + .highestCommittedQC() + .getProcessedCommit(hasher) + .ifPresentOrElse( + processedQcCommit -> { + switch (processedQcCommit) { + case ProcessedQcCommit.OfConensusQc ofConensusQc -> { + final var ledgerExtension = + LedgerExtension.create(transactions, ofConensusQc.ledgerProof()); + metrics + .ledger() + .commit() + .measure( + () -> + this.commit( + ledgerExtension, + Option.some( + highQcUpdate + .serializedVertexStoreState() + .value()))); + } + case ProcessedQcCommit.OfInitialEpochQc ofInitialEpochQc -> { + // no-op, ignore + this.metrics.ledger().ignoredBftCommittedUpdates().inc(); + } + } + }, + () -> { + // no-op, ignore + this.metrics.ledger().ignoredBftCommittedUpdates().inc(); + }); + }); }; } - private void updateCommittedVerticesMetrics(BFTCommittedUpdate committedUpdate) { + private void updateCommittedVerticesMetrics(ImmutableList committedVertices) { final var numCommittedFallbackVertices = - committedUpdate.committed().stream().filter(v -> v.vertex().isFallback()).count(); + committedVertices.stream().filter(v -> v.vertex().isFallback()).count(); final var numCommittedNonFallbackVertices = - committedUpdate.committed().size() - numCommittedFallbackVertices; + committedVertices.size() - numCommittedFallbackVertices; metrics .bft() @@ -356,12 +364,12 @@ private void updateCommittedVerticesMetrics(BFTCommittedUpdate committedUpdate) } public EventProcessor syncEventProcessor() { - return p -> metrics.ledger().commit().measure(() -> this.commit(p, null)); + return p -> metrics.ledger().commit().measure(() -> this.commit(p, Option.empty())); } /** * Appends a useful suffix of the given {@link LedgerExtension} to the persistent ledger (while - * also writing the new {@link VertexStoreState}, if non-{@literal null}). + * also writing the new serialized {@link VertexStoreState}, if present). * *

A "useful suffix" is a sub-list of transactions that are not yet persisted in the local * ledger. The implementation will resolve this sub-list based on the end state version from the @@ -388,7 +396,7 @@ public EventProcessor syncEventProcessor() { * InvalidCommitRequestException} is propagated from the Rust state computer. * */ - private void commit(LedgerExtension ledgerExtension, @Nullable VertexStoreState vertexStore) { + private void commit(LedgerExtension ledgerExtension, Option serializedVertexStoreState) { final var proofToCommit = ledgerExtension.proof(); final int extensionTransactionCount; // for metrics purposes only @@ -406,7 +414,7 @@ private void commit(LedgerExtension ledgerExtension, @Nullable VertexStoreState final var extensionToCommit = ledgerExtension.getExtensionFrom(latestStateVersion); // persist - this.latestProof = this.stateComputer.commit(extensionToCommit, vertexStore); + this.latestProof = this.stateComputer.commit(extensionToCommit, serializedVertexStoreState); extensionTransactionCount = extensionToCommit.transactions().size(); } @@ -415,7 +423,7 @@ private void commit(LedgerExtension ledgerExtension, @Nullable VertexStoreState // synchronization theoretically needed here). this.metrics.ledger().stateVersion().set(this.latestProof.resultantStateVersion()); - if (vertexStore == null) { + if (serializedVertexStoreState.isPresent()) { this.metrics.ledger().syncTransactionsProcessed().inc(extensionTransactionCount); } else { this.metrics.ledger().bftTransactionsProcessed().inc(extensionTransactionCount); diff --git a/core/src/main/java/com/radixdlt/modules/DispatcherModule.java b/core/src/main/java/com/radixdlt/modules/DispatcherModule.java index 4cdef036f0..7e6afa0316 100644 --- a/core/src/main/java/com/radixdlt/modules/DispatcherModule.java +++ b/core/src/main/java/com/radixdlt/modules/DispatcherModule.java @@ -220,9 +220,6 @@ public void configure() { final var highQcUpdateKey = new TypeLiteral>() {}; Multibinder.newSetBinder(binder(), highQcUpdateKey, ProcessOnDispatch.class); Multibinder.newSetBinder(binder(), highQcUpdateKey); - final var committedUpdateKey = new TypeLiteral>() {}; - Multibinder.newSetBinder(binder(), committedUpdateKey); - Multibinder.newSetBinder(binder(), committedUpdateKey, ProcessOnDispatch.class); final var syncUpdateKey = new TypeLiteral>() {}; Multibinder.newSetBinder(binder(), syncUpdateKey, ProcessOnDispatch.class); @@ -381,18 +378,9 @@ private ScheduledEventDispatcher scheduledTimeoutDispatch @Provides private EventDispatcher bftInsertUpdateEventDispatcher( - @ProcessOnDispatch Set> processors, - Environment environment, - Metrics metrics) { + @ProcessOnDispatch Set> processors, Environment environment) { var dispatcher = environment.getDispatcher(BFTInsertUpdate.class); return update -> { - if (update.getSiblingsCount() > 1) { - metrics.bft().vertexStore().forks().inc(); - } - if (!update.getInserted().getVertexWithHash().vertex().hasDirectParent()) { - metrics.bft().vertexStore().indirectParents().inc(); - } - metrics.bft().vertexStore().size().set(update.getVertexStoreSize()); dispatcher.dispatch(update); processors.forEach(p -> p.process(update)); }; @@ -400,13 +388,8 @@ private EventDispatcher bftInsertUpdateEventDispatcher( @Provides private EventDispatcher bftRebuildUpdateEventDispatcher( - Environment environment, Metrics metrics) { - var dispatcher = environment.getDispatcher(BFTRebuildUpdate.class); - return update -> { - metrics.bft().vertexStore().size().set(update.getVertexStoreState().getVertices().size()); - metrics.bft().vertexStore().rebuilds().inc(); - dispatcher.dispatch(update); - }; + Environment environment) { + return environment.getDispatcher(BFTRebuildUpdate.class); } @Provides @@ -421,23 +404,10 @@ private EventDispatcher bftHighQCUpdateEventDispatcher( @Provides private EventDispatcher syncUpdateEventDispatcher( - @ProcessOnDispatch Set> processors, Metrics metrics) { + @ProcessOnDispatch Set> processors) { return commit -> processors.forEach(e -> e.process(commit)); } - @Provides - private EventDispatcher committedUpdateEventDispatcher( - @ProcessOnDispatch Set> processors, - Environment environment, - Metrics metrics) { - var dispatcher = environment.getDispatcher(BFTCommittedUpdate.class); - return commit -> { - metrics.bft().vertexStore().size().set(commit.vertexStoreSize()); - processors.forEach(e -> e.process(commit)); - dispatcher.dispatch(commit); - }; - } - @Provides private EventDispatcher localConsensusTimeoutDispatcher( @ProcessOnDispatch Set> syncProcessors, diff --git a/core/src/main/java/com/radixdlt/modules/LedgerModule.java b/core/src/main/java/com/radixdlt/modules/LedgerModule.java index a9a167082d..ff82c37c54 100644 --- a/core/src/main/java/com/radixdlt/modules/LedgerModule.java +++ b/core/src/main/java/com/radixdlt/modules/LedgerModule.java @@ -68,7 +68,7 @@ import com.google.inject.Scopes; import com.google.inject.multibindings.ProvidesIntoSet; import com.radixdlt.consensus.Ledger; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.environment.EventProcessor; import com.radixdlt.environment.ProcessOnDispatch; import com.radixdlt.ledger.LedgerExtension; @@ -91,8 +91,8 @@ private EventProcessor syncToLedgerCommittor( @ProvidesIntoSet @ProcessOnDispatch - private EventProcessor bftToLedgerCommittor( + private EventProcessor bftToLedgerCommittor( StateComputerLedger stateComputerLedger) { - return stateComputerLedger.bftCommittedUpdateEventProcessor(); + return stateComputerLedger.bftHighQcUpdateEventProcessor(); } } diff --git a/core/src/main/java/com/radixdlt/modules/SystemInfoModule.java b/core/src/main/java/com/radixdlt/modules/SystemInfoModule.java index b7b0c79ef0..0c07f76dd4 100644 --- a/core/src/main/java/com/radixdlt/modules/SystemInfoModule.java +++ b/core/src/main/java/com/radixdlt/modules/SystemInfoModule.java @@ -69,7 +69,6 @@ import com.google.inject.TypeLiteral; import com.google.inject.multibindings.Multibinder; import com.google.inject.multibindings.ProvidesIntoSet; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.epoch.EpochRoundUpdate; import com.radixdlt.consensus.liveness.EpochLocalTimeoutOccurrence; @@ -89,7 +88,6 @@ protected void configure() { .permitDuplicates(); eventBinder.addBinding().toInstance(EpochRoundUpdate.class); eventBinder.addBinding().toInstance(EpochLocalTimeoutOccurrence.class); - eventBinder.addBinding().toInstance(BFTCommittedUpdate.class); eventBinder.addBinding().toInstance(BFTHighQCUpdate.class); } diff --git a/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java b/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java index 13892a497e..72390cd2e3 100644 --- a/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java +++ b/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java @@ -84,7 +84,6 @@ import com.radixdlt.monitoring.Metrics; import com.radixdlt.p2p.NodeId; import com.radixdlt.protocol.RustProtocolUpdate; -import com.radixdlt.serialization.DsonOutput; import com.radixdlt.serialization.Serialization; import com.radixdlt.statecomputer.RustStateComputer; import com.radixdlt.statecomputer.commit.*; @@ -283,21 +282,17 @@ public StateComputerLedger.StateComputerPrepareResult prepare( } @Override - public LedgerProofBundle commit(LedgerExtension ledgerExtension, VertexStoreState vertexStore) { + public LedgerProofBundle commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { final var proof = ledgerExtension.proof(); final var header = proof.ledgerHeader(); - final Option vertexStoreBytes; - if (vertexStore != null) { - vertexStoreBytes = - Option.some(serialization.toDson(vertexStore.toSerialized(), DsonOutput.Output.ALL)); - } else { - vertexStoreBytes = Option.none(); - } - var commitRequest = new CommitRequest( - ledgerExtension.transactions(), proof, vertexStoreBytes, Option.from(selfValidatorId)); + ledgerExtension.transactions(), + proof, + serializedVertexStoreState, + Option.from(selfValidatorId)); final var result = stateComputer.commit(commitRequest); final var commitSummary = diff --git a/core/src/main/java/com/radixdlt/rev2/modules/MockedVertexStoreModule.java b/core/src/main/java/com/radixdlt/rev2/modules/MockedVertexStoreModule.java index 444177534d..56702d8c6b 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/MockedVertexStoreModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/MockedVertexStoreModule.java @@ -66,7 +66,6 @@ import com.google.inject.AbstractModule; import com.radixdlt.consensus.vertexstore.PersistentVertexStore; -import com.radixdlt.consensus.vertexstore.VertexStoreState; public final class MockedVertexStoreModule extends AbstractModule { @@ -77,7 +76,7 @@ public void configure() { private static class MockedPersistentVertexStore implements PersistentVertexStore { @Override - public void save(VertexStoreState vertexStoreState) { + public void save(byte[] serializedVertexStoreState) { // Nothing to do here } } diff --git a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java index f96bb36986..26f43ca72a 100644 --- a/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java +++ b/core/src/main/java/com/radixdlt/rev2/modules/REv2StateManagerModule.java @@ -88,7 +88,6 @@ import com.radixdlt.protocol.RustProtocolUpdate; import com.radixdlt.recovery.VertexStoreRecovery; import com.radixdlt.rev2.*; -import com.radixdlt.serialization.DsonOutput; import com.radixdlt.serialization.Serialization; import com.radixdlt.state.RustStateReader; import com.radixdlt.statecomputer.RustStateComputer; @@ -275,12 +274,10 @@ TestStateReader testStateReader(NodeRustEnvironment nodeRustEnvironment) { } @Provides - PersistentVertexStore vertexStore( - VertexStoreRecovery recovery, Metrics metrics, Serialization serialization) { - return s -> { + PersistentVertexStore vertexStore(VertexStoreRecovery recovery, Metrics metrics) { + return serializedVertexStoreState -> { metrics.misc().vertexStoreSaved().inc(); - var vertexStoreBytes = serialization.toDson(s.toSerialized(), DsonOutput.Output.ALL); - recovery.saveVertexStore(vertexStoreBytes); + recovery.saveVertexStore(serializedVertexStoreState); }; } @@ -288,14 +285,30 @@ PersistentVertexStore vertexStore( @ProcessOnDispatch EventProcessor onQCUpdatePersistVertexStore( PersistentVertexStore persistentVertexStore) { - return update -> persistentVertexStore.save(update.getVertexStoreState()); + return update -> { + // We're only persisting the vertex store state here if the update + // doesn't carry a commit. Otherwise, the vertex store state is + // already persisted alongside the commit, so no need to repeat it here. + if (update.committedVertices().isEmpty()) { + persistentVertexStore.save(update.serializedVertexStoreState().value()); + } + }; } @ProvidesIntoSet @ProcessOnDispatch EventProcessor onInsertUpdatePersistVertexStore( PersistentVertexStore persistentVertexStore) { - return update -> persistentVertexStore.save(update.getVertexStoreState()); + return update -> + persistentVertexStore.save(update.serializedVertexStoreState().value()); + } + + @ProvidesIntoSet + @ProcessOnDispatch + EventProcessor onRebuildUpdatePersistVertexStore( + PersistentVertexStore persistentVertexStore) { + return update -> + persistentVertexStore.save(update.serializedVertexStoreState().value()); } }); diff --git a/core/src/test-core/java/com/radixdlt/environment/NoEpochsConsensusModule.java b/core/src/test-core/java/com/radixdlt/environment/NoEpochsConsensusModule.java index 55ed7447fc..d636757a55 100644 --- a/core/src/test-core/java/com/radixdlt/environment/NoEpochsConsensusModule.java +++ b/core/src/test-core/java/com/radixdlt/environment/NoEpochsConsensusModule.java @@ -82,6 +82,7 @@ import com.radixdlt.consensus.sync.*; import com.radixdlt.consensus.vertexstore.VertexStore; import com.radixdlt.consensus.vertexstore.VertexStoreAdapter; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.consensus.vertexstore.VertexStoreJavaImpl; import com.radixdlt.crypto.Hasher; import com.radixdlt.ledger.LedgerProofBundle; @@ -89,6 +90,7 @@ import com.radixdlt.messaging.core.GetVerticesRequestRateLimit; import com.radixdlt.monitoring.Metrics; import com.radixdlt.p2p.NodeId; +import com.radixdlt.serialization.Serialization; import com.radixdlt.sync.messages.local.LocalSyncRequest; import com.radixdlt.utils.TimeSupplier; import java.util.Comparator; @@ -115,7 +117,6 @@ public void configure() { eventBinder.addBinding().toInstance(BFTRebuildUpdate.class); eventBinder.addBinding().toInstance(BFTInsertUpdate.class); eventBinder.addBinding().toInstance(BFTHighQCUpdate.class); - eventBinder.addBinding().toInstance(BFTCommittedUpdate.class); eventBinder.addBinding().toInstance(Proposal.class); eventBinder.addBinding().toInstance(Vote.class); eventBinder.addBinding().toInstance(LedgerUpdate.class); @@ -301,8 +302,20 @@ private BFTSync bftSync( @Provides @Singleton - private VertexStore vertexStore(BFTConfiguration bftConfiguration, Ledger ledger, Hasher hasher) { - return VertexStoreJavaImpl.create(bftConfiguration.getVertexStoreState(), ledger, hasher); + private VertexStore vertexStore( + BFTConfiguration bftConfiguration, + Ledger ledger, + Hasher hasher, + Serialization serialization, + Metrics metrics, + VertexStoreConfig vertexStoreConfig) { + return new VertexStoreJavaImpl( + ledger, + hasher, + serialization, + metrics, + vertexStoreConfig, + bftConfiguration.getVertexStoreState()); } @Provides @@ -311,14 +324,9 @@ private VertexStoreAdapter vertexStoreAdapter( VertexStore vertexStore, EventDispatcher updateSender, EventDispatcher rebuildUpdateDispatcher, - EventDispatcher highQCUpdateEventDispatcher, - EventDispatcher committedSender) { + EventDispatcher highQCUpdateEventDispatcher) { return new VertexStoreAdapter( - vertexStore, - highQCUpdateEventDispatcher, - updateSender, - rebuildUpdateDispatcher, - committedSender); + vertexStore, highQCUpdateEventDispatcher, updateSender, rebuildUpdateDispatcher); } @ProvidesIntoSet diff --git a/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicNodes.java b/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicNodes.java index 13ab70d87f..1f2b682eb8 100644 --- a/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicNodes.java +++ b/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicNodes.java @@ -107,7 +107,8 @@ public final class DeterministicNodes implements AutoCloseable { private final ControlledAddressBook addressBook; private final Map nodeConfigs; private final Module baseModule; - private final Module overrideModule; + + private Module overrideModule; // Network private final DeterministicNetwork network; @@ -166,6 +167,14 @@ public Integer apply(NodeId nodeId) { } } + void setOverrideModule(Module overrideModule) { + this.overrideModule = overrideModule; + } + + Module getOverrideModule() { + return this.overrideModule; + } + private Injector createBFTInstance( int nodeIndex, Module baseModule, Module overrideModule, long time) { var config = this.nodeConfigs.get(nodeIndex); diff --git a/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicTest.java b/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicTest.java index 5e33c46c57..d3f3560b4d 100644 --- a/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicTest.java +++ b/core/src/test-core/java/com/radixdlt/harness/deterministic/DeterministicTest.java @@ -302,6 +302,17 @@ public void restartNodeWithConfig(int nodeIndex, PhysicalNodeConfig config) { this.startNode(nodeIndex); } + public void restartNodeWithOverrideModule(int nodeIndex, Module overrideModule) { + this.shutdownNode(nodeIndex); + // Keep the current module + final var origOverrideModule = this.nodes.getOverrideModule(); + // Use the provided module for starting this node + this.nodes.setOverrideModule(overrideModule); + this.startNode(nodeIndex); + // Restore the original module + this.nodes.setOverrideModule(origOverrideModule); + } + public static class NeverReachedStateException extends IllegalStateException { private NeverReachedStateException(int max) { super("Never reached state after " + max + " messages"); diff --git a/core/src/test-core/java/com/radixdlt/harness/deterministic/SafetyCheckerModule.java b/core/src/test-core/java/com/radixdlt/harness/deterministic/SafetyCheckerModule.java index 795ed4d86a..b15c8dcbf4 100644 --- a/core/src/test-core/java/com/radixdlt/harness/deterministic/SafetyCheckerModule.java +++ b/core/src/test-core/java/com/radixdlt/harness/deterministic/SafetyCheckerModule.java @@ -69,7 +69,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Scopes; import com.google.inject.multibindings.ProvidesIntoSet; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.harness.invariants.SafetyChecker; import com.radixdlt.harness.simulation.TestInvariant; import java.util.Optional; @@ -84,7 +84,7 @@ public void configure() { @ProvidesIntoSet public NodeEvents.NodeEventProcessor safetyCheckProcessor(SafetyChecker safetyChecker) { return new NodeEvents.NodeEventProcessor<>( - BFTCommittedUpdate.class, + BFTHighQCUpdate.class, (node, update) -> { Optional maybeError = safetyChecker.process(node, update); diff --git a/core/src/test-core/java/com/radixdlt/harness/deterministic/invariants/DeterministicMonitors.java b/core/src/test-core/java/com/radixdlt/harness/deterministic/invariants/DeterministicMonitors.java index 2c9e6edee4..b791287a2c 100644 --- a/core/src/test-core/java/com/radixdlt/harness/deterministic/invariants/DeterministicMonitors.java +++ b/core/src/test-core/java/com/radixdlt/harness/deterministic/invariants/DeterministicMonitors.java @@ -70,7 +70,6 @@ import com.google.inject.multibindings.ProvidesIntoSet; import com.radixdlt.consensus.ConsensusByzantineEvent; import com.radixdlt.consensus.QuorumCertificate; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.epoch.EpochRound; @@ -156,13 +155,10 @@ public void next(ControlledMessage message, long currentTime) { } final QuorumCertificate highQC; - switch (message.message()) { - case BFTHighQCUpdate update -> highQC = update.getHighQC().highestQC(); - case BFTCommittedUpdate committed -> highQC = - committed.vertexStoreState().getHighQC().highestQC(); - default -> { - return; - } + if (message.message() instanceof BFTHighQCUpdate update) { + highQC = update.newHighQc().highestQC(); + } else { + return; } var header = highQC.getProposedHeader(); diff --git a/core/src/test-core/java/com/radixdlt/harness/invariants/SafetyChecker.java b/core/src/test-core/java/com/radixdlt/harness/invariants/SafetyChecker.java index 762a0ef98a..2f274741d4 100644 --- a/core/src/test-core/java/com/radixdlt/harness/invariants/SafetyChecker.java +++ b/core/src/test-core/java/com/radixdlt/harness/invariants/SafetyChecker.java @@ -69,7 +69,7 @@ import com.google.inject.Inject; import com.radixdlt.consensus.BFTHeader; import com.radixdlt.consensus.VertexWithHash; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.epoch.EpochRound; import com.radixdlt.consensus.vertexstore.ExecutedVertex; @@ -158,11 +158,11 @@ private Optional process( } public Optional process( - NodeId node, BFTCommittedUpdate committedUpdate) { - ImmutableList vertices = committedUpdate.committed(); - for (ExecutedVertex vertex : vertices) { + NodeId node, BFTHighQCUpdate highQCUpdate) { + final var committedVertices = highQCUpdate.committedVertices().orElse(ImmutableList.of()); + for (ExecutedVertex committedVertex : committedVertices) { Optional maybeError = - process(node, vertex.getVertexWithHash()); + process(node, committedVertex.getVertexWithHash()); if (maybeError.isPresent()) { return maybeError; } diff --git a/core/src/test-core/java/com/radixdlt/harness/predicates/NodePredicate.java b/core/src/test-core/java/com/radixdlt/harness/predicates/NodePredicate.java index 9f260ef76d..89c80a5c25 100644 --- a/core/src/test-core/java/com/radixdlt/harness/predicates/NodePredicate.java +++ b/core/src/test-core/java/com/radixdlt/harness/predicates/NodePredicate.java @@ -68,6 +68,7 @@ import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.liveness.PacemakerState; import com.radixdlt.consensus.safety.SafetyRules; +import com.radixdlt.monitoring.Metrics; import com.radixdlt.statecomputer.commit.LedgerProof; import com.radixdlt.sync.TransactionsAndProofReader; import com.radixdlt.testutil.TestStateReader; @@ -167,6 +168,14 @@ public static Predicate bftAtOrOverRound(Round round) { return i -> i.getInstance(PacemakerState.class).highQC().getHighestRound().gte(round); } + public static Predicate atOrOverRound(Round round) { + return metricsPredicate(metrics -> metrics.bft().pacemaker().round().get() >= round.number()); + } + + public static Predicate metricsPredicate(Predicate predicate) { + return i -> predicate.test(i.getInstance(Metrics.class)); + } + public static Predicate votedAtRound(Round round) { return i -> i.getInstance(SafetyRules.class).getLastVote(round).isPresent(); } diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/SimulationNodeEventsModule.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/SimulationNodeEventsModule.java index 4c4d7d31a2..f02dfdb9db 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/SimulationNodeEventsModule.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/SimulationNodeEventsModule.java @@ -66,7 +66,6 @@ import com.google.inject.AbstractModule; import com.google.inject.multibindings.ProvidesIntoSet; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Self; import com.radixdlt.consensus.liveness.EpochLocalTimeoutOccurrence; @@ -99,13 +98,6 @@ private EventProcessor requestProcessor( return nodeEvents.processor(node, GetVerticesRequest.class); } - @ProvidesIntoSet - @ProcessOnDispatch - private EventProcessor committedProcessor( - @Self NodeId node, NodeEvents nodeEvents) { - return nodeEvents.processor(node, BFTCommittedUpdate.class); - } - @ProvidesIntoSet @ProcessOnDispatch private EventProcessor highQCProcessor( diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ConsensusMonitors.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ConsensusMonitors.java index 0e6ea82d0a..ba93e7a27c 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ConsensusMonitors.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ConsensusMonitors.java @@ -67,7 +67,7 @@ import com.google.inject.AbstractModule; import com.google.inject.Module; import com.google.inject.multibindings.ProvidesIntoMap; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.liveness.EpochLocalTimeoutOccurrence; import com.radixdlt.consensus.liveness.LocalTimeoutOccurrence; @@ -174,7 +174,8 @@ public static Module noneCommitted() { @ProvidesIntoMap @MonitorKey(Monitor.CONSENSUS_NONE_COMMITTED) TestInvariant noneCommittedInvariant(NodeEvents nodeEvents) { - return new EventNeverOccursInvariant<>(nodeEvents, BFTCommittedUpdate.class, u -> true); + return new EventNeverOccursInvariant<>( + nodeEvents, BFTHighQCUpdate.class, ev -> ev.committedVertices().isPresent()); } }; } diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java index 0324e8ae20..094da7c690 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java @@ -66,7 +66,6 @@ import com.google.common.collect.Ordering; import com.radixdlt.consensus.QuorumCertificate; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.epoch.EpochRound; @@ -99,14 +98,14 @@ public Observable check(RunningNetwork network) { emitter -> { nodeEvents.addListener( (node, highQCUpdate) -> { - emitter.onNext(highQCUpdate.getHighQC().highestQC()); + emitter.onNext(highQCUpdate.newHighQc().highestQC()); }, BFTHighQCUpdate.class); nodeEvents.addListener( - (node, committed) -> { - emitter.onNext(committed.vertexStoreState().getHighQC().highestQC()); + (node, highQcUpdate) -> { + emitter.onNext(highQcUpdate.newHighQc().highestQC()); }, - BFTCommittedUpdate.class); + BFTHighQCUpdate.class); }) .serialize() .map(QuorumCertificate::getProposedHeader) diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ProposerTimestampChecker.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ProposerTimestampChecker.java index f2b011d65c..791029e88e 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ProposerTimestampChecker.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/ProposerTimestampChecker.java @@ -64,7 +64,8 @@ package com.radixdlt.harness.simulation.monitors.consensus; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.google.common.collect.ImmutableList; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.vertexstore.ExecutedVertex; import com.radixdlt.harness.simulation.TestInvariant; import com.radixdlt.harness.simulation.monitors.NodeEvents; @@ -80,14 +81,14 @@ public ProposerTimestampChecker(NodeEvents nodeEvents) { @Override public Observable check(RunningNetwork network) { - return Observable.create( + return Observable.create( emitter -> nodeEvents.addListener( - (node, update) -> emitter.onNext(update), BFTCommittedUpdate.class)) + (node, update) -> emitter.onNext(update), BFTHighQCUpdate.class)) .serialize() .flatMap( e -> { - for (ExecutedVertex v : e.committed()) { + for (ExecutedVertex v : e.committedVertices().or(ImmutableList.of())) { final var proposerTimestamp = v.getLedgerHeader().proposerTimestamp(); final var prevTimestamp = v.vertex().parentLedgerHeader().proposerTimestamp(); if (proposerTimestamp < prevTimestamp) { diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/SafetyInvariant.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/SafetyInvariant.java index 808bad445c..c6889f42a4 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/SafetyInvariant.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/SafetyInvariant.java @@ -64,7 +64,7 @@ package com.radixdlt.harness.simulation.monitors.consensus; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.harness.invariants.SafetyChecker; import com.radixdlt.harness.simulation.TestInvariant; import com.radixdlt.harness.simulation.monitors.NodeEvents; @@ -85,11 +85,10 @@ public SafetyInvariant(NodeEvents nodeEvents) { @Override public Observable check(RunningNetwork network) { final SafetyChecker safetyChecker = new SafetyChecker(network.getNodes()); - return Observable.>create( + return Observable.>create( emitter -> nodeEvents.addListener( - (node, update) -> emitter.onNext(Pair.of(node, update)), - BFTCommittedUpdate.class)) + (node, update) -> emitter.onNext(Pair.of(node, update)), BFTHighQCUpdate.class)) .serialize() .flatMap( e -> { diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/epochs/EpochRoundInvariant.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/epochs/EpochRoundInvariant.java index 3ef5c68644..fedeeb5302 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/epochs/EpochRoundInvariant.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/epochs/EpochRoundInvariant.java @@ -64,7 +64,8 @@ package com.radixdlt.harness.simulation.monitors.epochs; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.google.common.collect.ImmutableList; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.bft.Round; import com.radixdlt.harness.simulation.TestInvariant; import com.radixdlt.harness.simulation.monitors.NodeEvents; @@ -74,22 +75,25 @@ /** Invariant which checks that a committed vertex never goes above some round */ public class EpochRoundInvariant implements TestInvariant { - private final NodeEvents commits; + private final NodeEvents nodeEvents; private final Round epochMaxRound; - public EpochRoundInvariant(Round epochMaxRound, NodeEvents commits) { - this.commits = commits; + public EpochRoundInvariant(Round epochMaxRound, NodeEvents nodeEvents) { + this.nodeEvents = nodeEvents; this.epochMaxRound = Objects.requireNonNull(epochMaxRound); } @Override public Observable check(RunningNetwork network) { - return Observable.create( + return Observable.create( emitter -> - this.commits.addListener( - (node, commit) -> emitter.onNext(commit), BFTCommittedUpdate.class)) + this.nodeEvents.addListener( + (node, commit) -> emitter.onNext(commit), BFTHighQCUpdate.class)) .serialize() - .concatMap(committedUpdate -> Observable.fromStream(committedUpdate.committed().stream())) + .concatMap( + highQCUpdate -> + Observable.fromStream( + highQCUpdate.committedVertices().or(ImmutableList.of()).stream())) .flatMap( vertex -> { final Round round = vertex.getRound(); diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/ledger/ConsensusToLedgerCommittedInvariant.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/ledger/ConsensusToLedgerCommittedInvariant.java index e5033d4f91..5690207a94 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/ledger/ConsensusToLedgerCommittedInvariant.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/ledger/ConsensusToLedgerCommittedInvariant.java @@ -64,7 +64,8 @@ package com.radixdlt.harness.simulation.monitors.ledger; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; +import com.google.common.collect.ImmutableList; +import com.radixdlt.consensus.bft.BFTHighQCUpdate; import com.radixdlt.consensus.vertexstore.ExecutedVertex; import com.radixdlt.harness.simulation.TestInvariant; import com.radixdlt.harness.simulation.monitors.NodeEvents; @@ -79,7 +80,7 @@ /** * Checks to make sure that everything committed by consensus eventually makes it to the ledger of - * atleast one node (TODO: test for every node) + * at least one node (TODO: test for every node) */ public class ConsensusToLedgerCommittedInvariant implements TestInvariant { private final NodeEvents commits; @@ -102,15 +103,14 @@ public Observable check(RunningNetwork network) { }) .subscribe(committedTxns::onNext); - return Observable.create( + return Observable.create( emitter -> - commits.addListener( - (node, event) -> emitter.onNext(event), BFTCommittedUpdate.class)) + commits.addListener((node, event) -> emitter.onNext(event), BFTHighQCUpdate.class)) .serialize() .concatMap( committedUpdate -> Observable.fromStream( - committedUpdate.committed().stream() + committedUpdate.committedVertices().orElse(ImmutableList.of()).stream() .flatMap(ExecutedVertex::successfulTransactions))) .flatMapMaybe( txn -> diff --git a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java index 61501e08c7..a1bac28d67 100644 --- a/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java +++ b/core/src/test-core/java/com/radixdlt/modules/FunctionalRadixNodeModule.java @@ -73,6 +73,7 @@ import com.radixdlt.consensus.epoch.EpochsConsensusModule; import com.radixdlt.consensus.liveness.ProposalGenerator; import com.radixdlt.consensus.sync.BFTSyncPatienceMillis; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.environment.NoEpochsConsensusModule; import com.radixdlt.environment.NoEpochsSyncModule; import com.radixdlt.environment.NodeAutoCloseable; @@ -131,18 +132,21 @@ public static final class ConsensusConfig { private final double pacemakerBackoffRate; private final long additionalRoundTimeIfProposalReceivedMs; private final long timeoutQuorumResolutionDelayMs; + private final VertexStoreConfig vertexStoreConfig; - private ConsensusConfig( + public ConsensusConfig( int bftSyncPatienceMillis, long pacemakerBaseTimeoutMs, double pacemakerBackoffRate, long additionalRoundTimeIfProposalReceivedMs, - long timeoutQuorumResolutionDelayMs) { + long timeoutQuorumResolutionDelayMs, + VertexStoreConfig vertexStoreConfig) { this.bftSyncPatienceMillis = bftSyncPatienceMillis; this.pacemakerBaseTimeoutMs = pacemakerBaseTimeoutMs; this.pacemakerBackoffRate = pacemakerBackoffRate; this.additionalRoundTimeIfProposalReceivedMs = additionalRoundTimeIfProposalReceivedMs; this.timeoutQuorumResolutionDelayMs = timeoutQuorumResolutionDelayMs; + this.vertexStoreConfig = vertexStoreConfig; } public static ConsensusConfig of( @@ -156,7 +160,8 @@ public static ConsensusConfig of( pacemakerBaseTimeoutMs, pacemakerBackoffRate, additionalRoundTimeIfProposalReceivedMs, - timeoutQuorumResolutionDelayMs); + timeoutQuorumResolutionDelayMs, + VertexStoreConfig.testingDefault()); } public static ConsensusConfig of(long pacemakerBaseTimeoutMs) { @@ -172,7 +177,8 @@ public static ConsensusConfig of( pacemakerBaseTimeoutMs, 2.0, additionalRoundTimeIfProposalReceivedMs, - pacemakerBaseTimeoutMs / 2); + pacemakerBaseTimeoutMs / 2, + VertexStoreConfig.testingDefault()); } public static ConsensusConfig of() { @@ -182,10 +188,11 @@ public static ConsensusConfig of() { pacemakerBaseTimeoutMs, 2.0, pacemakerBaseTimeoutMs /* double the timeout if proposal was received */, - 2000); + 2000, + VertexStoreConfig.testingDefault()); } - private AbstractModule asModule() { + public AbstractModule asModule() { return new AbstractModule() { @Override protected void configure() { @@ -199,6 +206,7 @@ protected void configure() { .annotatedWith(TimeoutQuorumResolutionDelayMs.class) .to(timeoutQuorumResolutionDelayMs); bindConstant().annotatedWith(PacemakerMaxExponent.class).to(0); + bind(VertexStoreConfig.class).toInstance(vertexStoreConfig); } }; } diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java index a919638579..e4f7ac0052 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java @@ -67,9 +67,9 @@ import com.google.inject.Inject; import com.radixdlt.consensus.LedgerHashes; import com.radixdlt.consensus.vertexstore.ExecutedVertex; -import com.radixdlt.consensus.vertexstore.VertexStoreState; import com.radixdlt.crypto.Hasher; import com.radixdlt.environment.EventDispatcher; +import com.radixdlt.lang.Option; import com.radixdlt.ledger.*; import com.radixdlt.ledger.StateComputerLedger.StateComputer; import com.radixdlt.ledger.StateComputerLedger.StateComputerPrepareResult; @@ -136,8 +136,9 @@ public StateComputerPrepareResult prepare( } @Override - public LedgerProofBundle commit(LedgerExtension ledgerExtension, VertexStoreState vertexStore) { - final var proof = this.stateComputer.commit(ledgerExtension, vertexStore); + public LedgerProofBundle commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + final var proof = this.stateComputer.commit(ledgerExtension, serializedVertexStoreState); this.mempool.handleTransactionsCommitted( ledgerExtension.transactions().stream() // This undoes the (hacky) re-mapping done by a fake `prepare()` using `MockExecuted` diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java index 027750c7ae..19214e2190 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java @@ -125,7 +125,7 @@ public StateComputerLedger.StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, VertexStoreState vertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { latestProof = new LedgerProofBundle( ledgerExtension.proof(), diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java index 75e53f6270..d8659096be 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java @@ -70,9 +70,9 @@ import com.radixdlt.consensus.bft.*; import com.radixdlt.consensus.bft.Round; import com.radixdlt.consensus.vertexstore.ExecutedVertex; -import com.radixdlt.consensus.vertexstore.VertexStoreState; import com.radixdlt.crypto.Hasher; import com.radixdlt.environment.EventDispatcher; +import com.radixdlt.lang.Option; import com.radixdlt.ledger.*; import com.radixdlt.ledger.StateComputerLedger.ExecutedTransaction; import com.radixdlt.ledger.StateComputerLedger.StateComputer; @@ -146,7 +146,8 @@ public StateComputerPrepareResult prepare( } @Override - public LedgerProofBundle commit(LedgerExtension ledgerExtension, VertexStoreState vertexStore) { - return this.stateComputer.commit(ledgerExtension, vertexStore); + public LedgerProofBundle commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + return this.stateComputer.commit(ledgerExtension, serializedVertexStoreState); } } diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java index 04640b7fcf..0cda0cb637 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java @@ -92,6 +92,7 @@ public final class StatelessComputer implements StateComputerLedger.StateCompute private final StatelessTransactionVerifier verifier; private final EventDispatcher ledgerUpdateDispatcher; private final Hasher hasher; + private final LedgerHashes fixedLedgerHashes; private int successCount = 0; private int invalidCount = 0; @@ -99,9 +100,18 @@ public StatelessComputer( StatelessTransactionVerifier verifier, EventDispatcher ledgerUpdateDispatcher, Hasher hasher) { + this(verifier, ledgerUpdateDispatcher, hasher, LedgerHashes.zero()); + } + + public StatelessComputer( + StatelessTransactionVerifier verifier, + EventDispatcher ledgerUpdateDispatcher, + Hasher hasher, + LedgerHashes fixedLedgerHashes) { this.verifier = verifier; this.ledgerUpdateDispatcher = ledgerUpdateDispatcher; this.hasher = hasher; + this.fixedLedgerHashes = fixedLedgerHashes; } public int getSuccessCount() { @@ -144,7 +154,7 @@ public StateComputerLedger.StateComputerPrepareResult prepare( invalidCount += invalidTransactionCount; return new StateComputerLedger.StateComputerPrepareResult( - successfulTransactions, invalidTransactionCount, LedgerHashes.zero()); + successfulTransactions, invalidTransactionCount, fixedLedgerHashes); } private LedgerUpdate generateLedgerUpdate(LedgerExtension ledgerExtension) { @@ -194,7 +204,7 @@ private LedgerUpdate generateLedgerUpdate(LedgerExtension ledgerExtension) { @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, VertexStoreState vertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { var ledgerUpdate = this.generateLedgerUpdate(ledgerExtension); ledgerUpdateDispatcher.dispatch(ledgerUpdate); return ledgerUpdate.committedProof(); diff --git a/core/src/test/java/com/radixdlt/consensus/bft/BFTHighQCUpdateTest.java b/core/src/test/java/com/radixdlt/consensus/bft/BFTHighQCUpdateTest.java deleted file mode 100644 index 66df67ddae..0000000000 --- a/core/src/test/java/com/radixdlt/consensus/bft/BFTHighQCUpdateTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.consensus.bft; - -import com.google.common.hash.HashCode; -import com.radixdlt.crypto.HashUtils; -import nl.jqno.equalsverifier.EqualsVerifier; -import org.junit.Test; - -public class BFTHighQCUpdateTest { - @Test - public void equalsContract() { - EqualsVerifier.forClass(BFTHighQCUpdate.class) - .withPrefabValues(HashCode.class, HashUtils.random256(), HashUtils.random256()) - .verify(); - } -} diff --git a/core/src/test/java/com/radixdlt/consensus/bft/BFTInsertUpdateTest.java b/core/src/test/java/com/radixdlt/consensus/bft/BFTInsertUpdateTest.java deleted file mode 100644 index 4d6da84820..0000000000 --- a/core/src/test/java/com/radixdlt/consensus/bft/BFTInsertUpdateTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/* Copyright 2021 Radix Publishing Ltd incorporated in Jersey (Channel Islands). - * - * Licensed under the Radix License, Version 1.0 (the "License"); you may not use this - * file except in compliance with the License. You may obtain a copy of the License at: - * - * radixfoundation.org/licenses/LICENSE-v1 - * - * The Licensor hereby grants permission for the Canonical version of the Work to be - * published, distributed and used under or by reference to the Licensor’s trademark - * Radix ® and use of any unregistered trade names, logos or get-up. - * - * The Licensor provides the Work (and each Contributor provides its Contributions) on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, - * including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, - * MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. - * - * Whilst the Work is capable of being deployed, used and adopted (instantiated) to create - * a distributed ledger it is your responsibility to test and validate the code, together - * with all logic and performance of that code under all foreseeable scenarios. - * - * The Licensor does not make or purport to make and hereby excludes liability for all - * and any representation, warranty or undertaking in any form whatsoever, whether express - * or implied, to any entity or person, including any representation, warranty or - * undertaking, as to the functionality security use, value or other characteristics of - * any distributed ledger nor in respect the functioning or value of any tokens which may - * be created stored or transferred using the Work. The Licensor does not warrant that the - * Work or any use of the Work complies with any law or regulation in any territory where - * it may be implemented or used or that it will be appropriate for any specific purpose. - * - * Neither the licensor nor any current or former employees, officers, directors, partners, - * trustees, representatives, agents, advisors, contractors, or volunteers of the Licensor - * shall be liable for any direct or indirect, special, incidental, consequential or other - * losses of any kind, in tort, contract or otherwise (including but not limited to loss - * of revenue, income or profits, or loss of use or data, or loss of reputation, or loss - * of any economic or other opportunity of whatsoever nature or howsoever arising), arising - * out of or in connection with (without limitation of any use, misuse, of any ledger system - * or use made or its functionality or any performance or operation of any code or protocol - * caused by bugs or programming or logic errors or otherwise); - * - * A. any offer, purchase, holding, use, sale, exchange or transmission of any - * cryptographic keys, tokens or assets created, exchanged, stored or arising from any - * interaction with the Work; - * - * B. any failure in a transmission or loss of any token or assets keys or other digital - * artefacts due to errors in transmission; - * - * C. bugs, hacks, logic errors or faults in the Work or any communication; - * - * D. system software or apparatus including but not limited to losses caused by errors - * in holding or transmitting tokens by any third-party; - * - * E. breaches or failure of security including hacker attacks, loss or disclosure of - * password, loss of private key, unauthorised use or misuse of such passwords or keys; - * - * F. any losses including loss of anticipated savings or other benefits resulting from - * use of the Work or any changes to the Work (however implemented). - * - * You are solely responsible for; testing, validating and evaluation of all operation - * logic, functionality, security and appropriateness of using the Work for any commercial - * or non-commercial purpose and for any reproduction or redistribution by You of the - * Work. You assume all risks associated with Your use of the Work and the exercise of - * permissions under this License. - */ - -package com.radixdlt.consensus.bft; - -import com.google.common.hash.HashCode; -import com.radixdlt.crypto.HashUtils; -import nl.jqno.equalsverifier.EqualsVerifier; -import org.junit.Test; - -public class BFTInsertUpdateTest { - @Test - public void equalsContract() { - EqualsVerifier.forClass(BFTInsertUpdate.class) - .withPrefabValues(HashCode.class, HashUtils.random256(), HashUtils.random256()) - .verify(); - } -} diff --git a/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java b/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java index da971f0f31..0a0237a183 100644 --- a/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java +++ b/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java @@ -84,6 +84,7 @@ import com.radixdlt.consensus.sync.*; import com.radixdlt.consensus.vertexstore.ExecutedVertex; import com.radixdlt.consensus.vertexstore.PersistentVertexStore; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.consensus.vertexstore.VertexStoreState; import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.crypto.Hasher; @@ -167,7 +168,7 @@ public StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, VertexStoreState vertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { // No-op // `closestEpochProofOnOrBefore` isn't really correct here, but that's fine return new LedgerProofBundle( @@ -191,8 +192,6 @@ protected void configure() { .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) .toInstance(rmock(EventDispatcher.class)); - bind(new TypeLiteral>() {}) - .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) @@ -257,6 +256,7 @@ protected void configure() { bindConstant().annotatedWith(PacemakerMaxExponent.class).to(0); bindConstant().annotatedWith(AdditionalRoundTimeIfProposalReceivedMs.class).to(10L); bindConstant().annotatedWith(TimeoutQuorumResolutionDelayMs.class).to(10L); + bind(VertexStoreConfig.class).toInstance(VertexStoreConfig.testingDefault()); bind(TimeSupplier.class).toInstance(System::currentTimeMillis); bind(new TypeLiteral>() {}).toInstance(rmock(Consumer.class)); diff --git a/core/src/test/java/com/radixdlt/consensus/liveness/PacemakerTest.java b/core/src/test/java/com/radixdlt/consensus/liveness/PacemakerTest.java index b166223bc9..1849f62d54 100644 --- a/core/src/test/java/com/radixdlt/consensus/liveness/PacemakerTest.java +++ b/core/src/test/java/com/radixdlt/consensus/liveness/PacemakerTest.java @@ -208,14 +208,12 @@ public void when_local_timeout__then_send_empty_vote_if_no_previous() { when(bftHeader.getRound()).thenReturn(round); HighQC highQC = mock(HighQC.class); BFTInsertUpdate bftInsertUpdate = mock(BFTInsertUpdate.class); - when(bftInsertUpdate.getHeader()).thenReturn(bftHeader); ExecutedVertex executedVertex = mock(ExecutedVertex.class); when(executedVertex.getRound()).thenReturn(round); when(executedVertex.getLedgerHeader()).thenReturn(mock(LedgerHeader.class)); VertexStoreState vertexStoreState = mock(VertexStoreState.class); when(vertexStoreState.getHighQC()).thenReturn(highQC); - when(bftInsertUpdate.getInserted()).thenReturn(executedVertex); - when(bftInsertUpdate.getVertexStoreState()).thenReturn(vertexStoreState); + when(bftInsertUpdate.insertedVertex()).thenReturn(executedVertex); final var vertexHash = hasher.hashDsonEncoded(Vertex.createFallback(highestQc, round, leader)); when(executedVertex.getVertexHash()).thenReturn(vertexHash); diff --git a/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java b/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java index 612c479cba..7044c938a4 100644 --- a/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java @@ -79,15 +79,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; -import com.radixdlt.consensus.bft.BFTCommittedUpdate; -import com.radixdlt.consensus.bft.BFTHighQCUpdate; -import com.radixdlt.consensus.bft.BFTInsertUpdate; -import com.radixdlt.consensus.bft.BFTRebuildUpdate; -import com.radixdlt.consensus.bft.BFTValidatorId; -import com.radixdlt.consensus.bft.Round; +import com.radixdlt.consensus.bft.*; +import com.radixdlt.crypto.HashUtils; import com.radixdlt.crypto.Hasher; import com.radixdlt.environment.EventDispatcher; +import com.radixdlt.monitoring.Metrics; +import com.radixdlt.monitoring.MetricsInitializer; import com.radixdlt.serialization.DefaultSerialization; +import com.radixdlt.serialization.Serialization; import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.ZeroHasher; import java.util.List; @@ -111,14 +110,19 @@ public class VertexStoreTest { private EventDispatcher bftUpdateSender; private EventDispatcher rebuildUpdateEventDispatcher; private EventDispatcher bftHighQCUpdateEventDispatcher; - private EventDispatcher committedSender; - private Hasher hasher = new Blake2b256Hasher(DefaultSerialization.getInstance()); + private Serialization serialization = DefaultSerialization.getInstance(); + private Hasher hasher = new Blake2b256Hasher(serialization); + private Metrics metrics = new MetricsInitializer().initialize(); private static final LedgerHeader MOCKED_HEADER = LedgerHeader.create(0, Round.epochInitial(), 0, LedgerHashes.zero(), 0, 0); @Before public void setUp() { + setUp(VertexStoreConfig.testingDefault()); + } + + public void setUp(VertexStoreConfig vertexStoreConfig) { // No type check issues with mocking generic here Ledger ssc = mock(Ledger.class); this.ledger = ssc; @@ -135,23 +139,24 @@ public void setUp() { this.bftUpdateSender = rmock(EventDispatcher.class); this.rebuildUpdateEventDispatcher = rmock(EventDispatcher.class); this.bftHighQCUpdateEventDispatcher = rmock(EventDispatcher.class); - this.committedSender = rmock(EventDispatcher.class); this.genesisVertex = Vertex.createInitialEpochVertex(MOCKED_HEADER).withId(ZeroHasher.INSTANCE); this.genesisHash = genesisVertex.hash(); this.rootQC = QuorumCertificate.createInitialEpochQC(genesisVertex, MOCKED_HEADER); this.underlyingVertexStore = - VertexStoreJavaImpl.create( - VertexStoreState.create(HighQC.ofInitialEpochQc(rootQC), genesisVertex, hasher), + new VertexStoreJavaImpl( ledger, - hasher); + hasher, + serialization, + metrics, + vertexStoreConfig, + VertexStoreState.create(HighQC.ofInitialEpochQc(rootQC), genesisVertex, hasher)); this.vertexStoreAdapter = new VertexStoreAdapter( underlyingVertexStore, bftHighQCUpdateEventDispatcher, bftUpdateSender, - rebuildUpdateEventDispatcher, - committedSender); + rebuildUpdateEventDispatcher); AtomicReference lastParentHeader = new AtomicReference<>(new BFTHeader(Round.epochInitial(), genesisHash, MOCKED_HEADER)); @@ -321,12 +326,16 @@ public void adding_a_qc_with_commit_should_commit_vertices_to_ledger() { assertThat(vertexStoreAdapter.highQC().highestCommittedQC()).isEqualTo(qc); assertThat(vertexStoreAdapter.getVertices(vertices.get(2).hash(), 3)) .hasValue(ImmutableList.of(vertices.get(2), vertices.get(1), vertices.get(0))); - verify(committedSender, times(1)) + verify(bftHighQCUpdateEventDispatcher, times(1)) .dispatch( argThat( u -> - u.committed().size() == 1 - && u.committed().get(0).getVertexWithHash().equals(vertices.get(0)))); + u.committedVertices().unwrap().size() == 1 + && u.committedVertices() + .unwrap() + .get(0) + .getVertexWithHash() + .equals(vertices.get(0)))); assertTrue(isVertexStoreChildrenMappingTidy(underlyingVertexStore)); } @@ -377,7 +386,7 @@ public void rebuilding_should_emit_updates() { .dispatch( argThat( u -> { - List sentVertices = u.getVertexStoreState().getVertices(); + List sentVertices = u.vertexStoreState().getVertices(); return sentVertices.equals(vertices.subList(1, vertices.size())); })); } @@ -398,4 +407,32 @@ public void inserting_a_tc_should_only_replace_tcs_for_lower_rounds() { vertexStoreAdapter.insertTimeoutCertificate(initialTC); assertEquals(higherTC, vertexStoreAdapter.highQC().highestTC().orElse(null)); } + + @Test + public void vertex_store_should_not_insert_a_vertex_when_size_limit_is_reached() { + final var config = new VertexStoreConfig(1_000_000); + setUp(config); + final var parentHeader = new BFTHeader(Round.epochInitial(), genesisHash, MOCKED_HEADER); + final var parent = createVertex(parentHeader, parentHeader, parentHeader, new byte[] {0}); + vertexStoreAdapter.insertVertex(parent); + + // Try to insert as many vertices as possible + var numInserted = 0; + var inserted = true; + while (inserted) { + final var vertex = + createVertex( + parentHeader, parentHeader, mockedHeaderOf(parent), HashUtils.random256().asBytes()); + inserted = vertexStoreAdapter.insertVertex(vertex); + numInserted += 1; + } + + // We're expecting around 650 vertices to be inserted (no need to check for a specific value) + assertTrue(numInserted > 600 && numInserted < 700); + + // And the size should be close to the limit + final var size = vertexStoreAdapter.getCurrentSerializedSizeBytes(); + assertTrue( + size < config.maxSerializedSizeBytes() && size + 5000 > config.maxSerializedSizeBytes()); + } } diff --git a/core/src/test/java/com/radixdlt/modules/ConsensusModuleTest.java b/core/src/test/java/com/radixdlt/modules/ConsensusModuleTest.java index 117edfaaa5..496d87c4da 100644 --- a/core/src/test/java/com/radixdlt/modules/ConsensusModuleTest.java +++ b/core/src/test/java/com/radixdlt/modules/ConsensusModuleTest.java @@ -84,6 +84,7 @@ import com.radixdlt.consensus.sync.*; import com.radixdlt.consensus.vertexstore.PersistentVertexStore; import com.radixdlt.consensus.vertexstore.VertexStoreAdapter; +import com.radixdlt.consensus.vertexstore.VertexStoreConfig; import com.radixdlt.consensus.vertexstore.VertexStoreState; import com.radixdlt.crypto.ECKeyPair; import com.radixdlt.crypto.Hasher; @@ -178,8 +179,6 @@ protected void configure() { .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) .toInstance(rmock(EventDispatcher.class)); - bind(new TypeLiteral>() {}) - .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) .toInstance(rmock(EventDispatcher.class)); bind(new TypeLiteral>() {}) @@ -237,6 +236,7 @@ protected void configure() { bindConstant().annotatedWith(PacemakerMaxExponent.class).to(6); bindConstant().annotatedWith(AdditionalRoundTimeIfProposalReceivedMs.class).to(1000L); bindConstant().annotatedWith(TimeoutQuorumResolutionDelayMs.class).to(1000L); + bind(VertexStoreConfig.class).toInstance(VertexStoreConfig.testingDefault()); ECKeyPair ecKeyPair = ECKeyPair.generateNew(); bind(HashSigner.class).toInstance(ecKeyPair::sign); diff --git a/docker/config/default.config.envsubst b/docker/config/default.config.envsubst index 5bd31df0bc..e34fcdad75 100644 --- a/docker/config/default.config.envsubst +++ b/docker/config/default.config.envsubst @@ -44,3 +44,5 @@ testing_forks.fork_config_name=$RADIXDLT_TESTING_FORK_CONFIG_NAME consensus.use_genesis_for_validator_address=$RADIXDLT_CONSENSUS_USE_GENESIS_FOR_VALIDATOR_ADDRESS consensus.validator_address=$RADIXDLT_CONSENSUS_VALIDATOR_ADDRESS + +bft.vertex_store.max_serialized_size_bytes=$RADIXDLT_BFT_VERTEX_STORE_MAX_SERIALIZED_SIZE_BYTES From 783d36fcee57171406e99dba64030f3e6d21e756 Mon Sep 17 00:00:00 2001 From: Lukasz Gasior Date: Wed, 6 Mar 2024 17:39:19 +0100 Subject: [PATCH 2/4] Rename vertex store size metric to vertexCount --- common/src/main/java/com/radixdlt/monitoring/Metrics.java | 2 +- .../consensus/DivergentExecutionLivenessBreakTest.java | 4 ++-- .../radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/com/radixdlt/monitoring/Metrics.java b/common/src/main/java/com/radixdlt/monitoring/Metrics.java index 736c10de9c..46b1b3ae0f 100644 --- a/common/src/main/java/com/radixdlt/monitoring/Metrics.java +++ b/common/src/main/java/com/radixdlt/monitoring/Metrics.java @@ -233,7 +233,7 @@ public record Sync( Counter invalidEpochInitialQcSyncStates) {} public record VertexStore( - Gauge size, + Gauge vertexCount, Gauge byteSize, Counter forks, Counter rebuilds, diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java index 68490431ee..9e7433385a 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java @@ -285,7 +285,7 @@ public void test_divergent_execution_liveness_break_and_recovery() { assertTrue(metrics.bft().divergentVertexExecutions().getSum() > 1); // Cross-check another metric to verify that vertex store // indeed holds more vertices than expected in a healthy scenario. - assertTrue(metrics.bft().vertexStore().size().get() >= 20); + assertTrue(metrics.bft().vertexStore().vertexCount().get() >= 20); }); // Another verification that we're in a liveness break @@ -334,7 +334,7 @@ public void test_divergent_execution_liveness_break_and_recovery() { // No more errors due to size limit assertEquals(0, (int) metrics.bft().vertexStore().errorsDueToSizeLimit().get()); // There are no more than 3 vertices (as expected in a healthy network) - assertTrue(metrics.bft().vertexStore().size().get() <= 3); + assertTrue(metrics.bft().vertexStore().vertexCount().get() <= 3); // No timeouts assertEquals( 0, diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java index cd148baf9b..63076a44a7 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java @@ -178,7 +178,7 @@ public Result tryRebuild(VertexStoreState vertexSt this.executedVertices.put(executedVertex.getVertexHash(), executedVertex); } - metrics.bft().vertexStore().size().set(vertexStoreState.getVertices().size()); + metrics.bft().vertexStore().vertexCount().set(vertexStoreState.getVertices().size()); metrics.bft().vertexStore().rebuilds().inc(); return Result.success(new RebuildSummary(vertexStoreState, serializedVertexStoreState)); @@ -217,7 +217,7 @@ public InsertQcResult insertQc(QuorumCertificate qc) { committedUpdate = Option.empty(); } - metrics.bft().vertexStore().size().set(vertices.size()); + metrics.bft().vertexStore().vertexCount().set(vertices.size()); if (isHighQC || committedUpdate.isPresent()) { // We have either received a new highQc, or some vertices @@ -402,7 +402,7 @@ private Result insertVertexInternal( trackCurrentStateSize(postInsertSerializedState); // Update the metrics - metrics.bft().vertexStore().size().set(vertices.size()); + metrics.bft().vertexStore().vertexCount().set(vertices.size()); if (siblings.size() > 1) { metrics.bft().vertexStore().forks().inc(); } From a79c14bdd8ab2fff3e42c72f3ea93c48b9d52bb6 Mon Sep 17 00:00:00 2001 From: Lukasz Gasior Date: Wed, 6 Mar 2024 17:54:48 +0100 Subject: [PATCH 3/4] Use Multimap for VertexStore's vertexChildren --- .../java/com/radixdlt/RadixNodeModule.java | 4 +- .../vertexstore/VertexStoreJavaImpl.java | 42 +++++++------------ 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/core/src/main/java/com/radixdlt/RadixNodeModule.java b/core/src/main/java/com/radixdlt/RadixNodeModule.java index db1ebfb404..3e81eb298a 100644 --- a/core/src/main/java/com/radixdlt/RadixNodeModule.java +++ b/core/src/main/java/com/radixdlt/RadixNodeModule.java @@ -167,8 +167,8 @@ protected void configure() { Preconditions.checkArgument( vertexStoreConfig.maxSerializedSizeBytes() >= VertexStoreConfig.MIN_MAX_SERIALIZED_SIZE_BYTES, - "Invalid configuration: bft.vertex_store.max_serialized_size_byte must be at least " - + VertexStoreConfig.MIN_MAX_SERIALIZED_SIZE_BYTES); + "Invalid configuration: bft.vertex_store.max_serialized_size_byte must be at least {}", + VertexStoreConfig.MIN_MAX_SERIALIZED_SIZE_BYTES); // System (e.g. time, random) install(new SystemModule()); diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java index 63076a44a7..e74230ba74 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java @@ -65,7 +65,9 @@ package com.radixdlt.consensus.vertexstore; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Multimap; import com.google.common.hash.HashCode; import com.radixdlt.consensus.BFTHeader; import com.radixdlt.consensus.HighQC; @@ -99,7 +101,7 @@ public final class VertexStoreJavaImpl implements VertexStore { private final VertexStoreConfig config; private final Map vertices = new HashMap<>(); - private final Map> vertexChildren = new HashMap<>(); + private final Multimap vertexChildren = HashMultimap.create(); private final Map executedVertices = new HashMap<>(); // These should never be null @@ -133,13 +135,10 @@ private void resetToState(VertexStoreState state, WrappedByteArray serializedSta this.vertices.clear(); this.executedVertices.clear(); this.vertexChildren.clear(); - this.vertexChildren.put(rootVertex.hash(), new HashSet<>()); for (var vertexWithHash : state.getVertices()) { this.vertices.put(vertexWithHash.hash(), vertexWithHash); - this.vertexChildren.put(vertexWithHash.hash(), new HashSet<>()); - var siblings = this.vertexChildren.get(vertexWithHash.vertex().getParentVertexId()); - siblings.add(vertexWithHash.hash()); + this.vertexChildren.put(vertexWithHash.vertex().getParentVertexId(), vertexWithHash.hash()); } trackCurrentStateSize(serializedState); @@ -190,7 +189,7 @@ public InsertQcResult insertQc(QuorumCertificate qc) { return new VertexStore.InsertQcResult.VertexIsMissing(); } - final var hasAnyChildren = !vertexChildren.get(qc.getProposedHeader().getVertexId()).isEmpty(); + final var hasAnyChildren = vertexChildren.containsKey(qc.getProposedHeader().getVertexId()); if (hasAnyChildren) { // TODO: Check to see if qc's match in case there's a fault return new VertexStore.InsertQcResult.Ignored(); @@ -393,9 +392,7 @@ private Result insertVertexInternal( // The vertex was executed successfully, so we're inserting it vertices.put(executedVertex.getVertexHash(), executedVertex.getVertexWithHash()); executedVertices.put(executedVertex.getVertexHash(), executedVertex); - vertexChildren.put(executedVertex.getVertexHash(), new HashSet<>()); - Set siblings = vertexChildren.get(executedVertex.getParentId()); - siblings.add(executedVertex.getVertexHash()); + vertexChildren.put(executedVertex.getParentId(), executedVertex.getVertexHash()); // We've already calculated the post-insert state (and verified // its size against the limit), so we can just use it here. @@ -403,7 +400,8 @@ private Result insertVertexInternal( // Update the metrics metrics.bft().vertexStore().vertexCount().set(vertices.size()); - if (siblings.size() > 1) { + final var vertexAndSiblings = vertexChildren.get(executedVertex.getParentId()); + if (vertexAndSiblings.size() > 1) { metrics.bft().vertexStore().forks().inc(); } if (!vertexWithHash.vertex().hasDirectParent()) { @@ -417,10 +415,9 @@ private Result insertVertexInternal( private void removeVertexAndPruneInternal(HashCode vertexId, Optional skip) { Optional.ofNullable(vertices.remove(vertexId)) - .flatMap( + .ifPresent( removedVertex -> - Optional.ofNullable(vertexChildren.get(removedVertex.vertex().getParentVertexId()))) - .ifPresent(siblings -> siblings.remove(vertexId)); + vertexChildren.remove(removedVertex.vertex().getParentVertexId(), vertexId)); executedVertices.remove(vertexId); @@ -428,12 +425,10 @@ private void removeVertexAndPruneInternal(HashCode vertexId, Optional return; } - var children = vertexChildren.remove(vertexId); - if (children != null) { - for (HashCode child : children) { - if (!skip.map(child::equals).orElse(false)) { - removeVertexAndPruneInternal(child, Optional.empty()); - } + final var children = vertexChildren.removeAll(vertexId); + for (HashCode child : children) { + if (!Optional.of(child).equals(skip)) { + removeVertexAndPruneInternal(child, Optional.empty()); } } } @@ -480,13 +475,8 @@ private VertexStoreState getState() { private void getChildrenVerticesList( VertexWithHash parent, ImmutableList.Builder builder) { - Set childrenIds = this.vertexChildren.get(parent.hash()); - if (childrenIds == null) { - return; - } - - for (HashCode childId : childrenIds) { - final var v = vertices.get(childId); + for (HashCode child : this.vertexChildren.get(parent.hash())) { + final var v = vertices.get(child); builder.add(v); getChildrenVerticesList(v, builder); } From 1ad56a4e9702ba8361e220707ab11b9b383b4d46 Mon Sep 17 00:00:00 2001 From: Lukasz Gasior Date: Wed, 12 Jun 2024 14:44:22 +0200 Subject: [PATCH 4/4] Address code review comments --- .../DivergentExecutionLivenessBreakTest.java | 4 +- .../consensus/bft/BFTHighQCUpdate.java | 14 ++++- .../consensus/bft/BFTInsertUpdate.java | 9 ++- .../consensus/bft/BFTRebuildUpdate.java | 10 ++- .../com/radixdlt/consensus/sync/BFTSync.java | 7 ++- .../vertexstore/VertexStoreAdapter.java | 32 ++++++---- .../vertexstore/VertexStoreJavaImpl.java | 10 ++- .../vertexstore/VertexStoreState.java | 15 ++--- .../radixdlt/ledger/StateComputerLedger.java | 62 ++++++++++--------- .../com/radixdlt/rev2/REv2StateComputer.java | 5 +- .../monitors/consensus/LivenessInvariant.java | 5 -- .../MockedMempoolStateComputer.java | 3 +- .../statecomputer/MockedStateComputer.java | 3 +- .../MockedStateComputerWithEpochs.java | 3 +- .../statecomputer/StatelessComputer.java | 3 +- .../consensus/epoch/EpochManagerTest.java | 3 +- .../vertexstore/VertexStoreTest.java | 17 ++--- 17 files changed, 127 insertions(+), 78 deletions(-) diff --git a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java index 54e2f77ba3..4fe001f045 100644 --- a/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java +++ b/core/src/integration/java/com/radixdlt/integration/steady_state/deterministic/consensus/DivergentExecutionLivenessBreakTest.java @@ -96,6 +96,7 @@ import com.radixdlt.rev2.REv2StateComputer; import com.radixdlt.rev2.REv2TransactionsAndProofReader; import com.radixdlt.transactions.RawNotarizedTransaction; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; import java.util.Random; import java.util.function.Consumer; @@ -195,7 +196,8 @@ public StateComputerLedger.StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, + Option serializedVertexStoreState) { return underlyingStateComputer.commit(ledgerExtension, serializedVertexStoreState); } }; diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java index c5cc9ac910..6d249a574b 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTHighQCUpdate.java @@ -70,6 +70,7 @@ import com.radixdlt.consensus.vertexstore.ExecutedVertex; import com.radixdlt.lang.Option; import com.radixdlt.utils.WrappedByteArray; +import java.util.AbstractCollection; /** * An event emitted when vertex store updates its highQC, which possibly results in some vertices @@ -79,4 +80,15 @@ public record BFTHighQCUpdate( HighQC newHighQc, Option> committedVertices, WrappedByteArray serializedVertexStoreState) - implements LocalEvent {} + implements LocalEvent { + + @Override + public String toString() { + return String.format( + "%s[newHighQc=%s numCommittedVertices=%s serializedVertexStoreStateSize=%s]", + getClass().getSimpleName(), + newHighQc, + committedVertices.map(AbstractCollection::size).orElse(0), + serializedVertexStoreState.size()); + } +} diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java index a7c4e144ba..e42c5906fc 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTInsertUpdate.java @@ -71,4 +71,11 @@ /** An event emitted after a vertex has been inserted into the vertex store. */ public record BFTInsertUpdate( ExecutedVertex insertedVertex, WrappedByteArray serializedVertexStoreState) - implements LocalEvent {} + implements LocalEvent { + @Override + public String toString() { + return String.format( + "%s[insertedVertex=%s serializedVertexStoreStateSize=%s]", + getClass().getSimpleName(), insertedVertex(), serializedVertexStoreState.size()); + } +} diff --git a/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java b/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java index 8881395334..a821d4ea40 100644 --- a/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java +++ b/core/src/main/java/com/radixdlt/consensus/bft/BFTRebuildUpdate.java @@ -71,4 +71,12 @@ /** An even emitted when the vertex store has been rebuilt. */ public record BFTRebuildUpdate( VertexStoreState vertexStoreState, WrappedByteArray serializedVertexStoreState) - implements LocalEvent {} + implements LocalEvent { + + @Override + public String toString() { + return String.format( + "%s[serializedVertexStoreStateSize=%s]", + getClass().getSimpleName(), serializedVertexStoreState.size()); + } +} diff --git a/core/src/main/java/com/radixdlt/consensus/sync/BFTSync.java b/core/src/main/java/com/radixdlt/consensus/sync/BFTSync.java index 259e6aa206..eb38a88ab5 100644 --- a/core/src/main/java/com/radixdlt/consensus/sync/BFTSync.java +++ b/core/src/main/java/com/radixdlt/consensus/sync/BFTSync.java @@ -67,6 +67,7 @@ import static java.util.function.Predicate.not; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.hash.HashCode; import com.google.common.util.concurrent.RateLimiter; import com.radixdlt.consensus.*; @@ -244,7 +245,7 @@ public SyncResult syncToQC(HighQC highQC, @Nullable NodeId author, HighQcSource return SyncResult.INVALID; } - return switch (vertexStore.insertQc(qc)) { + return switch (vertexStore.insertQuorumCertificate(qc)) { case VertexStore.InsertQcResult.Inserted ignored -> { // QC was inserted, try TC too (as it can be higher), and then process a new highQC highQC.highestTC().map(vertexStore::insertTimeoutCertificate); @@ -464,8 +465,8 @@ private void rebuildAndSyncQC(SyncState syncState) { // TODO: check if there are any vertices which haven't been local sync processed yet if (requiresLedgerSync(syncState)) { syncState.fetched.sort(Comparator.comparing(v -> v.vertex().getRound())); - ImmutableList nonRootVertices = - syncState.fetched.stream().skip(1).collect(ImmutableList.toImmutableList()); + ImmutableSet nonRootVertices = + syncState.fetched.stream().skip(1).collect(ImmutableSet.toImmutableSet()); final var syncStateHighestCommittedQc = syncState.highQC.highestCommittedQC(); final var syncStateHighestTc = syncState.highQC.highestTC(); diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java index e1a3482ab7..d00a7416d8 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreAdapter.java @@ -133,10 +133,15 @@ public boolean insertTimeoutCertificate(TimeoutCertificate timeoutCertificate) { }; } - public VertexStore.InsertQcResult insertQc(QuorumCertificate qc) { + public VertexStore.InsertQcResult insertQuorumCertificate(QuorumCertificate qc) { final var result = vertexStore.insertQc(qc); - if (result instanceof VertexStore.InsertQcResult.Inserted inserted) { - dispatchPostQcInsertionEvents(inserted); + switch (result) { + case VertexStore.InsertQcResult.Inserted inserted -> this.highQCUpdateDispatcher.dispatch( + new BFTHighQCUpdate( + inserted.newHighQc(), + inserted.committedUpdate().map(VertexStore.CommittedUpdate::committedVertices), + inserted.serializedVertexStoreState())); + default -> {} // no-op } return result; } @@ -151,18 +156,21 @@ public List getPathFromRoot(HashCode vertexHash) { public void insertVertexChain(VertexChain vertexChain) { final var result = vertexStore.insertVertexChain(vertexChain); - result.insertedQcs().forEach(this::dispatchPostQcInsertionEvents); + result + .insertedQcs() + .forEach( + inserted -> { + this.highQCUpdateDispatcher.dispatch( + new BFTHighQCUpdate( + inserted.newHighQc(), + inserted + .committedUpdate() + .map(VertexStore.CommittedUpdate::committedVertices), + inserted.serializedVertexStoreState())); + }); result.insertUpdates().forEach(bftUpdateDispatcher::dispatch); } - private void dispatchPostQcInsertionEvents(VertexStore.InsertQcResult.Inserted inserted) { - this.highQCUpdateDispatcher.dispatch( - new BFTHighQCUpdate( - inserted.newHighQc(), - inserted.committedUpdate().map(VertexStore.CommittedUpdate::committedVertices), - inserted.serializedVertexStoreState())); - } - public Optional> getVertices(HashCode vertexId, int count) { return vertexStore.getVertices(vertexId, count).toOptional(); } diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java index e74230ba74..ad7f8190df 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreJavaImpl.java @@ -67,6 +67,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multimap; import com.google.common.hash.HashCode; import com.radixdlt.consensus.BFTHeader; @@ -425,7 +426,10 @@ private void removeVertexAndPruneInternal(HashCode vertexId, Optional return; } - final var children = vertexChildren.removeAll(vertexId); + // Note that we need a copy of the children list here (.stream().toList()) - that is because + // we're removing the items in the loop (otherwise, this could lead to + // ConcurrentModificationException) + final var children = vertexChildren.get(vertexId).stream().toList(); for (HashCode child : children) { if (!Optional.of(child).equals(skip)) { removeVertexAndPruneInternal(child, Optional.empty()); @@ -468,13 +472,13 @@ public boolean containsVertex(HashCode vertexId) { private VertexStoreState getState() { // TODO: store list dynamically rather than recomputing - ImmutableList.Builder verticesBuilder = ImmutableList.builder(); + ImmutableSet.Builder verticesBuilder = ImmutableSet.builder(); getChildrenVerticesList(this.rootVertex, verticesBuilder); return VertexStoreState.create(this.highQC(), this.rootVertex, verticesBuilder.build(), hasher); } private void getChildrenVerticesList( - VertexWithHash parent, ImmutableList.Builder builder) { + VertexWithHash parent, ImmutableSet.Builder builder) { for (HashCode child : this.vertexChildren.get(parent.hash())) { final var v = vertices.get(child); builder.add(v); diff --git a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java index b7b61bd863..211e42e435 100644 --- a/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java +++ b/core/src/main/java/com/radixdlt/consensus/vertexstore/VertexStoreState.java @@ -67,6 +67,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.Round; @@ -94,10 +95,10 @@ public final class VertexStoreState { private final VertexWithHash root; private final HighQC highQC; - private final ImmutableList vertices; + private final ImmutableSet vertices; private VertexStoreState( - HighQC highQC, VertexWithHash root, ImmutableList vertices) { + HighQC highQC, VertexWithHash root, ImmutableSet vertices) { this.highQC = highQC; this.root = root; this.vertices = vertices; @@ -123,11 +124,11 @@ public static VertexStoreState createNewForNextEpoch( } public static VertexStoreState create(HighQC highQC, VertexWithHash root, Hasher hasher) { - return create(highQC, root, ImmutableList.of(), hasher); + return create(highQC, root, ImmutableSet.of(), hasher); } public static VertexStoreState create( - HighQC highQC, VertexWithHash root, ImmutableList vertices, Hasher hasher) { + HighQC highQC, VertexWithHash root, ImmutableSet vertices, Hasher hasher) { final var processedQcCommit = highQC .highestCommittedQC() @@ -216,7 +217,7 @@ public VertexStoreState withVertex(VertexWithHash vertex) { return new VertexStoreState( this.highQC, this.root, - ImmutableList.builder().addAll(this.vertices).add(vertex).build()); + ImmutableSet.builder().addAll(this.vertices).add(vertex).build()); } public SerializedVertexStoreState toSerialized() { @@ -236,7 +237,7 @@ public VertexWithHash getRoot() { return root; } - public ImmutableList getVertices() { + public ImmutableSet getVertices() { return vertices; } @@ -309,7 +310,7 @@ public VertexStoreState toVertexStoreState(Hasher hasher) { var rootWithHash = root.withId(hasher); var verticesWithHash = - vertices.stream().map(v -> v.withId(hasher)).collect(ImmutableList.toImmutableList()); + vertices.stream().map(v -> v.withId(hasher)).collect(ImmutableSet.toImmutableSet()); return VertexStoreState.create(highQC, rootWithHash, verticesWithHash, hasher); } diff --git a/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java b/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java index a8cb50d9a5..2460bfd189 100644 --- a/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java +++ b/core/src/main/java/com/radixdlt/ledger/StateComputerLedger.java @@ -85,6 +85,7 @@ import com.radixdlt.transactions.RawLedgerTransaction; import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.TimeSupplier; +import com.radixdlt.utils.WrappedByteArray; import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -162,7 +163,7 @@ StateComputerPrepareResult prepare( RoundDetails roundDetails); LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState); + LedgerExtension ledgerExtension, Option serializedVertexStoreState); } private final StateComputer stateComputer; @@ -301,40 +302,21 @@ public EventProcessor bftHighQcUpdateEventProcessor() { .ifPresent( committedVertices -> { updateCommittedVerticesMetrics(committedVertices); - - final ImmutableList transactions = - committedVertices.stream() - .flatMap(ExecutedVertex::successfulTransactions) - .map(ExecutedTransaction::transaction) - .collect(ImmutableList.toImmutableList()); - highQcUpdate .newHighQc() .highestCommittedQC() .getProcessedCommit(hasher) .ifPresentOrElse( processedQcCommit -> { - switch (processedQcCommit) { - case ProcessedQcCommit.OfConensusQc ofConensusQc -> { - final var ledgerExtension = - new LedgerExtension(transactions, ofConensusQc.ledgerProof()); - metrics - .ledger() - .commit() - .measure( - () -> - this.commit( - ledgerExtension, - Option.some( - highQcUpdate - .serializedVertexStoreState() - .value()))); - } - case ProcessedQcCommit.OfInitialEpochQc ofInitialEpochQc -> { - // no-op, ignore - this.metrics.ledger().ignoredBftCommittedUpdates().inc(); - } - } + final var transactions = + committedVertices.stream() + .flatMap(ExecutedVertex::successfulTransactions) + .map(ExecutedTransaction::transaction) + .collect(ImmutableList.toImmutableList()); + processQcCommit( + processedQcCommit, + transactions, + highQcUpdate.serializedVertexStoreState()); }, () -> { // no-op, ignore @@ -344,6 +326,25 @@ public EventProcessor bftHighQcUpdateEventProcessor() { }; } + private void processQcCommit( + ProcessedQcCommit processedQcCommit, + ImmutableList transactions, + WrappedByteArray serializedVertexStoreState) { + switch (processedQcCommit) { + case ProcessedQcCommit.OfConensusQc ofConensusQc -> { + final var ledgerExtension = new LedgerExtension(transactions, ofConensusQc.ledgerProof()); + metrics + .ledger() + .commit() + .measure(() -> this.commit(ledgerExtension, Option.some(serializedVertexStoreState))); + } + case ProcessedQcCommit.OfInitialEpochQc ofInitialEpochQc -> { + // no-op, ignore + this.metrics.ledger().ignoredBftCommittedUpdates().inc(); + } + } + } + private void updateCommittedVerticesMetrics(ImmutableList committedVertices) { final var numCommittedFallbackVertices = committedVertices.stream().filter(v -> v.vertex().isFallback()).count(); @@ -396,7 +397,8 @@ public EventProcessor syncEventProcessor() { * InvalidCommitRequestException} is propagated from the Rust state computer. * */ - private void commit(LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + private void commit( + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { final var proofToCommit = ledgerExtension.proof(); final int extensionTransactionCount; // for metrics purposes only diff --git a/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java b/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java index 7d97f5fee2..d6795406db 100644 --- a/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java +++ b/core/src/main/java/com/radixdlt/rev2/REv2StateComputer.java @@ -90,6 +90,7 @@ import com.radixdlt.transactions.PreparedNotarizedTransaction; import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.UInt64; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; @@ -283,7 +284,7 @@ public StateComputerLedger.StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { final var proof = ledgerExtension.proof(); final var header = proof.ledgerHeader(); @@ -291,7 +292,7 @@ public LedgerProofBundle commit( new CommitRequest( ledgerExtension.transactions(), proof, - serializedVertexStoreState, + serializedVertexStoreState.map(WrappedByteArray::value), Option.from(selfValidatorId)); final var result = stateComputer.commit(commitRequest); diff --git a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java index 094da7c690..af160b94af 100644 --- a/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java +++ b/core/src/test-core/java/com/radixdlt/harness/simulation/monitors/consensus/LivenessInvariant.java @@ -101,11 +101,6 @@ public Observable check(RunningNetwork network) { emitter.onNext(highQCUpdate.newHighQc().highestQC()); }, BFTHighQCUpdate.class); - nodeEvents.addListener( - (node, highQcUpdate) -> { - emitter.onNext(highQcUpdate.newHighQc().highestQC()); - }, - BFTHighQCUpdate.class); }) .serialize() .map(QuorumCertificate::getProposedHeader) diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java index e4f7ac0052..9dda57b234 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedMempoolStateComputer.java @@ -78,6 +78,7 @@ import com.radixdlt.p2p.NodeId; import com.radixdlt.targeted.mempool.SimpleMempool; import com.radixdlt.transactions.RawNotarizedTransaction; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; import java.util.Set; import javax.annotation.Nullable; @@ -137,7 +138,7 @@ public StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { final var proof = this.stateComputer.commit(ledgerExtension, serializedVertexStoreState); this.mempool.handleTransactionsCommitted( ledgerExtension.transactions().stream() diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java index 19214e2190..46a257bda1 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputer.java @@ -82,6 +82,7 @@ import com.radixdlt.statecomputer.commit.CommitSummary; import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.UInt32; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -125,7 +126,7 @@ public StateComputerLedger.StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { latestProof = new LedgerProofBundle( ledgerExtension.proof(), diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java index d8659096be..f89f5312eb 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/MockedStateComputerWithEpochs.java @@ -80,6 +80,7 @@ import com.radixdlt.mempool.MempoolAdd; import com.radixdlt.p2p.NodeId; import com.radixdlt.transactions.RawNotarizedTransaction; +import com.radixdlt.utils.WrappedByteArray; import java.util.Collections; import java.util.List; import java.util.Objects; @@ -147,7 +148,7 @@ public StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { return this.stateComputer.commit(ledgerExtension, serializedVertexStoreState); } } diff --git a/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java b/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java index 0cda0cb637..75bed7c9ab 100644 --- a/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java +++ b/core/src/test-core/java/com/radixdlt/statecomputer/StatelessComputer.java @@ -81,6 +81,7 @@ import com.radixdlt.statecomputer.commit.CommitSummary; import com.radixdlt.transactions.RawNotarizedTransaction; import com.radixdlt.utils.UInt32; +import com.radixdlt.utils.WrappedByteArray; import java.util.ArrayList; import java.util.List; @@ -204,7 +205,7 @@ private LedgerUpdate generateLedgerUpdate(LedgerExtension ledgerExtension) { @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { var ledgerUpdate = this.generateLedgerUpdate(ledgerExtension); ledgerUpdateDispatcher.dispatch(ledgerUpdate); return ledgerUpdate.committedProof(); diff --git a/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java b/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java index dcb005bfe4..d3e01fcd51 100644 --- a/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java +++ b/core/src/test/java/com/radixdlt/consensus/epoch/EpochManagerTest.java @@ -118,6 +118,7 @@ import com.radixdlt.utils.TimeSupplier; import com.radixdlt.utils.UInt192; import com.radixdlt.utils.UInt32; +import com.radixdlt.utils.WrappedByteArray; import java.util.List; import java.util.Optional; import java.util.function.Consumer; @@ -168,7 +169,7 @@ public StateComputerPrepareResult prepare( @Override public LedgerProofBundle commit( - LedgerExtension ledgerExtension, Option serializedVertexStoreState) { + LedgerExtension ledgerExtension, Option serializedVertexStoreState) { // No-op // `closestEpochProofOnOrBefore` isn't really correct here, but that's fine return new LedgerProofBundle( diff --git a/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java b/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java index 7044c938a4..04c7955d37 100644 --- a/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java +++ b/core/src/test/java/com/radixdlt/consensus/vertexstore/VertexStoreTest.java @@ -77,6 +77,7 @@ import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.hash.HashCode; import com.radixdlt.consensus.*; import com.radixdlt.consensus.bft.*; @@ -91,6 +92,7 @@ import com.radixdlt.utils.ZeroHasher; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -210,7 +212,7 @@ public void adding_a_qc_should_update_highest_qc() { // Act QuorumCertificate qc = vertices.get(1).vertex().getQCToParent(); - vertexStoreAdapter.insertQc(qc); + vertexStoreAdapter.insertQuorumCertificate(qc); // Assert assertThat(vertexStoreAdapter.highQC().highestQC()).isEqualTo(qc); @@ -278,7 +280,7 @@ removed from the vertex store (including their children). .getQCToParent(); // Act - final var insertQcResult = vertexStoreAdapter.insertQc(qcForVertexG); + final var insertQcResult = vertexStoreAdapter.insertQuorumCertificate(qcForVertexG); assertTrue(insertQcResult instanceof VertexStore.InsertQcResult.Inserted); assertEquals(vertexStoreAdapter.getRoot().hash(), vertexD.hash()); @@ -318,7 +320,7 @@ public void adding_a_qc_with_commit_should_commit_vertices_to_ledger() { // Act QuorumCertificate qc = vertices.get(3).vertex().getQCToParent(); - final var insertQcResult = vertexStoreAdapter.insertQc(qc); + final var insertQcResult = vertexStoreAdapter.insertQuorumCertificate(qc); // Assert assertTrue(insertQcResult instanceof VertexStore.InsertQcResult.Inserted); @@ -359,7 +361,7 @@ public void adding_a_qc_which_needs_sync_should_return_a_matching_result() { // Act QuorumCertificate qc = this.nextVertex.get().vertex().getQCToParent(); - final var insertQcResult = vertexStoreAdapter.insertQc(qc); + final var insertQcResult = vertexStoreAdapter.insertQuorumCertificate(qc); // Assert assertTrue(insertQcResult instanceof VertexStore.InsertQcResult.VertexIsMissing); @@ -371,11 +373,12 @@ public void rebuilding_should_emit_updates() { final var vertices = Stream.generate(this.nextVertex).limit(4).toList(); final var qc = vertices.get(3).vertex().getQCToParent(); + final var nonRootVertices = vertices.stream().skip(1).collect(ImmutableSet.toImmutableSet()); VertexStoreState vertexStoreState = VertexStoreState.create( HighQC.from(qc, qc, vertexStoreAdapter.highQC().highestTC()), vertices.get(0), - vertices.stream().skip(1).collect(ImmutableList.toImmutableList()), + nonRootVertices, hasher); // Act @@ -386,8 +389,8 @@ public void rebuilding_should_emit_updates() { .dispatch( argThat( u -> { - List sentVertices = u.vertexStoreState().getVertices(); - return sentVertices.equals(vertices.subList(1, vertices.size())); + Set sentVertices = u.vertexStoreState().getVertices(); + return sentVertices.equals(nonRootVertices); })); }