Merge pull request #4975 from onflow/jordan/vn-test-block-rate-fix

[Flaky Test] Modifies block rate in VN test to address sealing lagging finalization

engine/access/rpc/backend/backend.go

@@ -34,7 +34,7 @@ const DefaultMaxHeightRange = 250
 
 // DefaultSnapshotHistoryLimit the amount of blocks to look back in state
 // when recursively searching for a valid snapshot
-const DefaultSnapshotHistoryLimit = 50
+const DefaultSnapshotHistoryLimit = 500
 
 // DefaultLoggedScriptsCacheSize is the default size of the lookup cache used to dedupe logs of scripts sent to ENs
 // limiting cache size to 16MB and does not affect script execution, only for keeping logs tidy

integration/tests/epochs/cohort2/epoch_join_and_leave_vn_test.go

@@ -2,6 +2,7 @@ package cohort2
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/suite"
 
@@ -20,6 +21,8 @@ type EpochJoinAndLeaveVNSuite struct {
 func (s *EpochJoinAndLeaveVNSuite) SetupTest() {
 	// require approvals for seals to verify that the joining VN is producing valid seals in the second epoch
 	s.RequiredSealApprovals = 1
+	// slow down consensus, as sealing tends to lag behind
+	s.ConsensusProposalDuration = time.Millisecond * 250
 	// increase epoch length to account for greater sealing lag due to above
 	// NOTE: this value is set fairly aggressively to ensure shorter test times.
 	// If flakiness due to failure to complete staking operations in time is observed,
@@ -28,7 +31,7 @@ func (s *EpochJoinAndLeaveVNSuite) SetupTest() {
 	s.DKGPhaseLen = 100
 	s.EpochLen = 450
 	s.EpochCommitSafetyThreshold = 20
-	s.DynamicEpochTransitionSuite.SetupTest()
+	s.DynamicEpochTransitionSuite.Suite.SetupTest()
 }
 
 // TestEpochJoinAndLeaveVN should update verification nodes and assert healthy network conditions

integration/tests/epochs/suite.go

@@ -64,10 +64,16 @@ type Suite struct {
 	// Whether approvals are required for sealing (we only enable for VN tests because
 	// requiring approvals requires a longer DKG period to avoid flakiness)
 	RequiredSealApprovals uint // defaults to 0 (no approvals required)
+	// Consensus Node proposal duration
+	ConsensusProposalDuration time.Duration
 }
 
 // SetupTest is run automatically by the testing framework before each test case.
 func (s *Suite) SetupTest() {
+	// If unset, use default value 100ms
+	if s.ConsensusProposalDuration == 0 {
+		s.ConsensusProposalDuration = time.Millisecond * 100
+	}
 
 	minEpochLength := s.StakingAuctionLen + s.DKGPhaseLen*3 + 20
 	// ensure epoch lengths are set correctly
@@ -85,7 +91,7 @@ func (s *Suite) SetupTest() {
 		testnet.WithLogLevel(zerolog.WarnLevel)}
 
 	consensusConfigs := []func(config *testnet.NodeConfig){
-		testnet.WithAdditionalFlag("--cruise-ctl-fallback-proposal-duration=100ms"),
+		testnet.WithAdditionalFlag(fmt.Sprintf("--cruise-ctl-fallback-proposal-duration=%s", s.ConsensusProposalDuration)),
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-verification-seal-approvals=%d", s.RequiredSealApprovals)),
 		testnet.WithAdditionalFlag(fmt.Sprintf("--required-construction-seal-approvals=%d", s.RequiredSealApprovals)),
 		testnet.WithLogLevel(zerolog.WarnLevel)}
@@ -451,18 +457,20 @@ func (s *Suite) getContainerToReplace(role flow.Role) *testnet.Container {
 
 // AwaitEpochPhase waits for the given phase, in the given epoch.
 func (s *Suite) AwaitEpochPhase(ctx context.Context, expectedEpoch uint64, expectedPhase flow.EpochPhase, waitFor, tick time.Duration) {
+	var actualEpoch uint64
+	var actualPhase flow.EpochPhase
 	condition := func() bool {
 		snapshot, err := s.Client.GetLatestProtocolSnapshot(ctx)
 		require.NoError(s.T(), err)
 
-		actualEpoch, err := snapshot.Epochs().Current().Counter()
+		actualEpoch, err = snapshot.Epochs().Current().Counter()
 		require.NoError(s.T(), err)
-		actualPhase, err := snapshot.Phase()
+		actualPhase, err = snapshot.Phase()
 		require.NoError(s.T(), err)
 
 		return actualEpoch == expectedEpoch && actualPhase == expectedPhase
 	}
-	require.Eventuallyf(s.T(), condition, waitFor, tick, "did not reach expectedEpoch %d phase %s within %s", expectedEpoch, expectedPhase, waitFor)
+	require.Eventuallyf(s.T(), condition, waitFor, tick, "did not reach expectedEpoch %d phase %s within %s. Last saw epoch=%d and phase=%s", expectedEpoch, expectedPhase, waitFor, actualEpoch, actualPhase)
 }
 
 // AssertInEpochPhase checks if we are in the phase of the given epoch.
@@ -640,7 +648,7 @@ func (s *Suite) RunTestEpochJoinAndLeave(role flow.Role, checkNetworkHealth node
 
 	// wait for epoch setup phase before we start our container and pause the old container
 	s.TimedLogf("waiting for EpochSetup phase of first epoch to begin")
-	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, 3*time.Minute, 500*time.Millisecond)
+	s.AwaitEpochPhase(s.Ctx, 0, flow.EpochPhaseSetup, time.Minute, 500*time.Millisecond)
 	s.TimedLogf("successfully reached EpochSetup phase of first epoch")
 
 	// get the latest snapshot and start new container with it

integration/tests/lib/util.go

@@ -220,28 +220,28 @@ func WithChainID(chainID flow.ChainID) func(tx *sdk.Transaction) {
 
 // LogStatus logs current information about the test network state.
 func LogStatus(t *testing.T, ctx context.Context, log zerolog.Logger, client *testnet.Client) {
+	// retrieves latest FINALIZED snapshot
 	snapshot, err := client.GetLatestProtocolSnapshot(ctx)
 	if err != nil {
-		log.Err(err).Msg("failed to get sealed snapshot")
-		return
-	}
-	finalized, err := client.GetLatestFinalizedBlockHeader(ctx)
-	if err != nil {
-		log.Err(err).Msg("failed to get finalized header")
+		log.Err(err).Msg("failed to get finalized snapshot")
 		return
 	}
 
-	sealed, err := snapshot.Head()
+	sealingSegment, err := snapshot.SealingSegment()
 	require.NoError(t, err)
+	sealed := sealingSegment.Sealed()
+	finalized := sealingSegment.Finalized()
+
 	phase, err := snapshot.Phase()
 	require.NoError(t, err)
 	epoch := snapshot.Epochs().Current()
 	counter, err := epoch.Counter()
 	require.NoError(t, err)
 
-	log.Info().Uint64("final_height", finalized.Height).
-		Uint64("sealed_height", sealed.Height).
-		Uint64("sealed_view", sealed.View).
+	log.Info().Uint64("final_height", finalized.Header.Height).
+		Uint64("final_view", finalized.Header.View).
+		Uint64("sealed_height", sealed.Header.Height).
+		Uint64("sealed_view", sealed.Header.View).
 		Str("cur_epoch_phase", phase.String()).
 		Uint64("cur_epoch_counter", counter).
 		Msg("test run status")