Randomize rebroadcast backoff

gpbft/options.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	"math/rand"
 	"time"
 )
 
@@ -117,33 +118,35 @@ func WithMaxCachedMessagesPerInstance(v int) Option {
 	}
 }
 
-var defaultRebroadcastAfter = exponentialBackoffer(1.3, 3*time.Second, 30*time.Second)
+var defaultRebroadcastAfter = exponentialBackoffer(1.3, 0.1, 3*time.Second, 30*time.Second)
 
 // WithRebroadcastBackoff sets the duration after the gPBFT timeout has elapsed, at
 // which all messages in the current round are rebroadcast if the round cannot be
 // terminated, i.e. a strong quorum of senders has not yet been achieved.
 //
-// The backoff duration grows exponentially up to the configured max. Defaults to
-// exponent of 1.3 with 3s base backoff growing to a maximum of 30s.
-func WithRebroadcastBackoff(exponent float64, base, max time.Duration) Option {
+// The backoff duration grows exponentially up to the configured max and is randomized by "jitter"
+// (a number between 0 and 1). Defaults to exponent of 1.3 with 3s base backoff growing to a maximum
+// of 30s, randomized by 10% (5% in either direction).
+func WithRebroadcastBackoff(exponent, jitter float64, base, max time.Duration) Option {
 	return func(o *options) error {
 		if base <= 0 {
 			return fmt.Errorf("rebroadcast backoff duration must be greater than zero; got: %s", base)
 		}
 		if max < base {
 			return fmt.Errorf("rebroadcast backoff max duration must be greater than base; got: %s", max)
 		}
-		o.rebroadcastAfter = exponentialBackoffer(exponent, base, max)
+		o.rebroadcastAfter = exponentialBackoffer(exponent, jitter, base, max)
 		return nil
 	}
 }
 
-func exponentialBackoffer(exponent float64, base, maxBackoff time.Duration) func(attempt int) time.Duration {
+func exponentialBackoffer(exponent, jitter float64, base, maxBackoff time.Duration) func(attempt int) time.Duration {
 	return func(attempt int) time.Duration {
-		nextBackoff := float64(base) * math.Pow(exponent, float64(attempt))
+		shift := 1.0 + (rand.Float64()-0.5)*jitter
+		nextBackoff := float64(base) * math.Pow(exponent, float64(attempt)) * shift
 		if nextBackoff > float64(maxBackoff) {
 			return maxBackoff
 		}
-		return maxBackoff
+		return time.Duration(nextBackoff)
 	}
 }

gpbft/options_test.go

@@ -7,12 +7,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func Test_exponentialBackoffer(t *testing.T) {
+func Test_exponentialBackofferMax(t *testing.T) {
 	maxBackoff := 30 * time.Second
-	backoffer := exponentialBackoffer(1.3, 3*time.Second, maxBackoff)
+	backoffer := exponentialBackoffer(1.3, 0, 3*time.Second, maxBackoff)
+	var lastBackoff time.Duration
 	for i := 0; i < 10_000; i++ {
 		backoff := backoffer(i)
 		require.Positivef(t, backoff, "at %d", i)
-		require.LessOrEqualf(t, backoff, maxBackoff, "at %d", i)
+		if backoff != maxBackoff {
+			require.Less(t, backoff, maxBackoff, "at %d", i)
+			require.Greater(t, backoff, lastBackoff, "at %d", i)
+		}
+	}
+}
+
+func Test_exponentialBackofferSpread(t *testing.T) {
+	maxBackoff := 30 * time.Second
+	backoffer1 := exponentialBackoffer(1.3, 0.1, 3*time.Second, maxBackoff)
+	backoffer2 := exponentialBackoffer(1.3, 0.1, 3*time.Second, maxBackoff)
+
+	for i := 0; i < 8; i++ {
+		backoff1 := backoffer1(i)
+		backoff2 := backoffer2(i)
+		require.NotEqual(t, backoff1, backoff2, "backoffs were not randomized")
 	}
 }

manifest/manifest.go

@@ -34,6 +34,7 @@ var (
 		DeltaBackOffExponent:       2.0,
 		MaxLookaheadRounds:         5,
 		RebroadcastBackoffBase:     3 * time.Second,
+		RebroadcastBackoffSpread:   0.1,
 		RebroadcastBackoffExponent: 1.3,
 		RebroadcastBackoffMax:      30 * time.Second,
 	}
@@ -98,6 +99,7 @@ type GpbftConfig struct {
 
 	RebroadcastBackoffBase     time.Duration
 	RebroadcastBackoffExponent float64
+	RebroadcastBackoffSpread   float64
 	RebroadcastBackoffMax      time.Duration
 }
 
@@ -131,6 +133,7 @@ func (g *GpbftConfig) ToOptions() []gpbft.Option {
 		gpbft.WithMaxLookaheadRounds(g.MaxLookaheadRounds),
 		gpbft.WithRebroadcastBackoff(
 			DefaultGpbftConfig.RebroadcastBackoffExponent,
+			DefaultGpbftConfig.RebroadcastBackoffSpread,
 			DefaultGpbftConfig.RebroadcastBackoffBase,
 			DefaultGpbftConfig.RebroadcastBackoffMax,
 		),

test/constants_test.go

@@ -29,7 +29,7 @@ var (
 	testGpbftOptions = []gpbft.Option{
 		gpbft.WithDelta(200 * time.Millisecond),
 		gpbft.WithDeltaBackOffExponent(1.300),
-		gpbft.WithRebroadcastBackoff(1.3, time.Second, 5*time.Second),
+		gpbft.WithRebroadcastBackoff(1.3, 0, time.Second, 5*time.Second),
 	}
 )