Merge pull request #1235 from ecordell/connjitter

Add flags for maxlifetime jitter

internal/datastore/crdb/options.go

@@ -155,6 +155,14 @@ func ReadConnMaxLifetime(lifetime time.Duration) Option {
 	return func(po *crdbOptions) { po.readPoolOpts.ConnMaxLifetime = &lifetime }
 }
 
+// ReadConnMaxLifetimeJitter is an interval to wait up to after the max lifetime
+// to close the connection.
+//
+// This value defaults to 20% of the max lifetime.
+func ReadConnMaxLifetimeJitter(jitter time.Duration) Option {
+	return func(po *crdbOptions) { po.readPoolOpts.ConnMaxLifetimeJitter = &jitter }
+}
+
 // WriteConnMaxLifetime is the duration since creation after which a write
 // connection will be automatically closed.
 //
@@ -163,6 +171,14 @@ func WriteConnMaxLifetime(lifetime time.Duration) Option {
 	return func(po *crdbOptions) { po.writePoolOpts.ConnMaxLifetime = &lifetime }
 }
 
+// WriteConnMaxLifetimeJitter is an interval to wait up to after the max lifetime
+// to close the connection.
+//
+// This value defaults to 20% of the max lifetime.
+func WriteConnMaxLifetimeJitter(jitter time.Duration) Option {
+	return func(po *crdbOptions) { po.writePoolOpts.ConnMaxLifetimeJitter = &jitter }
+}
+
 // ReadConnsMinOpen is the minimum size of the connection pool used for reads.
 //
 // The health check will increase the number of connections to this amount if

internal/datastore/postgres/common/pgx.go

@@ -121,6 +121,7 @@ type TxFactory func(context.Context) (DBReader, common.TxCleanupFunc, error)
 type PoolOptions struct {
 	ConnMaxIdleTime         *time.Duration
 	ConnMaxLifetime         *time.Duration
+	ConnMaxLifetimeJitter   *time.Duration
 	ConnHealthCheckInterval *time.Duration
 	MinOpenConns            *int
 	MaxOpenConns            *int
@@ -153,5 +154,11 @@ func (opts PoolOptions) ConfigurePgx(pgxConfig *pgxpool.Config) {
 		pgxConfig.HealthCheckPeriod = *opts.ConnHealthCheckInterval
 	}
 
+	if opts.ConnMaxLifetimeJitter != nil {
+		pgxConfig.MaxConnLifetimeJitter = *opts.ConnMaxLifetimeJitter
+	} else if opts.ConnMaxLifetime != nil {
+		pgxConfig.MaxConnLifetimeJitter = time.Duration(0.2 * float64(*opts.ConnMaxLifetime))
+	}
+
 	ConfigurePGXLogger(pgxConfig.ConnConfig)
 }

internal/datastore/postgres/options.go

@@ -181,6 +181,22 @@ func WriteConnMaxLifetime(lifetime time.Duration) Option {
 	return func(po *postgresOptions) { po.writePoolOpts.ConnMaxLifetime = &lifetime }
 }
 
+// ReadConnMaxLifetimeJitter is an interval to wait up to after the max lifetime
+// to close the connection.
+//
+// This value defaults to 20% of the max lifetime.
+func ReadConnMaxLifetimeJitter(jitter time.Duration) Option {
+	return func(po *postgresOptions) { po.readPoolOpts.ConnMaxLifetimeJitter = &jitter }
+}
+
+// WriteConnMaxLifetimeJitter is an interval to wait up to after the max lifetime
+// to close the connection.
+//
+// This value defaults to 20% of the max lifetime.
+func WriteConnMaxLifetimeJitter(jitter time.Duration) Option {
+	return func(po *postgresOptions) { po.writePoolOpts.ConnMaxLifetimeJitter = &jitter }
+}
+
 // ReadConnsMinOpen is the minimum size of the connection pool used for reads.
 //
 // The health check will increase the number of connections to this amount if

pkg/cmd/datastore/datastore.go

@@ -41,6 +41,7 @@ var BuilderForEngine = map[string]engineBuilderFunc{
 type ConnPoolConfig struct {
 	MaxIdleTime         time.Duration
 	MaxLifetime         time.Duration
+	MaxLifetimeJitter   time.Duration
 	MaxOpenConns        int
 	MinOpenConns        int
 	HealthCheckInterval time.Duration
@@ -74,6 +75,7 @@ func RegisterConnPoolFlagsWithPrefix(flagSet *pflag.FlagSet, prefix string, defa
 	flagSet.IntVar(&opts.MaxOpenConns, flagName("max-open"), defaults.MaxOpenConns, "number of concurrent connections open in a remote datastore's connection pool")
 	flagSet.IntVar(&opts.MinOpenConns, flagName("min-open"), defaults.MinOpenConns, "number of minimum concurrent connections open in a remote datastore's connection pool")
 	flagSet.DurationVar(&opts.MaxLifetime, flagName("max-lifetime"), defaults.MaxLifetime, "maximum amount of time a connection can live in a remote datastore's connection pool")
+	flagSet.DurationVar(&opts.MaxLifetimeJitter, flagName("max-lifetime-jitter"), defaults.MaxLifetimeJitter, "waits rand(0, jitter) after a connection is open for max lifetime to actually close the connection (default: 20% of max lifetime)")
 	flagSet.DurationVar(&opts.MaxIdleTime, flagName("max-idletime"), defaults.MaxIdleTime, "maximum amount of time a connection can idle in a remote datastore's connection pool")
 	flagSet.DurationVar(&opts.HealthCheckInterval, flagName("healthcheck-interval"), defaults.HealthCheckInterval, "amount of time between connection health checks in a remote datastore's connection pool")
 }
@@ -328,11 +330,13 @@ func newCRDBDatastore(opts Config) (datastore.Datastore, error) {
 		crdb.ReadConnsMinOpen(opts.ReadConnPool.MinOpenConns),
 		crdb.ReadConnMaxIdleTime(opts.ReadConnPool.MaxIdleTime),
 		crdb.ReadConnMaxLifetime(opts.ReadConnPool.MaxLifetime),
+		crdb.ReadConnMaxLifetimeJitter(opts.ReadConnPool.MaxLifetimeJitter),
 		crdb.ReadConnHealthCheckInterval(opts.ReadConnPool.HealthCheckInterval),
 		crdb.WriteConnsMaxOpen(opts.WriteConnPool.MaxOpenConns),
 		crdb.WriteConnsMinOpen(opts.WriteConnPool.MinOpenConns),
 		crdb.WriteConnMaxIdleTime(opts.WriteConnPool.MaxIdleTime),
 		crdb.WriteConnMaxLifetime(opts.WriteConnPool.MaxLifetime),
+		crdb.WriteConnMaxLifetimeJitter(opts.WriteConnPool.MaxLifetimeJitter),
 		crdb.WriteConnHealthCheckInterval(opts.WriteConnPool.HealthCheckInterval),
 		crdb.SplitAtUsersetCount(opts.SplitQueryCount),
 		crdb.FollowerReadDelay(opts.FollowerReadDelay),
@@ -354,11 +358,13 @@ func newPostgresDatastore(opts Config) (datastore.Datastore, error) {
 		postgres.ReadConnsMinOpen(opts.ReadConnPool.MinOpenConns),
 		postgres.ReadConnMaxIdleTime(opts.ReadConnPool.MaxIdleTime),
 		postgres.ReadConnMaxLifetime(opts.ReadConnPool.MaxLifetime),
+		postgres.ReadConnMaxLifetimeJitter(opts.ReadConnPool.MaxLifetimeJitter),
 		postgres.ReadConnHealthCheckInterval(opts.ReadConnPool.HealthCheckInterval),
 		postgres.WriteConnsMaxOpen(opts.WriteConnPool.MaxOpenConns),
 		postgres.WriteConnsMinOpen(opts.WriteConnPool.MinOpenConns),
 		postgres.WriteConnMaxIdleTime(opts.WriteConnPool.MaxIdleTime),
 		postgres.WriteConnMaxLifetime(opts.WriteConnPool.MaxLifetime),
+		postgres.WriteConnMaxLifetimeJitter(opts.ReadConnPool.MaxLifetimeJitter),
 		postgres.WriteConnHealthCheckInterval(opts.WriteConnPool.HealthCheckInterval),
 		postgres.SplitAtUsersetCount(opts.SplitQueryCount),
 		postgres.GCInterval(opts.GCInterval),