Skip to content

Commit

Permalink
Allow setting replica snapshot interval offset
Browse files Browse the repository at this point in the history
If using S3 replicas the extra LIST call on startup may be
expensive in scale if a lot of Litestreams are started in quick
succession.

Allowing configuring the snapshot offset externally allows the
caller to spread around the snapshots without relying on access to
the remote replica making restarts no-op.
  • Loading branch information
hifi committed Nov 15, 2023
1 parent 85ddf32 commit d6a5f8f
Showing 1 changed file with 25 additions and 17 deletions.
42 changes: 25 additions & 17 deletions replica.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ type Replica struct {
// Frequency to create new snapshots.
SnapshotInterval time.Duration

// Duration added to snapshot interval before the first one. Skips checking replica.
SnapshotOffset time.Duration

// Time to keep snapshots and related WAL files.
// Database is snapshotted after interval, if needed, and older WAL files are discarded.
Retention time.Duration
Expand Down Expand Up @@ -738,26 +741,31 @@ func (r *Replica) snapshotter(ctx context.Context) {
}

logger := r.Logger()
if pos, err := r.db.Pos(); err != nil {
logger.Error("snapshotter cannot determine generation", "error", err)
} else if !pos.IsZero() {
if snapshot, err := r.maxSnapshot(ctx, pos.Generation); err != nil {
logger.Error("snapshotter cannot determine latest snapshot", "error", err)
} else if snapshot != nil {
nextSnapshot := r.SnapshotInterval - time.Since(snapshot.CreatedAt)
if nextSnapshot < 0 {
nextSnapshot = 0
if r.SnapshotOffset == 0 {
if pos, err := r.db.Pos(); err != nil {
logger.Error("snapshotter cannot determine generation", "error", err)
} else if !pos.IsZero() {
if snapshot, err := r.maxSnapshot(ctx, pos.Generation); err != nil {
logger.Error("snapshotter cannot determine latest snapshot", "error", err)
} else if snapshot != nil {
r.SnapshotOffset = r.SnapshotInterval - time.Since(snapshot.CreatedAt)
// ensure we will snapshot immediately if zero than less
if r.SnapshotOffset < 1 {
r.SnapshotOffset = 1
}
}
}
}

logger.Info("snapshot interval adjusted", "previous", snapshot.CreatedAt.Format(time.RFC3339), "next", nextSnapshot.String())
if r.SnapshotOffset > 0 {
logger.Info("snapshot interval adjusted", "next", time.Now().Add(r.SnapshotOffset).Format(time.RFC3339))

select {
case <-ctx.Done():
return
case <-time.After(nextSnapshot):
if _, err := r.Snapshot(ctx); err != nil && err != ErrNoGeneration {
logger.Error("snapshotter error", "error", err)
}
select {
case <-ctx.Done():
return
case <-time.After(r.SnapshotOffset):
if _, err := r.Snapshot(ctx); err != nil && err != ErrNoGeneration {
logger.Error("snapshotter error", "error", err)
}
}
}
Expand Down

0 comments on commit d6a5f8f

Please sign in to comment.