Mark finished backfills as complete in the backfill jobstore (#331)

bluesky-social · Sep 22, 2023 · b6a6ed5 · b6a6ed5
2 parents 3eba817 + 2bee518
commit b6a6ed5
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 0 deletions.
diff --git a/backfill/backfill.go b/backfill/backfill.go
@@ -225,6 +225,12 @@ func (b *Backfiller) FlushBuffer(ctx context.Context, job Job) int {
 		log.Error("failed to flush buffered ops", "error", err)
 	}
 
+	// Mark the job as "complete"
+	err = job.SetState(ctx, StateComplete)
+	if err != nil {
+		log.Error("failed to set job state", "error", err)
+	}
+
 	return processed
 }
 

diff --git a/search/server.go b/search/server.go
@@ -89,6 +89,7 @@ func NewServer(db *gorm.DB, escli *es.Client, dir identity.Directory, config Con
 	opts := backfill.DefaultBackfillOptions()
 	if config.BGSSyncRateLimit > 0 {
 		opts.SyncRequestsPerSecond = config.BGSSyncRateLimit
+		opts.ParallelBackfills = 2 * config.BGSSyncRateLimit
 	} else {
 		opts.SyncRequestsPerSecond = 8
 	}