From 254305ac5edc5c83ce9ab2aa9c376eac3452603c Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 22 Sep 2023 19:09:55 +0000 Subject: [PATCH 1/2] Set a job as complete after flushing the buffer --- backfill/backfill.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backfill/backfill.go b/backfill/backfill.go index 24404a6d6..2199147cb 100644 --- a/backfill/backfill.go +++ b/backfill/backfill.go @@ -225,6 +225,12 @@ func (b *Backfiller) FlushBuffer(ctx context.Context, job Job) int { log.Error("failed to flush buffered ops", "error", err) } + // Mark the job as "complete" + err = job.SetState(ctx, StateComplete) + if err != nil { + log.Error("failed to set job state", "error", err) + } + return processed } From 2bee518637eea3db2ca1e15d81924753d1d3c1ca Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 22 Sep 2023 19:29:25 +0000 Subject: [PATCH 2/2] Update parallel backfills limit --- search/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/search/server.go b/search/server.go index f1e456783..a69e6f4b3 100644 --- a/search/server.go +++ b/search/server.go @@ -89,6 +89,7 @@ func NewServer(db *gorm.DB, escli *es.Client, dir identity.Directory, config Con opts := backfill.DefaultBackfillOptions() if config.BGSSyncRateLimit > 0 { opts.SyncRequestsPerSecond = config.BGSSyncRateLimit + opts.ParallelBackfills = 2 * config.BGSSyncRateLimit } else { opts.SyncRequestsPerSecond = 8 }