From 59cb6d8004b31ccf01debdf26149700bec80e5cd Mon Sep 17 00:00:00 2001
From: Jaz Volpert <ericvolp12@gmail.com>
Date: Thu, 5 Oct 2023 19:49:49 +0000
Subject: [PATCH] Track repo compaction duration and log progress every once in
 a while

---
 bgs/bgs.go     | 14 +++++++++++++-
 bgs/metrics.go |  6 ++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/bgs/bgs.go b/bgs/bgs.go
index a5e04d539..a28e67363 100644
--- a/bgs/bgs.go
+++ b/bgs/bgs.go
@@ -1211,6 +1211,10 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 	ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction")
 	defer span.End()
 
+	log.Warn("starting repo compaction")
+
+	runStart := time.Now()
+
 	repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get repos to compact: %w", err)
@@ -1227,7 +1231,7 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 	}
 
 	results := make(map[models.Uid]*carstore.CompactionStats)
-	for _, r := range repos {
+	for i, r := range repos {
 		select {
 		case <-ctx.Done():
 			return &compactionStats{
@@ -1237,14 +1241,22 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 		default:
 		}
 
+		repostart := time.Now()
 		st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr)
 		if err != nil {
 			log.Errorf("failed to compact shards for user %d: %s", r.Usr, err)
 			continue
 		}
+		compactionDuration.Observe(time.Since(repostart).Seconds())
 		results[r.Usr] = st
+
+		if i%100 == 0 {
+			log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart))
+		}
 	}
 
+	log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart))
+
 	return &compactionStats{
 		Targets:   repos,
 		Completed: results,
diff --git a/bgs/metrics.go b/bgs/metrics.go
index 97f444c48..12b401e04 100644
--- a/bgs/metrics.go
+++ b/bgs/metrics.go
@@ -42,6 +42,12 @@ var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{
 	Help: "The total number of external users created",
 })
 
+var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "compaction_duration",
+	Help:    "A histogram of compaction latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 3, 14),
+})
+
 var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "bgs_new_users_discovered",
 	Help: "The total number of new users discovered directly from the firehose (not from refs)",