Track repo compaction duration and log progress every once in a while

bgs/bgs.go

@@ -1211,6 +1211,10 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 	ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction")
 	defer span.End()
 
+	log.Warn("starting repo compaction")
+
+	runStart := time.Now()
+
 	repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get repos to compact: %w", err)
@@ -1227,7 +1231,7 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 	}
 
 	results := make(map[models.Uid]*carstore.CompactionStats)
-	for _, r := range repos {
+	for i, r := range repos {
 		select {
 		case <-ctx.Done():
 			return &compactionStats{
@@ -1237,14 +1241,22 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp
 		default:
 		}
 
+		repostart := time.Now()
 		st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr)
 		if err != nil {
 			log.Errorf("failed to compact shards for user %d: %s", r.Usr, err)
 			continue
 		}
+		compactionDuration.Observe(time.Since(repostart).Seconds())
 		results[r.Usr] = st
+
+		if i%100 == 0 {
+			log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart))
+		}
 	}
 
+	log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart))
+
 	return &compactionStats{
 		Targets:   repos,
 		Completed: results,

bgs/metrics.go

@@ -42,6 +42,12 @@ var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{
 	Help: "The total number of external users created",
 })
 
+var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{
+	Name:    "compaction_duration",
+	Help:    "A histogram of compaction latencies",
+	Buckets: prometheus.ExponentialBuckets(0.001, 3, 14),
+})
+
 var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{
 	Name: "bgs_new_users_discovered",
 	Help: "The total number of new users discovered directly from the firehose (not from refs)",