diff --git a/bgs/bgs.go b/bgs/bgs.go index eac62775d..d84935413 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -281,6 +281,13 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { sendHeader = false } + if strings.HasPrefix(ctx.Path(), "/admin/") { + return ctx.JSON(500, map[string]any{ + "error": err.Error(), + }) + + } + log.Warnf("HANDLER ERROR: (%s) %s", ctx.Path(), err) if sendHeader { diff --git a/carstore/bs.go b/carstore/bs.go index 288236183..027542a00 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -23,6 +23,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" + logging "github.com/ipfs/go-log" car "github.com/ipld/go-car" carutil "github.com/ipld/go-car/util" cbg "github.com/whyrusleeping/cbor-gen" @@ -31,6 +32,8 @@ import ( "gorm.io/gorm" ) +var log = logging.Logger("carstore") + const MaxSliceLength = 2 << 20 type CarStore struct { @@ -1180,6 +1183,7 @@ type CompactionStats struct { SkippedShards int `json:"skippedShards"` ShardsDeleted int `json:"shardsDeleted"` RefsDeleted int `json:"refsDeleted"` + DupeCount int `json:"dupeCount"` } func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*CompactionStats, error) { @@ -1214,13 +1218,16 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } stale := make(map[cid.Cid]bool) + var dupes []cid.Cid var hasDirtyDupes bool for _, br := range staleRefs { if stale[br.Cid.CID] { + delete(stale, br.Cid.CID) // remove dupes from stale list, see comment below hasDirtyDupes = true - break + dupes = append(dupes, br.Cid.CID) + } else { + stale[br.Cid.CID] = true } - stale[br.Cid.CID] = true } if hasDirtyDupes { @@ -1233,7 +1240,10 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co // focus on compacting everything else. it leaves *some* dirty blocks // still around but we're doing that anyways since compaction isnt a // perfect process - return nil, fmt.Errorf("WIP: not currently handling this case") + + log.Warnw("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs)) + + //return nil, fmt.Errorf("WIP: not currently handling this case") } keep := make(map[cid.Cid]bool) @@ -1243,6 +1253,10 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } } + for _, dupe := range dupes { + keep[dupe] = true + } + results := aggrRefs(brefs, shardsById, stale) var sum int for _, r := range results { @@ -1335,6 +1349,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } stats.RefsDeleted = num + stats.DupeCount = len(dupes) return stats, nil }