be less lazy, skip specific cids

bluesky-social · Nov 21, 2023 · 0c23908 · 0c23908
1 parent 2e4caf4
commit 0c23908
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 14 deletions.
diff --git a/carstore/bs.go b/carstore/bs.go
@@ -861,7 +861,7 @@ func setToSlice(s map[cid.Cid]bool) []cid.Cid {
 	return out
 }
 
-func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block, skipmissing bool) (map[cid.Cid]bool, error) {
+func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, newcids map[cid.Cid]blockformat.Block, skipcids map[cid.Cid]bool) (map[cid.Cid]bool, error) {
 	ctx, span := otel.Tracer("repo").Start(ctx, "BlockDiff")
 	defer span.End()
 
@@ -899,13 +899,13 @@ func BlockDiff(ctx context.Context, bs blockstore.Blockstore, oldroot cid.Cid, n
 		c := queue[0]
 		queue = queue[1:]
 
+		if skipcids != nil && skipcids[c] {
+			continue
+		}
+
 		oblk, err := bs.Get(ctx, c)
 		if err != nil {
-			if skipmissing && ipld.IsNotFound(err) {
-				log.Warnw("missing block in old tree", "root", oldroot, "missing", c)
-			} else {
-				return nil, fmt.Errorf("get failed in old tree: %w", err)
-			}
+			return nil, fmt.Errorf("get failed in old tree: %w", err)
 		}
 
 		if err := cbg.ScanForLinks(bytes.NewReader(oblk.RawData()), func(lnk cid.Cid) {
@@ -963,8 +963,8 @@ func (cs *CarStore) ImportSlice(ctx context.Context, uid models.Uid, since *stri
 	return carr.Header.Roots[0], ds, nil
 }
 
-func (ds *DeltaSession) CalcDiff(ctx context.Context, skipmissing bool) error {
-	rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks, skipmissing)
+func (ds *DeltaSession) CalcDiff(ctx context.Context, skipcids map[cid.Cid]bool) error {
+	rmcids, err := BlockDiff(ctx, ds, ds.baseCid, ds.blks, skipcids)
 	if err != nil {
 		return fmt.Errorf("block diff failed (base=%s,rev=%s): %w", ds.baseCid, ds.lastRev, err)
 	}

diff --git a/carstore/repo_test.go b/carstore/repo_test.go
@@ -125,7 +125,7 @@ func TestBasicOperation(t *testing.T) {
 
 		rev = nrev
 
-		if err := ds.CalcDiff(ctx, false); err != nil {
+		if err := ds.CalcDiff(ctx, nil); err != nil {
 			t.Fatal(err)
 		}
 
@@ -217,7 +217,7 @@ func TestRepeatedCompactions(t *testing.T) {
 
 			rev = nrev
 
-			if err := ds.CalcDiff(ctx, false); err != nil {
+			if err := ds.CalcDiff(ctx, nil); err != nil {
 				t.Fatal(err)
 			}
 
@@ -356,7 +356,7 @@ func BenchmarkRepoWritesCarstore(b *testing.B) {
 		}
 
 		rev = nrev
-		if err := ds.CalcDiff(ctx, false); err != nil {
+		if err := ds.CalcDiff(ctx, nil); err != nil {
 			b.Fatal(err)
 		}
 

diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go
@@ -552,7 +552,7 @@ func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint,
 		return err
 	}
 
-	var badPrev bool
+	var skipcids map[cid.Cid]bool
 	if ds.BaseCid().Defined() {
 		oldrepo, err := repo.OpenRepo(ctx, ds, ds.BaseCid())
 		if err != nil {
@@ -565,11 +565,13 @@ func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint,
 		// and this becomes no longer an issue
 		prev, _ := oldrepo.PrevCommit(ctx)
 		if prev != nil {
-			badPrev = true
+			skipcids = map[cid.Cid]bool{
+				*prev: true,
+			}
 		}
 	}
 
-	if err := ds.CalcDiff(ctx, badPrev); err != nil {
+	if err := ds.CalcDiff(ctx, skipcids); err != nil {
 		return fmt.Errorf("failed while calculating mst diff (since=%v): %w", since, err)
 
 	}