From b97d9e6d07f6914baee2f5c16e7677704fa5625a Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 19 Sep 2023 14:58:19 -0700 Subject: [PATCH 1/4] split up batches of deletes during compaction --- carstore/bs.go | 70 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index cc903ed5a..f6c2fdcfe 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -965,36 +965,52 @@ func (cs *CarStore) deleteShards(ctx context.Context, shs []*CarShard) error { ctx, span := otel.Tracer("carstore").Start(ctx, "deleteShard") defer span.End() - var ids []uint - for _, sh := range shs { - ids = append(ids, sh.ID) - } + deleteSlice := func(ctx context.Context, subs []*CarShard) error { + var ids []uint + for _, sh := range subs { + ids = append(ids, sh.ID) + } - txn := cs.meta.Begin() + txn := cs.meta.Begin() - if err := txn.Delete(&CarShard{}, "id in (?)", ids).Error; err != nil { - return err - } + if err := txn.Delete(&CarShard{}, "id in (?)", ids).Error; err != nil { + return err + } - if err := txn.Delete(&blockRef{}, "shard in (?)", ids).Error; err != nil { - return err - } + if err := txn.Delete(&blockRef{}, "shard in (?)", ids).Error; err != nil { + return err + } - if err := txn.Commit().Error; err != nil { - return err - } + if err := txn.Commit().Error; err != nil { + return err + } - var outErr error - for _, sh := range shs { - if err := cs.deleteShardFile(ctx, sh); err != nil { - if !os.IsNotExist(err) { - return err + var outErr error + for _, sh := range subs { + if err := cs.deleteShardFile(ctx, sh); err != nil { + if !os.IsNotExist(err) { + return err + } + outErr = err } - outErr = err + } + + return outErr + } + + chunkSize := 100 + for i := 0; i < len(shs); i += chunkSize { + sl := shs[i:] + if len(sl) > chunkSize { + sl = sl[:chunkSize] + } + + if err := deleteSlice(ctx, sl); err != nil { + return err } } - return outErr + return nil } type shardStat struct { @@ -1265,8 +1281,16 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) erro } } - if err := cs.meta.Delete(&staleRef{}, "id in (?)", staleToDelete).Error; err != nil { - return err + chunkSize := 100 + for i := 0; i < len(staleToDelete); i += chunkSize { + sl := staleToDelete[i:] + if len(sl) > chunkSize { + sl = sl[:chunkSize] + } + + if err := cs.meta.Delete(&staleRef{}, "id in (?)", sl).Error; err != nil { + return err + } } return nil From 9732b5d39abee629e1ed29d1c8988179c4b6e240 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 19 Sep 2023 15:20:36 -0700 Subject: [PATCH 2/4] add some more tracing --- carstore/bs.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/carstore/bs.go b/carstore/bs.go index f6c2fdcfe..c459aedaa 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1260,6 +1260,13 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) erro // now we need to delete the staleRefs we successfully cleaned up // we can delete a staleRef if all the shards that have blockRefs with matching stale refs were processed + return cs.deleteStaleRefs(ctx, brefs, staleRefs, removedShards) +} + +func (cs *CarStore) deleteStaleRefs(ctx context.Context, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) error { + ctx, span := otel.Tracer("carstore").Start(ctx, "deleteStaleRefs") + defer span.End() + brByCid := make(map[cid.Cid][]blockRef) for _, br := range brefs { brByCid[br.Cid.CID] = append(brByCid[br.Cid.CID], br) From 78495fa02d6562d6575419a7a748c0f664e2c5f7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 19 Sep 2023 16:59:33 -0700 Subject: [PATCH 3/4] fixup indexing on stale refs --- carstore/bs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index c459aedaa..a73c60818 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -91,9 +91,9 @@ type blockRef struct { } type staleRef struct { - ID uint `gorm:"primarykey"` - Cid models.DbCID `gorm:"index"` - Usr models.Uid + ID uint `gorm:"primarykey"` + Cid models.DbCID + Usr models.Uid `gorm:"index"` } type userView struct { From b05881606e1bea2c8f6719091553fd3fdfdfabda Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 19 Sep 2023 17:19:30 -0700 Subject: [PATCH 4/4] compaction database tracing --- carstore/bs.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index a73c60818..86d3cfaaa 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1151,7 +1151,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) erro span.SetAttributes(attribute.Int64("user", int64(user))) var shards []CarShard - if err := cs.meta.Find(&shards, "usr = ?", user).Error; err != nil { + if err := cs.meta.WithContext(ctx).Find(&shards, "usr = ?", user).Error; err != nil { return err } @@ -1166,12 +1166,12 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) erro } var brefs []blockRef - if err := cs.meta.Debug().Raw(`select * from block_refs where shard in (?)`, shardIds).Scan(&brefs).Error; err != nil { + if err := cs.meta.WithContext(ctx).Raw(`select * from block_refs where shard in (?)`, shardIds).Scan(&brefs).Error; err != nil { return err } var staleRefs []staleRef - if err := cs.meta.Debug().Find(&staleRefs, "usr = ?", user).Error; err != nil { + if err := cs.meta.WithContext(ctx).Find(&staleRefs, "usr = ?", user).Error; err != nil { return err }