diff --git a/persist/sqlite/blocks.go b/persist/sqlite/blocks.go index b9945d1..1bc5514 100644 --- a/persist/sqlite/blocks.go +++ b/persist/sqlite/blocks.go @@ -153,3 +153,28 @@ INNER JOIN blocks AS b ON (b.id = fs.child_id)` }) return } + +// Pinned returns the next n pinned CIDs. If there are no remaining CIDs, the +// returned value should be (nil, nil) +func (s *Store) Pinned(offset, limit int) (roots []cid.Cid, err error) { + err = s.transaction(func(tx *txn) error { + rows, err := tx.Query(`SELECT b.cid FROM pinned_blocks pb +INNER JOIN blocks b ON (b.id=pb.block_id) +ORDER BY b.id ASC +LIMIT $1 OFFSET $2`, limit, offset) + if err != nil { + return fmt.Errorf("failed to query root cids: %w", err) + } + defer rows.Close() + + for rows.Next() { + var root cid.Cid + if err := rows.Scan(dbDecode(&root)); err != nil { + return fmt.Errorf("failed to scan root cid: %w", err) + } + roots = append(roots, root) + } + return rows.Err() + }) + return +} diff --git a/renterd/persist.go b/renterd/persist.go index 5cd4a3c..7e8674c 100644 --- a/renterd/persist.go +++ b/renterd/persist.go @@ -14,6 +14,7 @@ type ( Pin(PinnedBlock) error Unpin(c cid.Cid) error + Pinned(offset, limit int) (roots []cid.Cid, err error) BlockLocation(c cid.Cid) (bucket, key string, err error) } ) diff --git a/renterd/store.go b/renterd/store.go index adb8b54..f1d26c3 100644 --- a/renterd/store.go +++ b/renterd/store.go @@ -173,38 +173,26 @@ func (bs *BlockStore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { log := bs.log.Named("AllKeysChan") ch := make(chan cid.Cid) go func() { - var marker string - for { - resp, err := bs.busClient.ListObjects(ctx, bs.bucket, api.ListObjectOptions{ - Marker: marker, - Limit: 100, - }) + + for i := 0; ; i += 1000 { + cids, err := bs.metadata.Pinned(i, 1000) if err != nil { + bs.log.Error("failed to get root CIDs", zap.Error(err)) close(ch) return - } else if len(resp.Objects) == 0 { + } else if len(cids) == 0 { close(ch) return } - for _, obj := range resp.Objects { - name := obj.Name - if p := strings.Split(obj.Name, "/"); len(p) > 1 { - name = p[len(p)-1] - } - c, err := cid.Parse(name) - if err != nil { - log.Debug("skipping invalid key", zap.String("key", obj.Name), zap.String("name", name)) - continue - } - log.Debug("found key", zap.Stringer("cid", c)) + log.Debug("got root CIDs", zap.Int("count", len(cids))) + for _, c := range cids { select { case ch <- c: case <-ctx.Done(): close(ch) return } - marker = obj.Name } } }()