From 1ca9d91466cc71ce266fc23d3eab4110702ce2ea Mon Sep 17 00:00:00 2001 From: Changxin Miao Date: Tue, 30 Jul 2024 22:24:47 +0800 Subject: [PATCH 1/2] Reduce scan interval in cleanupDeletedFiles Signed-off-by: Changxin Miao --- pkg/meta/base.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 8538cca7a15f..cb429191a32f 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -604,8 +604,8 @@ func (m *baseMeta) Init(format *Format, force bool) error { func (m *baseMeta) cleanupDeletedFiles() { for { - utils.SleepWithJitter(time.Minute) - if ok, err := m.en.setIfSmall("lastCleanupFiles", time.Now().Unix(), int64(time.Minute.Seconds())*9/10); err != nil { + utils.SleepWithJitter(time.Hour) + if ok, err := m.en.setIfSmall("lastCleanupFiles", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil { logger.Warnf("checking counter lastCleanupFiles: %s", err) } else if ok { files, err := m.en.doFindDeletedFiles(time.Now().Add(-time.Hour).Unix(), 10000) From 7ae688754fb493c088ccd00edd2217175eda98ac Mon Sep 17 00:00:00 2001 From: Changxin Miao Date: Fri, 6 Dec 2024 14:24:58 +0800 Subject: [PATCH 2/2] Larger scan batch Signed-off-by: Changxin Miao --- pkg/meta/base.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index cb429191a32f..e4cdcaa37177 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -608,14 +608,18 @@ func (m *baseMeta) cleanupDeletedFiles() { if ok, err := m.en.setIfSmall("lastCleanupFiles", time.Now().Unix(), int64(time.Hour.Seconds())*9/10); err != nil { logger.Warnf("checking counter lastCleanupFiles: %s", err) } else if ok { - files, err := m.en.doFindDeletedFiles(time.Now().Add(-time.Hour).Unix(), 10000) + files, err := m.en.doFindDeletedFiles(time.Now().Add(-time.Hour).Unix(), 6e5) if err != nil { logger.Warnf("scan deleted files: %s", err) continue } + start := time.Now() for inode, length := range files { logger.Debugf("cleanup chunks of inode %d with %d bytes", inode, length) m.en.doDeleteFileData(inode, length) + if time.Since(start) > 50*time.Minute { // Yield my time slice to avoid conflicts with other clients + break + } } } }