From 943807f6568841d3b2d19ed1d9a8fb2ebef85f12 Mon Sep 17 00:00:00 2001 From: wayblink Date: Fri, 25 Oct 2024 15:41:32 +0800 Subject: [PATCH] optimize retry in backup prepare Signed-off-by: wayblink --- core/backup_impl_create_backup.go | 23 +++++++++++++++++++---- core/storage/local_chunk_manager.go | 3 +-- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index 6a5d854d..c26f74f7 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -698,10 +698,25 @@ func (b *BackupContext) executeCreateBackup(ctx context.Context, request *backup for _, collection := range toBackupCollections { collectionClone := collection job := func(ctx context.Context) error { - err := retry.Do(ctx, func() error { - return b.backupCollectionPrepare(ctx, backupInfo, collectionClone, request.GetForce()) - }, retry.Sleep(120*time.Second), retry.Attempts(128)) - return err + retryForSpecificError := func(retries int, delay time.Duration) error { + for i := 0; i < retries; i++ { + err := b.backupCollectionPrepare(ctx, backupInfo, collectionClone, request.GetForce()) + // If no error, return successfully + if err == nil { + return nil + } + // Retry only for the specific error + if strings.Contains(err.Error(), "rate limit exceeded") { + fmt.Printf("Attempt %d: Temporary error occurred, retrying...\n", i+1) + time.Sleep(delay) + continue + } + // Return immediately for any other error + return err + } + return fmt.Errorf("operation failed after %d retries", retries) + } + return retryForSpecificError(10, 10*time.Second) } jobId := b.getBackupCollectionWorkerPool().SubmitWithId(job) jobIds = append(jobIds, jobId) diff --git a/core/storage/local_chunk_manager.go b/core/storage/local_chunk_manager.go index 71fa338e..47de78d5 100644 --- a/core/storage/local_chunk_manager.go +++ b/core/storage/local_chunk_manager.go @@ -234,7 +234,6 @@ func (lcm *LocalChunkManager) UploadObject(ctx context.Context, i UploadObjectIn return err } - fmt.Println("Successfully written to file!") return nil } @@ -356,7 +355,7 @@ func CopyDir(source string, dest string) (err error) { } func CopyFile(source string, dest string) (err error) { - + // get properties of source parent dir sourceParentDir := filepath.Dir(source) sourceParentDirInfo, err := os.Stat(sourceParentDir)