From b0c5cbde1febd6b3408162b588e4124fd14aaca3 Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Wed, 9 Oct 2024 12:30:46 +0200 Subject: [PATCH] feat(backup_service): add source paths to backup --- internal/backup_operations/make_backup.go | 32 ++++--- internal/connectors/client/connector.go | 47 +++++------ internal/connectors/client/mock.go | 6 ++ internal/connectors/db/process_result_set.go | 8 ++ internal/connectors/db/yql/queries/write.go | 1 + .../connectors/db/yql/queries/write_test.go | 3 +- internal/types/backup.go | 12 +-- internal/types/settings.go | 21 +++-- .../schedule_watcher/schedule_watcher_test.go | 36 +++++++- .../watchers/ttl_watcher/ttl_watcher_test.go | 12 ++- internal/watchers/watcher.go | 21 +++-- pkg/proto/ydbcp/v1alpha1/backup.pb.go | 84 +++++++++++-------- pkg/proto/ydbcp/v1alpha1/backup.proto | 1 + 13 files changed, 184 insertions(+), 100 deletions(-) diff --git a/internal/backup_operations/make_backup.go b/internal/backup_operations/make_backup.go index 511eccc9..5d51065d 100644 --- a/internal/backup_operations/make_backup.go +++ b/internal/backup_operations/make_backup.go @@ -122,18 +122,23 @@ func MakeBackup( sourcePaths = append(sourcePaths, fullPath) } + pathsForExport, err := clientConn.PreparePathsForExport(ctx, client, sourcePaths, req.GetSourcePathsToExclude()) + if err != nil { + xlog.Error(ctx, "error preparing paths for export", zap.Error(err)) + return nil, nil, status.Errorf(codes.Unknown, "error preparing paths for export, dsn %s", dsn) + } + s3Settings := types.ExportSettings{ - Endpoint: s3.Endpoint, - Region: s3.Region, - Bucket: s3.Bucket, - AccessKey: accessKey, - SecretKey: secretKey, - Description: "ydbcp backup", // TODO: the description shoud be better - NumberOfRetries: 10, // TODO: get it from configuration - SourcePaths: sourcePaths, - SourcePathToExclude: req.GetSourcePathsToExclude(), - DestinationPrefix: destinationPrefix, - S3ForcePathStyle: s3.S3ForcePathStyle, + Endpoint: s3.Endpoint, + Region: s3.Region, + Bucket: s3.Bucket, + AccessKey: accessKey, + SecretKey: secretKey, + Description: "ydbcp backup", // TODO: the description shoud be better + NumberOfRetries: 10, // TODO: get it from configuration + SourcePaths: pathsForExport, + DestinationPrefix: destinationPrefix, + S3ForcePathStyle: s3.S3ForcePathStyle, } clientOperationID, err := clientConn.ExportToS3(ctx, client, s3Settings) @@ -165,8 +170,9 @@ func MakeBackup( CreatedAt: now, Creator: subject, }, - ScheduleID: scheduleId, - ExpireAt: expireAt, + ScheduleID: scheduleId, + ExpireAt: expireAt, + SourcePaths: pathsForExport, } op := &types.TakeBackupOperation{ diff --git a/internal/connectors/client/connector.go b/internal/connectors/client/connector.go index 0c95055c..395a81d6 100644 --- a/internal/connectors/client/connector.go +++ b/internal/connectors/client/connector.go @@ -35,6 +35,7 @@ type ClientConnector interface { Open(ctx context.Context, dsn string) (*ydb.Driver, error) Close(ctx context.Context, clientDb *ydb.Driver) error + PreparePathsForExport(ctx context.Context, clientDb *ydb.Driver, sourcePaths []string, sourcePathsToExclude []string) ([]string, error) ExportToS3(ctx context.Context, clientDb *ydb.Driver, s3Settings types.ExportSettings) (string, error) ImportFromS3(ctx context.Context, clientDb *ydb.Driver, s3Settings types.ImportSettings) (string, error) GetOperationStatus( @@ -173,13 +174,17 @@ func listDirectory(ctx context.Context, clientDb *ydb.Driver, initialPath string return result, nil } -func prepareItemsForExport( - ctx context.Context, clientDb *ydb.Driver, s3Settings types.ExportSettings, -) ([]*Ydb_Export.ExportToS3Settings_Item, error) { +func (d *ClientYdbConnector) PreparePathsForExport( + ctx context.Context, clientDb *ydb.Driver, sourcePaths []string, sourcePathsToExclude []string, +) ([]string, error) { + if clientDb == nil { + return nil, fmt.Errorf("unititialized client db driver") + } + sources := make([]string, 0) - exclusions := make([]regexp.Regexp, len(s3Settings.SourcePathToExclude)) + exclusions := make([]regexp.Regexp, len(sourcePathsToExclude)) - for i, excludePath := range s3Settings.SourcePathToExclude { + for i, excludePath := range sourcePathsToExclude { reg, err := regexp.Compile(excludePath) if err != nil { return nil, fmt.Errorf("error compiling exclude path regexp: %s", err.Error()) @@ -188,8 +193,8 @@ func prepareItemsForExport( exclusions[i] = *reg } - if len(s3Settings.SourcePaths) > 0 { - for _, sourcePath := range s3Settings.SourcePaths { + if len(sourcePaths) > 0 { + for _, sourcePath := range sourcePaths { list, err := listDirectory(ctx, clientDb, sourcePath, exclusions) if err != nil { return nil, err @@ -207,9 +212,18 @@ func prepareItemsForExport( sources = append(sources, list...) } - items := make([]*Ydb_Export.ExportToS3Settings_Item, len(sources)) + return sources, nil +} + +func (d *ClientYdbConnector) ExportToS3( + ctx context.Context, clientDb *ydb.Driver, s3Settings types.ExportSettings, +) (string, error) { + if clientDb == nil { + return "", fmt.Errorf("unititialized client db driver") + } - for i, source := range sources { + items := make([]*Ydb_Export.ExportToS3Settings_Item, len(s3Settings.SourcePaths)) + for i, source := range s3Settings.SourcePaths { // Destination prefix format: s3_destination_prefix/rel_source_path destinationPrefix := path.Join( s3Settings.DestinationPrefix, @@ -222,21 +236,6 @@ func prepareItemsForExport( } } - return items, nil -} - -func (d *ClientYdbConnector) ExportToS3( - ctx context.Context, clientDb *ydb.Driver, s3Settings types.ExportSettings, -) (string, error) { - if clientDb == nil { - return "", fmt.Errorf("unititialized client db driver") - } - - items, err := prepareItemsForExport(ctx, clientDb, s3Settings) - if err != nil { - return "", fmt.Errorf("error preparing list of items for export: %s", err.Error()) - } - exportClient := Ydb_Export_V1.NewExportServiceClient(ydb.GRPCConn(clientDb)) xlog.Info( ctx, "Exporting data to s3", diff --git a/internal/connectors/client/mock.go b/internal/connectors/client/mock.go index a45c65d3..e214058d 100644 --- a/internal/connectors/client/mock.go +++ b/internal/connectors/client/mock.go @@ -52,6 +52,12 @@ func (m *MockClientConnector) Close(_ context.Context, _ *ydb.Driver) error { return nil } +func (m *MockClientConnector) PreparePathsForExport( + _ context.Context, _ *ydb.Driver, sourcePaths []string, _ []string, +) ([]string, error) { + return sourcePaths, nil +} + func (m *MockClientConnector) ExportToS3(_ context.Context, _ *ydb.Driver, s3Settings types.ExportSettings) (string, error) { objects := make([]ObjectPath, 0) for _, source := range s3Settings.SourcePaths { diff --git a/internal/connectors/db/process_result_set.go b/internal/connectors/db/process_result_set.go index a7a9c075..9ee9c15d 100644 --- a/internal/connectors/db/process_result_set.go +++ b/internal/connectors/db/process_result_set.go @@ -74,6 +74,7 @@ func ReadBackupFromResultSet(res result.Result) (*types.Backup, error) { message *string size *int64 scheduleId *string + sourcePaths *string creator *string completedAt *time.Time @@ -95,6 +96,7 @@ func ReadBackupFromResultSet(res result.Result) (*types.Backup, error) { named.Optional("size", &size), named.Optional("schedule_id", &scheduleId), named.Optional("expire_at", &expireAt), + named.Optional("paths", &sourcePaths), named.Optional("created_at", &createdAt), named.Optional("completed_at", &completedAt), @@ -104,6 +106,11 @@ func ReadBackupFromResultSet(res result.Result) (*types.Backup, error) { return nil, err } + sourcePathsSlice := make([]string, 0) + if sourcePaths != nil { + sourcePathsSlice = strings.Split(*sourcePaths, ",") + } + return &types.Backup{ ID: backupId, ContainerID: containerId, @@ -119,6 +126,7 @@ func ReadBackupFromResultSet(res result.Result) (*types.Backup, error) { Size: Int64OrZero(size), ScheduleID: scheduleId, ExpireAt: expireAt, + SourcePaths: sourcePathsSlice, }, nil } diff --git a/internal/connectors/db/yql/queries/write.go b/internal/connectors/db/yql/queries/write.go index 46f12cce..71ffb2b5 100644 --- a/internal/connectors/db/yql/queries/write.go +++ b/internal/connectors/db/yql/queries/write.go @@ -254,6 +254,7 @@ func BuildCreateBackupQuery(b types.Backup, index int) WriteSingleTableQueryImpl d.AddValueParam("$status", table_types.StringValueFromString(b.Status)) d.AddValueParam("$message", table_types.StringValueFromString(b.Message)) d.AddValueParam("$size", table_types.Int64Value(b.Size)) + d.AddValueParam("$paths", table_types.StringValueFromString(strings.Join(b.SourcePaths, ","))) if b.ScheduleID != nil { d.AddValueParam("$schedule_id", table_types.StringValueFromString(*b.ScheduleID)) } diff --git a/internal/connectors/db/yql/queries/write_test.go b/internal/connectors/db/yql/queries/write_test.go index 25cd35e9..109ade9a 100644 --- a/internal/connectors/db/yql/queries/write_test.go +++ b/internal/connectors/db/yql/queries/write_test.go @@ -57,7 +57,7 @@ UPDATE Operations SET status = $status_1, message = $message_1 WHERE id = $id_1` func TestQueryBuilder_CreateCreate(t *testing.T) { const ( - queryString = `UPSERT INTO Backups (id, container_id, database, endpoint, s3_endpoint, s3_region, s3_bucket, s3_path_prefix, status, message, size, initiated, created_at) VALUES ($id_0, $container_id_0, $database_0, $endpoint_0, $s3_endpoint_0, $s3_region_0, $s3_bucket_0, $s3_path_prefix_0, $status_0, $message_0, $size_0, $initiated_0, $created_at_0); + queryString = `UPSERT INTO Backups (id, container_id, database, endpoint, s3_endpoint, s3_region, s3_bucket, s3_path_prefix, status, message, size, paths, initiated, created_at) VALUES ($id_0, $container_id_0, $database_0, $endpoint_0, $s3_endpoint_0, $s3_region_0, $s3_bucket_0, $s3_path_prefix_0, $status_0, $message_0, $size_0, $paths_0, $initiated_0, $created_at_0); UPSERT INTO Operations (id, type, status, message, initiated, created_at, container_id, database, endpoint, backup_id, operation_id) VALUES ($id_1, $type_1, $status_1, $message_1, $initiated_1, $created_at_1, $container_id_1, $database_1, $endpoint_1, $backup_id_1, $operation_id_1)` ) opId := types.GenerateObjectID() @@ -112,6 +112,7 @@ UPSERT INTO Operations (id, type, status, message, initiated, created_at, contai table.ValueParam("$status_0", table_types.StringValueFromString(types.BackupStateAvailable)), table.ValueParam("$message_0", table_types.StringValueFromString("msg backup")), table.ValueParam("$size_0", table_types.Int64Value(0)), + table.ValueParam("$paths_0", table_types.StringValueFromString("")), table.ValueParam("$initiated_0", table_types.StringValueFromString("author")), table.ValueParam( "$created_at_0", diff --git a/internal/types/backup.go b/internal/types/backup.go index d00ca241..48cf54ad 100644 --- a/internal/types/backup.go +++ b/internal/types/backup.go @@ -51,6 +51,7 @@ type Backup struct { Size int64 ScheduleID *string ExpireAt *time.Time + SourcePaths []string } func (o *Backup) String() string { @@ -76,11 +77,12 @@ func (o *Backup) Proto() *pb.Backup { Bucket: o.S3Bucket, PathPrefix: o.S3PathPrefix, }, - Audit: o.AuditInfo, - Size: o.Size, - Status: pb.Backup_Status(pb.Backup_Status_value[o.Status]), - Message: o.Message, - ExpireAt: nil, + Audit: o.AuditInfo, + Size: o.Size, + Status: pb.Backup_Status(pb.Backup_Status_value[o.Status]), + Message: o.Message, + ExpireAt: nil, + SourcePaths: o.SourcePaths, } if o.ScheduleID != nil { backup.ScheduleId = *o.ScheduleID diff --git a/internal/types/settings.go b/internal/types/settings.go index ca202641..02d0a6ef 100644 --- a/internal/types/settings.go +++ b/internal/types/settings.go @@ -11,17 +11,16 @@ func MakeYdbConnectionString(params YdbConnectionParams) string { } type ExportSettings struct { - Endpoint string - Region string - Bucket string - AccessKey string - SecretKey string - Description string - NumberOfRetries uint32 - SourcePaths []string - SourcePathToExclude []string - DestinationPrefix string - S3ForcePathStyle bool + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + Description string + NumberOfRetries uint32 + SourcePaths []string + DestinationPrefix string + S3ForcePathStyle bool } type ImportSettings struct { diff --git a/internal/watchers/schedule_watcher/schedule_watcher_test.go b/internal/watchers/schedule_watcher/schedule_watcher_test.go index 7cbede35..a034b34c 100644 --- a/internal/watchers/schedule_watcher/schedule_watcher_test.go +++ b/internal/watchers/schedule_watcher/schedule_watcher_test.go @@ -78,6 +78,7 @@ func TestScheduleWatcherSimple(t *testing.T) { queries.NewWriteTableQueryMock, ) + scheduleWatcherActionCompleted := make(chan struct{}) _ = NewScheduleWatcher( ctx, &wg, @@ -85,6 +86,7 @@ func TestScheduleWatcherSimple(t *testing.T) { dbConnector, handler, watchers.WithTickerProvider(tickerProvider), + watchers.WithActionCompletedChannel(&scheduleWatcherActionCompleted), ) // Wait for the ticker to be initialized @@ -96,7 +98,15 @@ func TestScheduleWatcherSimple(t *testing.T) { } fakeTicker.Send(clock.Now()) - cancel() + + // Wait for the watcher action to be completed + select { + case <-ctx.Done(): + t.Error("action wasn't completed") + case <-scheduleWatcherActionCompleted: + cancel() + } + wg.Wait() // check operation status (should be pending) @@ -190,6 +200,7 @@ func TestScheduleWatcherTwoSchedulesOneBackup(t *testing.T) { queries.NewWriteTableQueryMock, ) + scheduleWatcherActionCompleted := make(chan struct{}) _ = NewScheduleWatcher( ctx, &wg, @@ -197,6 +208,7 @@ func TestScheduleWatcherTwoSchedulesOneBackup(t *testing.T) { dbConnector, handler, watchers.WithTickerProvider(tickerProvider), + watchers.WithActionCompletedChannel(&scheduleWatcherActionCompleted), ) // Wait for the ticker to be initialized @@ -208,7 +220,15 @@ func TestScheduleWatcherTwoSchedulesOneBackup(t *testing.T) { } fakeTicker.Send(clock.Now()) - cancel() + + // Wait for the watcher action to be completed + select { + case <-ctx.Done(): + t.Error("action wasn't completed") + case <-scheduleWatcherActionCompleted: + cancel() + } + wg.Wait() // check operation status (should be pending) @@ -309,6 +329,7 @@ func TestScheduleWatcherTwoBackups(t *testing.T) { queries.NewWriteTableQueryMock, ) + scheduleWatcherActionCompleted := make(chan struct{}) _ = NewScheduleWatcher( ctx, &wg, @@ -316,6 +337,7 @@ func TestScheduleWatcherTwoBackups(t *testing.T) { dbConnector, handler, watchers.WithTickerProvider(tickerProvider), + watchers.WithActionCompletedChannel(&scheduleWatcherActionCompleted), ) // Wait for the ticker to be initialized @@ -327,7 +349,15 @@ func TestScheduleWatcherTwoBackups(t *testing.T) { } fakeTicker.Send(clock.Now()) - cancel() + + // Wait for the watcher action to be completed + select { + case <-ctx.Done(): + t.Error("action wasn't completed") + case <-scheduleWatcherActionCompleted: + cancel() + } + wg.Wait() // check operation status (should be pending) diff --git a/internal/watchers/ttl_watcher/ttl_watcher_test.go b/internal/watchers/ttl_watcher/ttl_watcher_test.go index e4b66a3c..e2e5a913 100644 --- a/internal/watchers/ttl_watcher/ttl_watcher_test.go +++ b/internal/watchers/ttl_watcher/ttl_watcher_test.go @@ -45,12 +45,15 @@ func TestTtlWatcher(t *testing.T) { db := db.NewMockDBConnector( db.WithBackups(backupMap), ) + + ttlWatcherActionCompleted := make(chan struct{}) _ = NewTtlWatcher( ctx, &wg, db, queries.NewWriteTableQueryMock, watchers.WithTickerProvider(tickerProvider), + watchers.WithActionCompletedChannel(&ttlWatcherActionCompleted), ) // Wait for the ticker to be initialized @@ -65,7 +68,14 @@ func TestTtlWatcher(t *testing.T) { t0 := clock.Now().Add(time.Hour) fakeTicker.Send(t0) - cancel() + // Wait for the watcher action to be completed + select { + case <-ctx.Done(): + t.Error("action wasn't completed") + case <-ttlWatcherActionCompleted: + cancel() + } + wg.Wait() // Check that DeleteBackup operation was created diff --git a/internal/watchers/watcher.go b/internal/watchers/watcher.go index de5db433..5e1df91c 100644 --- a/internal/watchers/watcher.go +++ b/internal/watchers/watcher.go @@ -13,11 +13,12 @@ import ( type WatcherAction func(context.Context, time.Duration) type WatcherImpl struct { - ctx context.Context - period time.Duration - tickerProvider ticker.TickerProvider - action WatcherAction - prefixName string + ctx context.Context + period time.Duration + tickerProvider ticker.TickerProvider + action WatcherAction + prefixName string + actionCompleted *chan struct{} } type Option func(*WatcherImpl) @@ -28,6 +29,12 @@ func WithTickerProvider(ticketProvider ticker.TickerProvider) Option { } } +func WithActionCompletedChannel(actionCompleted *chan struct{}) Option { + return func(o *WatcherImpl) { + o.actionCompleted = actionCompleted + } +} + func NewWatcher( ctx context.Context, wg *sync.WaitGroup, @@ -67,6 +74,10 @@ func (o *WatcherImpl) run(wg *sync.WaitGroup) { xlog.Debug(o.ctx, fmt.Sprintf("Starting %s watcher action", o.prefixName)) o.action(o.ctx, o.period) xlog.Debug(o.ctx, fmt.Sprintf("%s watcher action was completed", o.prefixName)) + + if o.actionCompleted != nil { + close(*o.actionCompleted) + } } } } diff --git a/pkg/proto/ydbcp/v1alpha1/backup.pb.go b/pkg/proto/ydbcp/v1alpha1/backup.pb.go index b6a9e129..0a7274d2 100644 --- a/pkg/proto/ydbcp/v1alpha1/backup.pb.go +++ b/pkg/proto/ydbcp/v1alpha1/backup.pb.go @@ -156,6 +156,7 @@ type Backup struct { Message string `protobuf:"bytes,9,opt,name=message,proto3" json:"message,omitempty"` ExpireAt *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=expire_at,json=expireAt,proto3" json:"expire_at,omitempty"` ScheduleId string `protobuf:"bytes,11,opt,name=schedule_id,json=scheduleId,proto3" json:"schedule_id,omitempty"` + SourcePaths []string `protobuf:"bytes,12,rep,name=source_paths,json=sourcePaths,proto3" json:"source_paths,omitempty"` } func (x *Backup) Reset() { @@ -267,6 +268,13 @@ func (x *Backup) GetScheduleId() string { return "" } +func (x *Backup) GetSourcePaths() []string { + if x != nil { + return x.SourcePaths + } + return nil +} + type S3Location struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -408,7 +416,7 @@ var file_ydbcp_v1alpha1_backup_proto_rawDesc = []byte{ 0x2f, 0x62, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0e, 0x79, 0x64, 0x62, 0x63, 0x70, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb5, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xd8, 0x04, 0x0a, 0x06, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -436,43 +444,45 @@ var file_ydbcp_v1alpha1_backup_proto_rawDesc = []byte{ 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x08, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, - 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x22, 0x7e, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, - 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x45, 0x4e, 0x44, - 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, 0x4c, 0x41, 0x42, - 0x4c, 0x45, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x03, 0x12, - 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x0b, - 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x52, - 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x45, 0x4c, 0x45, - 0x54, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x22, 0x79, 0x0a, 0x0a, 0x53, 0x33, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, - 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, - 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x22, 0x9f, 0x01, 0x0a, 0x09, 0x41, 0x75, 0x64, 0x69, 0x74, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x5f, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x50, 0x61, 0x74, 0x68, 0x73, 0x22, 0x7e, 0x0a, 0x06, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x55, 0x4e, + 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, + 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x41, 0x56, 0x41, 0x49, + 0x4c, 0x41, 0x42, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, + 0x10, 0x03, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, + 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x0b, + 0x0a, 0x07, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x44, + 0x45, 0x4c, 0x45, 0x54, 0x49, 0x4e, 0x47, 0x10, 0x07, 0x22, 0x79, 0x0a, 0x0a, 0x53, 0x33, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x6e, 0x64, 0x70, 0x6f, + 0x69, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, + 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, + 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x70, 0x61, 0x74, 0x68, 0x5f, 0x70, 0x72, 0x65, 0x66, + 0x69, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x70, 0x61, 0x74, 0x68, 0x50, 0x72, + 0x65, 0x66, 0x69, 0x78, 0x22, 0x9f, 0x01, 0x0a, 0x09, 0x41, 0x75, 0x64, 0x69, 0x74, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x39, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x3d, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x64, 0x41, 0x74, 0x12, 0x3d, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x41, 0x74, 0x2a, 0x5d, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x41, 0x54, 0x41, 0x42, 0x41, 0x53, 0x45, 0x5f, 0x4e, - 0x41, 0x4d, 0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x10, - 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x5f, 0x41, 0x54, 0x10, - 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x45, 0x58, 0x50, 0x49, 0x52, 0x45, 0x5f, 0x41, 0x54, 0x10, 0x03, - 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x5f, 0x41, 0x54, - 0x10, 0x04, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x79, 0x64, 0x62, 0x2d, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x79, 0x64, - 0x62, 0x63, 0x70, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x79, 0x64, - 0x62, 0x63, 0x70, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, 0x79, 0x64, 0x62, - 0x63, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x64, 0x41, 0x74, 0x2a, 0x5d, 0x0a, 0x0b, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x41, 0x54, 0x41, 0x42, 0x41, 0x53, + 0x45, 0x5f, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x41, 0x54, + 0x55, 0x53, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x44, 0x5f, + 0x41, 0x54, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x45, 0x58, 0x50, 0x49, 0x52, 0x45, 0x5f, 0x41, + 0x54, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, + 0x5f, 0x41, 0x54, 0x10, 0x04, 0x42, 0x3e, 0x5a, 0x3c, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x79, 0x64, 0x62, 0x2d, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, + 0x2f, 0x79, 0x64, 0x62, 0x63, 0x70, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x2f, 0x79, 0x64, 0x62, 0x63, 0x70, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3b, + 0x79, 0x64, 0x62, 0x63, 0x70, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/proto/ydbcp/v1alpha1/backup.proto b/pkg/proto/ydbcp/v1alpha1/backup.proto index 38576649..6d6fa22a 100644 --- a/pkg/proto/ydbcp/v1alpha1/backup.proto +++ b/pkg/proto/ydbcp/v1alpha1/backup.proto @@ -35,6 +35,7 @@ message Backup { string message = 9; google.protobuf.Timestamp expire_at = 10; string schedule_id = 11; + repeated string source_paths = 12; } message S3Location {