Skip to content

Commit 59664a7

Browse files
ulya-sidorinaulya-sidorina
committed
feat(make_backup): add root_path for backup
1 parent 90dad61 commit 59664a7

File tree

19 files changed

+291
-186
lines changed

19 files changed

+291
-186
lines changed

‎internal/backup_operations/make_backup.go‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ type MakeBackupInternalRequest struct {
3030
ContainerID string
3131
DatabaseEndpoint string
3232
DatabaseName string
33+
RootPath string
3334
SourcePaths []string
3435
SourcePathsToExclude []string
3536
ScheduleID *string
@@ -42,6 +43,7 @@ func FromBackupSchedule(schedule *types.BackupSchedule) MakeBackupInternalReques
4243
ContainerID: schedule.ContainerID,
4344
DatabaseEndpoint: schedule.DatabaseEndpoint,
4445
DatabaseName: schedule.DatabaseName,
46+
RootPath: schedule.RootPath,
4547
SourcePaths: schedule.SourcePaths,
4648
SourcePathsToExclude: schedule.SourcePathsToExclude,
4749
ScheduleID: &schedule.ID,
@@ -57,6 +59,7 @@ func FromTBWROperation(tbwr *types.TakeBackupWithRetryOperation) MakeBackupInter
5759
ContainerID: tbwr.ContainerID,
5860
DatabaseEndpoint: tbwr.YdbConnectionParams.Endpoint,
5961
DatabaseName: tbwr.YdbConnectionParams.DatabaseName,
62+
RootPath: tbwr.RootPath,
6063
SourcePaths: tbwr.SourcePaths,
6164
SourcePathsToExclude: tbwr.SourcePathsToExclude,
6265
ScheduleID: tbwr.ScheduleID,
@@ -141,9 +144,14 @@ func ValidateSourcePaths(
141144
if req.ScheduleID != nil {
142145
ctx = xlog.With(ctx, zap.String("ScheduleID", *req.ScheduleID))
143146
}
147+
basePath, ok := SafePathJoin(req.DatabaseName, req.RootPath)
148+
if !ok {
149+
xlog.Error(ctx, "incorrect root path", zap.String("path", req.RootPath))
150+
return nil, status.Errorf(codes.InvalidArgument, "incorrect root path %s", req.RootPath)
151+
}
144152
sourcePaths := make([]string, 0, len(req.SourcePaths))
145153
for _, p := range req.SourcePaths {
146-
fullPath, ok := SafePathJoin(req.DatabaseName, p)
154+
fullPath, ok := SafePathJoin(basePath, p)
147155
if !ok {
148156
xlog.Error(ctx, "incorrect source path", zap.String("path", p))
149157
return nil, status.Errorf(codes.InvalidArgument, "incorrect source path %s", p)
@@ -345,6 +353,7 @@ func MakeBackup(
345353
SecretKey: secretKey,
346354
Description: "ydbcp backup", // TODO: the description shoud be better
347355
NumberOfRetries: 10, // TODO: get it from configuration
356+
RootPath: req.RootPath,
348357
SourcePaths: pathsForExport,
349358
DestinationPrefix: destinationPrefix,
350359
S3ForcePathStyle: s3.S3ForcePathStyle,
@@ -393,6 +402,7 @@ func MakeBackup(
393402
Endpoint: req.DatabaseEndpoint,
394403
DatabaseName: req.DatabaseName,
395404
},
405+
RootPath: req.RootPath,
396406
SourcePaths: req.SourcePaths,
397407
SourcePathsToExclude: req.SourcePathsToExclude,
398408
Audit: &pb.AuditInfo{

‎internal/connectors/client/connector.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -268,7 +268,7 @@ func (d *ClientYdbConnector) ExportToS3(
268268
}
269269

270270
if featureFlags.EnableNewPathsFormat {
271-
exportRequest.Settings.SourcePath = clientDb.Name()
271+
exportRequest.Settings.SourcePath = path.Join(clientDb.Name(), s3Settings.RootPath)
272272
exportRequest.Settings.DestinationPrefix = s3Settings.DestinationPrefix
273273
}
274274

‎internal/connectors/db/process_result_set.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ func ReadOperationFromResultSet(res query.Row) (types.Operation, error) {
143143
ydbOperationId *string
144144
operationStateBuf *string
145145
message *string
146+
rootPath *string
146147
sourcePaths *string
147148
sourcePathsToExclude *string
148149
creator *string
@@ -168,6 +169,7 @@ func ReadOperationFromResultSet(res query.Row) (types.Operation, error) {
168169
query.Named("operation_id", &ydbOperationId),
169170
query.Named("status", &operationStateBuf),
170171
query.Named("message", &message),
172+
query.Named("root_path", &rootPath),
171173
query.Named("paths", &sourcePaths),
172174
query.Named("paths_to_exclude", &sourcePathsToExclude),
173175

@@ -223,6 +225,7 @@ func ReadOperationFromResultSet(res query.Row) (types.Operation, error) {
223225
Endpoint: databaseEndpoint,
224226
DatabaseName: databaseName,
225227
},
228+
RootPath: StringOrEmpty(rootPath),
226229
SourcePaths: sourcePathsSlice,
227230
SourcePathsToExclude: sourcePathsToExcludeSlice,
228231
YdbOperationId: StringOrEmpty(ydbOperationId),
@@ -301,6 +304,7 @@ func ReadOperationFromResultSet(res query.Row) (types.Operation, error) {
301304
Endpoint: databaseEndpoint,
302305
DatabaseName: databaseName,
303306
},
307+
RootPath: StringOrEmpty(rootPath),
304308
SourcePaths: sourcePathsSlice,
305309
SourcePathsToExclude: sourcePathsToExcludeSlice,
306310
Audit: auditFromDb(creator, createdAt, completedAt),
@@ -330,6 +334,7 @@ func ReadBackupScheduleFromResultSet(res query.Row, withRPOInfo bool) (*types.Ba
330334
createdAt *time.Time
331335
name *string
332336
ttl *time.Duration
337+
rootPath *string
333338
sourcePaths *string
334339
sourcePathsToExclude *string
335340
recoveryPointObjective *time.Duration
@@ -351,6 +356,7 @@ func ReadBackupScheduleFromResultSet(res query.Row, withRPOInfo bool) (*types.Ba
351356
query.Named("created_at", &createdAt),
352357
query.Named("name", &name),
353358
query.Named("ttl", &ttl),
359+
query.Named("root_path", &rootPath),
354360
query.Named("paths", &sourcePaths),
355361
query.Named("paths_to_exclude", &sourcePathsToExclude),
356362
query.Named("recovery_point_objective", &recoveryPointObjective),
@@ -400,6 +406,7 @@ func ReadBackupScheduleFromResultSet(res query.Row, withRPOInfo bool) (*types.Ba
400406
ContainerID: containerID,
401407
DatabaseName: databaseName,
402408
DatabaseEndpoint: databaseEndpoint,
409+
RootPath: StringOrEmpty(rootPath),
403410
SourcePaths: sourcePathsSlice,
404411
SourcePathsToExclude: sourcePathsToExcludeSlice,
405412
Audit: auditFromDb(initiated, createdAt, nil),

‎internal/connectors/db/yql/queries/write.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,9 @@ func BuildCreateOperationQuery(operation types.Operation, index int) WriteSingle
117117
"$operation_id",
118118
table_types.StringValueFromString(tb.YdbOperationId),
119119
)
120+
if len(tb.RootPath) > 0 {
121+
d.AddValueParam("$root_path", table_types.StringValueFromString(tb.RootPath))
122+
}
120123
if len(tb.SourcePaths) > 0 {
121124
d.AddValueParam("$paths", table_types.StringValueFromString(types.SerializeSourcePaths(tb.SourcePaths)))
122125
}
@@ -146,6 +149,9 @@ func BuildCreateOperationQuery(operation types.Operation, index int) WriteSingle
146149
"$endpoint",
147150
table_types.StringValueFromString(tbwr.YdbConnectionParams.Endpoint),
148151
)
152+
if len(tbwr.RootPath) > 0 {
153+
d.AddValueParam("$root_path", table_types.StringValueFromString(tbwr.RootPath))
154+
}
149155
if len(tbwr.SourcePaths) > 0 {
150156
d.AddValueParam("$paths", table_types.StringValueFromString(types.SerializeSourcePaths(tbwr.SourcePaths)))
151157
}
@@ -357,6 +363,9 @@ func BuildCreateBackupScheduleQuery(schedule types.BackupSchedule, index int) Wr
357363
if schedule.ScheduleSettings.Ttl != nil {
358364
d.AddValueParam("$ttl", table_types.IntervalValueFromDuration(schedule.ScheduleSettings.Ttl.AsDuration()))
359365
}
366+
if len(schedule.RootPath) > 0 {
367+
d.AddValueParam("$root_path", table_types.StringValueFromString(schedule.RootPath))
368+
}
360369
if len(schedule.SourcePaths) > 0 {
361370
d.AddValueParam("$paths", table_types.StringValueFromString(types.SerializeSourcePaths(schedule.SourcePaths)))
362371
}

‎internal/connectors/db/yql/schema/create_tables.yql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ CREATE TABLE Operations (
5151
status String,
5252
message String,
5353

54+
root_path String,
5455
paths String,
5556
paths_to_exclude String,
5657
operation_id String,
@@ -79,6 +80,7 @@ CREATE TABLE BackupSchedules (
7980

8081
crontab String NOT NULL,
8182
ttl Interval,
83+
root_path String,
8284
paths String,
8385
paths_to_exclude String,
8486

‎internal/handlers/schedule_backup.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ func BackupScheduleHandler(
6767
Endpoint: schedule.DatabaseEndpoint,
6868
DatabaseName: schedule.DatabaseName,
6969
},
70+
RootPath: schedule.RootPath,
7071
SourcePaths: schedule.SourcePaths,
7172
SourcePathsToExclude: schedule.SourcePathsToExclude,
7273
Audit: &pb.AuditInfo{

‎internal/server/services/backup/backup_service.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,11 @@ func (s *BackupService) MakeBackup(ctx context.Context, req *pb.MakeBackupReques
112112
ctx = xlog.With(ctx, zap.String("SubjectID", subject))
113113
now := timestamppb.Now()
114114

115+
if len(req.RootPath) > 0 && !s.featureFlags.EnableNewPathsFormat {
116+
s.IncApiCallsCounter(methodName, codes.Unimplemented)
117+
return nil, status.Error(codes.Unimplemented, "backup root path is not supported yet")
118+
}
119+
115120
if req.EncryptionSettings != nil {
116121
s.IncApiCallsCounter(methodName, codes.Unimplemented)
117122
return nil, status.Error(codes.Unimplemented, "backup encryption is not supported yet")
@@ -126,6 +131,7 @@ func (s *BackupService) MakeBackup(ctx context.Context, req *pb.MakeBackupReques
126131
Endpoint: req.DatabaseEndpoint,
127132
DatabaseName: req.DatabaseName,
128133
},
134+
RootPath: req.RootPath,
129135
SourcePaths: req.SourcePaths,
130136
SourcePathsToExclude: req.SourcePathsToExclude,
131137
Audit: &pb.AuditInfo{

‎internal/server/services/backup_schedule/backup_schedule_service.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,11 @@ func (s *BackupScheduleService) CreateBackupSchedule(
129129
return nil, status.Error(codes.FailedPrecondition, "recovery point objective should be greater than 0")
130130
}
131131

132+
if len(request.RootPath) > 0 && !s.config.FeatureFlags.EnableNewPathsFormat {
133+
s.IncApiCallsCounter(methodName, codes.Unimplemented)
134+
return nil, status.Error(codes.Unimplemented, "backup root path is not supported yet")
135+
}
136+
132137
var scheduleName *string
133138
if len(request.ScheduleName) > 0 {
134139
scheduleName = &request.ScheduleName
@@ -139,6 +144,7 @@ func (s *BackupScheduleService) CreateBackupSchedule(
139144
ContainerID: request.ContainerId,
140145
DatabaseName: request.DatabaseName,
141146
DatabaseEndpoint: request.Endpoint,
147+
RootPath: request.RootPath,
142148
SourcePaths: request.SourcePaths,
143149
SourcePathsToExclude: request.SourcePathsToExclude,
144150
Audit: &pb.AuditInfo{

‎internal/types/backup_schedule.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type BackupSchedule struct {
2525
ContainerID string
2626
DatabaseName string
2727
DatabaseEndpoint string
28+
RootPath string
2829
SourcePaths []string
2930
SourcePathsToExclude []string
3031
Audit *pb.AuditInfo
@@ -74,6 +75,7 @@ func (b *BackupSchedule) Proto(clock clockwork.Clock) *pb.BackupSchedule {
7475
Id: b.ID,
7576
ContainerId: b.ContainerID,
7677
DatabaseName: b.DatabaseName,
78+
RootPath: b.RootPath,
7779
Endpoint: b.DatabaseEndpoint,
7880
Audit: b.Audit,
7981
Status: pb.BackupSchedule_Status(pb.BackupSchedule_Status_value[b.Status]),

‎internal/types/operation.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ type TakeBackupOperation struct {
4545
Message string
4646
YdbConnectionParams YdbConnectionParams
4747
YdbOperationId string
48+
RootPath string
4849
SourcePaths []string
4950
SourcePathsToExclude []string
5051
Audit *pb.AuditInfo
@@ -107,6 +108,7 @@ func (o *TakeBackupOperation) Proto() *pb.Operation {
107108
DatabaseEndpoint: o.YdbConnectionParams.Endpoint,
108109
YdbServerOperationId: o.YdbOperationId,
109110
BackupId: o.BackupID,
111+
RootPath: o.RootPath,
110112
SourcePaths: o.SourcePaths,
111113
SourcePathsToExclude: o.SourcePathsToExclude,
112114
RestorePaths: nil,

0 commit comments

Comments
 (0)