From eaf5a90fa435dfaa3e92807450024ff9f430ea49 Mon Sep 17 00:00:00 2001 From: Martin Martinez Rivera Date: Mon, 20 May 2019 15:29:17 -0700 Subject: [PATCH] Add field to backup requests to force a full backup. (#3387) This change adds a new field to the backup requests to force a full backup. Only the code that forces the full backup is included in this PR to keep it small. Manually tested using a minio instance locally. No existing tests will be broken by this change. Automated tests will be included later since the existing backup test needs some work before it can verify this feature. --- ee/backup/backup.go | 16 +++--- ee/backup/file_handler.go | 13 +++-- ee/backup/s3_handler.go | 11 ++-- protos/pb.proto | 5 ++ protos/pb/pb.pb.go | 111 ++++++++++++++++++++++++++------------ worker/backup_ee.go | 21 +++----- 6 files changed, 110 insertions(+), 67 deletions(-) diff --git a/ee/backup/backup.go b/ee/backup/backup.go index f0d9a1cd457..b7561c61c00 100644 --- a/ee/backup/backup.go +++ b/ee/backup/backup.go @@ -20,21 +20,20 @@ import ( "github.com/dgraph-io/badger" "github.com/dgraph-io/dgraph/protos/pb" - "github.com/dgraph-io/dgraph/x" - "github.com/golang/glog" ) -// ErrBackupNoChanges is returned when the manifest version is equal to the snapshot version. -// This means that no data updates happened since the last backup. -var ErrBackupNoChanges = x.Errorf("No changes since last backup, OK.") - // Request has all the information needed to perform a backup. type Request struct { DB *badger.DB // Badger pstore managed by this node. Backup *pb.BackupRequest Manifest *Manifest - Version uint64 + + // Version indicates the beginning timestamp from which the backup should start. + // For a partial backup, the Version is the largest Version from the previous manifest + // files. For a full backup, Version is set to zero so that all data is included. + // TODO(martinmr): rename this field to Since both here and in the manifest. + Version uint64 } // Process uses the request values to create a stream writer then hand off the data @@ -48,9 +47,6 @@ func (r *Request) Process(ctx context.Context) error { handler, err := r.newHandler() if err != nil { - if err != ErrBackupNoChanges { - glog.Errorf("Unable to get handler for request: %+v. Error: %v", r.Backup, err) - } return err } glog.V(3).Infof("Backup manifest version: %d", r.Version) diff --git a/ee/backup/file_handler.go b/ee/backup/file_handler.go index a05d0e2adc9..a4cca71f981 100644 --- a/ee/backup/file_handler.go +++ b/ee/backup/file_handler.go @@ -71,10 +71,7 @@ func (h *fileHandler) Create(uri *url.URL, req *Request) error { if err := h.readManifest(lastManifest, &m); err != nil { return err } - // No new changes since last check - if m.Version == req.Backup.SnapshotTs { - return ErrBackupNoChanges - } + // Return the version of last backup req.Version = m.Version } @@ -83,6 +80,12 @@ func (h *fileHandler) Create(uri *url.URL, req *Request) error { fileName = backupManifest } + // If a full backup is being forced, force the version to zero to stream all + // the contents from the database. + if req.Backup.ForceFull { + req.Version = 0 + } + dir = filepath.Join(uri.Path, fmt.Sprintf(backupPathFmt, req.Backup.UnixTs)) err := os.Mkdir(dir, 0700) if err != nil && !os.IsExist(err) { @@ -106,7 +109,7 @@ func (h *fileHandler) Load(uri *url.URL, fn loadFn) (uint64, error) { return 0, x.Errorf("The path %q does not exist or it is inaccessible.", uri.Path) } - // Get a lisst of all the manifest files at the location. + // Get a list of all the manifest files at the location. suffix := filepath.Join(string(filepath.Separator), backupManifest) manifests := x.WalkPathFunc(uri.Path, func(path string, isdir bool) bool { return !isdir && strings.HasSuffix(path, suffix) diff --git a/ee/backup/s3_handler.go b/ee/backup/s3_handler.go index 30e4521743a..add08ca6f35 100644 --- a/ee/backup/s3_handler.go +++ b/ee/backup/s3_handler.go @@ -180,10 +180,7 @@ func (h *s3Handler) Create(uri *url.URL, req *Request) error { if err := h.readManifest(mc, lastManifest, &m); err != nil { return err } - // No new changes since last check - if m.Version >= req.Backup.SnapshotTs { - return ErrBackupNoChanges - } + // Return the version of last backup req.Version = m.Version } @@ -192,6 +189,12 @@ func (h *s3Handler) Create(uri *url.URL, req *Request) error { objectName = backupManifest } + // If a full backup is being forced, force the version to zero to stream all + // the contents from the database. + if req.Backup.ForceFull { + req.Version = 0 + } + // The backup object is: folder1...folderN/dgraph.20181106.0113/r110001-g1.backup object := filepath.Join(h.objectPrefix, fmt.Sprintf(backupPathFmt, req.Backup.UnixTs), diff --git a/protos/pb.proto b/protos/pb.proto index 6f7b2014df7..44ca9bf589f 100644 --- a/protos/pb.proto +++ b/protos/pb.proto @@ -487,6 +487,11 @@ message BackupRequest { // True if no credentials should be used to access the S3 or minio bucket. // For example, when using a bucket with a public policy. bool anonymous = 10; + + // If true, previous backups will be ignored and a new full backup will be + // created. If false, the backup will be full or incremental depending on + // the existing backups. + bool force_full = 11; } message ExportRequest { diff --git a/protos/pb/pb.pb.go b/protos/pb/pb.pb.go index 61abb523498..c1ac1b451ae 100644 --- a/protos/pb/pb.pb.go +++ b/protos/pb/pb.pb.go @@ -3530,7 +3530,11 @@ type BackupRequest struct { SessionToken string `protobuf:"bytes,9,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"` // True if no credentials should be used to access the S3 or minio bucket. // For example, when using a bucket with a public policy. - Anonymous bool `protobuf:"varint,10,opt,name=anonymous,proto3" json:"anonymous,omitempty"` + Anonymous bool `protobuf:"varint,10,opt,name=anonymous,proto3" json:"anonymous,omitempty"` + // If true, previous backups will be ignored and a new full backup will be + // created. If false, the backup will be full or incremental depending on + // the existing backups. + ForceFull bool `protobuf:"varint,11,opt,name=force_full,json=forceFull,proto3" json:"force_full,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3639,6 +3643,13 @@ func (m *BackupRequest) GetAnonymous() bool { return false } +func (m *BackupRequest) GetForceFull() bool { + if m != nil { + return m.ForceFull + } + return false +} + type ExportRequest struct { GroupId uint32 `protobuf:"varint,1,opt,name=group_id,json=groupId,proto3" json:"group_id,omitempty"` ReadTs uint64 `protobuf:"varint,2,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` @@ -3776,13 +3787,13 @@ func init() { func init() { proto.RegisterFile("pb.proto", fileDescriptor_f80abaa17e25ccc8) } var fileDescriptor_f80abaa17e25ccc8 = []byte{ - // 3540 bytes of a gzipped FileDescriptorProto + // 3556 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x5a, 0xcd, 0x73, 0x24, 0x47, 0x56, 0x9f, 0xea, 0x8f, 0xea, 0xaa, 0xd7, 0x2d, 0x4d, 0x3b, 0xed, 0x1d, 0xb7, 0xb5, 0xeb, 0x19, 0xb9, 0xfc, 0x31, 0xf2, 0x78, 0xad, 0x19, 0xcb, 0x0b, 0xac, 0x97, 0xe0, 0xa0, 0x91, 0x7a, 0x06, 0xd9, 0xfa, 0x22, 0xbb, 0x35, 0xcb, 0xfa, 0x40, 0x47, 0xaa, 0x2a, 0xd5, 0xaa, 0x55, 0x75, 0x55, 0x51, 0x59, 0x2d, 0x5a, 0xbe, 0x71, 0x80, 0x08, 0x22, 0xb8, 0x71, 0xd9, 0x03, 0xc1, 0x61, 0x8f, - 0x5c, 0xe0, 0xb8, 0x67, 0x22, 0x88, 0xe0, 0x48, 0xf0, 0x17, 0x10, 0x86, 0x23, 0x67, 0x22, 0xb8, + 0x5c, 0xb8, 0xee, 0x99, 0x08, 0x22, 0xb8, 0x41, 0xf0, 0x17, 0x10, 0x86, 0x23, 0x67, 0x22, 0xb8, 0x11, 0xef, 0x65, 0xd6, 0x47, 0xf7, 0x68, 0xc6, 0x6b, 0x22, 0x38, 0x75, 0xbe, 0x8f, 0xfc, 0xfa, 0xbd, 0x97, 0x2f, 0x5f, 0xbe, 0x6a, 0x70, 0xd2, 0xf3, 0xed, 0x34, 0x4b, 0xf2, 0x84, 0x35, 0xd2, 0xf3, 0x0d, 0x57, 0xa4, 0xa1, 0x26, 0x37, 0x1e, 0x4e, 0xc3, 0xfc, 0x72, 0x7e, 0xbe, 0xed, 0x27, @@ -3827,7 +3838,7 @@ var fileDescriptor_f80abaa17e25ccc8 = []byte{ 0x5d, 0x2e, 0x2e, 0xf2, 0xbd, 0x24, 0xce, 0xe5, 0x22, 0x67, 0xeb, 0xd0, 0x08, 0x03, 0x82, 0xc8, 0xe6, 0x8d, 0x30, 0xc0, 0xc5, 0x4d, 0xb3, 0x64, 0x9e, 0x12, 0x42, 0x6b, 0x5c, 0x13, 0x04, 0x65, 0x10, 0x64, 0xb4, 0x62, 0x84, 0x32, 0x08, 0x32, 0xf6, 0x00, 0xba, 0x2a, 0x16, 0xa9, 0xba, 0x4c, - 0x72, 0x5c, 0x5c, 0x8b, 0x16, 0x07, 0x05, 0x6b, 0xac, 0xbc, 0x7f, 0xb6, 0xc0, 0x3e, 0x92, 0xb3, + 0x72, 0x5c, 0x5c, 0x8b, 0x16, 0x07, 0x05, 0x6b, 0xac, 0xbc, 0x7f, 0xb2, 0xc0, 0x3e, 0x92, 0xb3, 0x73, 0x99, 0xbd, 0x34, 0xcb, 0x3b, 0xe0, 0xd0, 0xc0, 0x93, 0x30, 0x30, 0x13, 0x75, 0x88, 0x3e, 0x08, 0x6e, 0x9d, 0xea, 0x1e, 0xd8, 0x91, 0x14, 0x08, 0xbe, 0xf6, 0x33, 0x43, 0x21, 0x36, 0x62, 0x36, 0x09, 0xa4, 0x08, 0x28, 0xf0, 0x38, 0xdc, 0x16, 0xb3, 0x7d, 0x29, 0x02, 0x5c, 0x5b, 0x24, @@ -3963,7 +3974,7 @@ var fileDescriptor_f80abaa17e25ccc8 = []byte{ 0x2a, 0xe3, 0xe0, 0xa0, 0x30, 0xa9, 0x26, 0x96, 0x67, 0x36, 0x45, 0x98, 0xf2, 0xe5, 0xfd, 0x0c, 0x7a, 0x45, 0x52, 0x7d, 0x24, 0x73, 0x41, 0xce, 0x16, 0x85, 0x32, 0xae, 0x39, 0xa2, 0xa3, 0x19, 0x63, 0xf5, 0x9a, 0x72, 0xaf, 0xb7, 0x0d, 0xb6, 0xf1, 0x64, 0x06, 0x2d, 0x3f, 0x09, 0xf4, 0x01, - 0x6a, 0x73, 0x6a, 0x23, 0x1c, 0x33, 0x35, 0x2d, 0x2e, 0xcf, 0x99, 0x9a, 0x7a, 0xff, 0xd8, 0x80, + 0x6a, 0x73, 0x6a, 0x23, 0x1c, 0x33, 0x35, 0x2d, 0x2e, 0xcf, 0x99, 0x9a, 0x7a, 0xff, 0xd2, 0x80, 0xb5, 0xa7, 0xc2, 0xbf, 0x9a, 0xa7, 0xc5, 0xed, 0x55, 0x7b, 0x19, 0x59, 0x4b, 0x2f, 0xa3, 0xd7, 0x14, 0x99, 0xdf, 0x86, 0xce, 0x3c, 0x0e, 0x17, 0x45, 0x5a, 0xe3, 0x72, 0x1b, 0x49, 0x5d, 0x5c, 0x8d, 0x12, 0x9f, 0x5e, 0x3c, 0x74, 0xe8, 0x5c, 0x5e, 0xd2, 0x54, 0xb7, 0xc0, 0x57, 0x94, 0xc1, @@ -3971,34 +3982,35 @@ var fileDescriptor_f80abaa17e25ccc8 = []byte{ 0xea, 0x49, 0xe3, 0x6a, 0xce, 0x57, 0xf2, 0x86, 0xc2, 0x83, 0xf4, 0x33, 0x99, 0x4f, 0xaa, 0x82, 0xa0, 0xab, 0x39, 0x28, 0x7e, 0x1f, 0xd6, 0x94, 0x54, 0x2a, 0x4c, 0xe2, 0x09, 0xdd, 0x27, 0xa6, 0x40, 0xd8, 0x33, 0xcc, 0x31, 0xf2, 0xd0, 0x0d, 0x44, 0x9c, 0xc4, 0x37, 0xb3, 0x64, 0xae, 0x8a, - 0xcf, 0x55, 0x25, 0xc3, 0xcb, 0x61, 0x6d, 0xb8, 0x48, 0xe9, 0x53, 0xc1, 0x77, 0xde, 0xf7, 0x35, - 0x30, 0x1b, 0x4b, 0x60, 0xae, 0x20, 0xd6, 0x2c, 0x11, 0xc3, 0x0c, 0x20, 0xc9, 0x66, 0x22, 0x37, - 0x78, 0x19, 0x6a, 0xe7, 0x9f, 0x2c, 0x68, 0xe1, 0x89, 0xc0, 0x47, 0xe6, 0x1f, 0x4a, 0x91, 0xe5, - 0xe7, 0x52, 0xe4, 0x6c, 0xc9, 0xfb, 0x37, 0x96, 0x28, 0xef, 0xce, 0x13, 0x8b, 0x6d, 0xeb, 0xaf, - 0x13, 0xc5, 0x47, 0x97, 0xb5, 0xe2, 0x5c, 0xd1, 0xb9, 0x5b, 0xd5, 0xdf, 0x22, 0xfd, 0x2f, 0x93, - 0x30, 0xde, 0xd3, 0x25, 0x7b, 0xb6, 0x7a, 0x0e, 0x57, 0x7b, 0xb0, 0x4f, 0xc1, 0x3e, 0x50, 0x78, - 0xe0, 0x5f, 0x56, 0xa5, 0xfb, 0xa4, 0x1e, 0x0b, 0xbc, 0x3b, 0x3b, 0xff, 0xd0, 0x84, 0xd6, 0xd7, - 0x32, 0x4b, 0xd8, 0x8f, 0xa1, 0x63, 0x0a, 0x72, 0xac, 0x56, 0x78, 0xdb, 0xa0, 0x84, 0x62, 0xa5, - 0x52, 0x47, 0xb3, 0xf4, 0xf5, 0x95, 0x54, 0xbd, 0x83, 0x59, 0x55, 0x2f, 0x7c, 0x69, 0x51, 0x5f, - 0x40, 0x7f, 0x94, 0x67, 0x52, 0xcc, 0x6a, 0xea, 0xcb, 0x40, 0xdd, 0xf6, 0xa8, 0x26, 0xbc, 0x3e, - 0x01, 0x5b, 0x47, 0xd5, 0x95, 0x0e, 0xab, 0xef, 0x63, 0x52, 0x7e, 0x08, 0xdd, 0xd1, 0x65, 0x32, - 0x8f, 0x82, 0x91, 0xcc, 0xae, 0x25, 0xab, 0x15, 0xc5, 0x37, 0x6a, 0x6d, 0xef, 0x0e, 0xdb, 0x02, - 0xd0, 0x81, 0xe3, 0x2c, 0x0c, 0x14, 0xeb, 0xa0, 0xec, 0x78, 0x3e, 0xd3, 0x83, 0xd6, 0x22, 0x8a, - 0xd6, 0xac, 0x05, 0xd7, 0xd7, 0x69, 0x7e, 0x0e, 0x6b, 0x7b, 0x74, 0xf9, 0x9c, 0x64, 0xbb, 0xe7, - 0x49, 0x96, 0xb3, 0xd5, 0xc2, 0xf8, 0xc6, 0x2a, 0xc3, 0xbb, 0xc3, 0x9e, 0x80, 0x33, 0xce, 0x6e, - 0xb4, 0xfe, 0x1b, 0xe6, 0x4e, 0xaa, 0xe6, 0xbb, 0x65, 0x97, 0x3b, 0xbf, 0x6e, 0x82, 0xfd, 0xf3, - 0x24, 0xbb, 0x92, 0x19, 0x7b, 0x04, 0x36, 0x15, 0x32, 0x8c, 0x1b, 0x95, 0x45, 0x8d, 0xdb, 0x26, - 0xfa, 0x00, 0x5c, 0x02, 0x65, 0x2c, 0xd4, 0x95, 0x36, 0x15, 0x7d, 0x3d, 0xd7, 0xb8, 0xe8, 0x6c, - 0x95, 0xec, 0xba, 0xae, 0x0d, 0x55, 0xd6, 0x75, 0x96, 0xaa, 0x0b, 0x1b, 0x1d, 0x5d, 0x2a, 0x18, - 0xa1, 0x6b, 0x3e, 0xb1, 0xd8, 0xc7, 0xd0, 0x1a, 0xe9, 0x9d, 0xa2, 0x52, 0xf5, 0x2d, 0x71, 0x63, - 0xbd, 0x60, 0x94, 0x23, 0x3f, 0x06, 0x5b, 0xa7, 0x32, 0x7a, 0x9b, 0x4b, 0xf9, 0xf9, 0x46, 0xbf, - 0xce, 0x32, 0x1d, 0x3e, 0x02, 0x5b, 0x87, 0x41, 0xdd, 0x61, 0x29, 0x24, 0x6e, 0x14, 0x76, 0xf0, - 0xee, 0xb0, 0x8f, 0xc1, 0xd6, 0x87, 0x5f, 0xeb, 0x2d, 0x05, 0x02, 0xbd, 0x3b, 0x1d, 0x7e, 0xb5, - 0xd7, 0x72, 0xe9, 0xcb, 0xb0, 0x96, 0xe1, 0xb0, 0x62, 0x47, 0xb7, 0x1c, 0xbd, 0x2f, 0x60, 0x6d, - 0x29, 0x1b, 0x62, 0x03, 0x42, 0xf9, 0x96, 0x04, 0x69, 0xb5, 0xf3, 0xd3, 0xfe, 0xbf, 0x7c, 0x7b, - 0xdf, 0xfa, 0xd7, 0x6f, 0xef, 0x5b, 0xff, 0xfe, 0xed, 0x7d, 0xeb, 0x57, 0xff, 0x71, 0xff, 0xce, - 0xb9, 0x4d, 0x7f, 0xb9, 0xf8, 0xfc, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0x60, 0xd2, 0xc3, 0xb7, - 0xb6, 0x21, 0x00, 0x00, + 0xcf, 0x55, 0x25, 0x03, 0x67, 0xa0, 0xc2, 0xcb, 0xe4, 0x62, 0x1e, 0x45, 0x26, 0xfa, 0xbb, 0xc4, + 0x79, 0x36, 0x8f, 0x22, 0x2f, 0x87, 0xb5, 0xe1, 0x22, 0xa5, 0x2f, 0x09, 0xdf, 0x99, 0x0e, 0xd4, + 0xb0, 0x6e, 0x2c, 0x61, 0xbd, 0x02, 0x68, 0xb3, 0x04, 0x14, 0x13, 0x84, 0x24, 0x9b, 0x89, 0xdc, + 0xc0, 0x69, 0xa8, 0x9d, 0x7f, 0xb4, 0xa0, 0x85, 0x07, 0x06, 0xdf, 0xa0, 0x7f, 0x28, 0x45, 0x96, + 0x9f, 0x4b, 0x91, 0xb3, 0xa5, 0xc3, 0xb1, 0xb1, 0x44, 0x79, 0x77, 0x9e, 0x58, 0x6c, 0x5b, 0x7f, + 0xbc, 0x28, 0xbe, 0xc9, 0xac, 0x15, 0xc7, 0x8e, 0x8e, 0xe5, 0xaa, 0xfe, 0x16, 0xe9, 0x7f, 0x99, + 0x84, 0xf1, 0x9e, 0xae, 0xe8, 0xb3, 0xd5, 0x63, 0xba, 0xda, 0x83, 0x7d, 0x0a, 0xf6, 0x81, 0xc2, + 0x78, 0xf0, 0xb2, 0x2a, 0x5d, 0x37, 0xf5, 0x50, 0xe1, 0xdd, 0xd9, 0xf9, 0x87, 0x26, 0xb4, 0xbe, + 0x96, 0x59, 0xc2, 0x7e, 0x0c, 0x1d, 0x53, 0xaf, 0x63, 0xb5, 0xba, 0xdc, 0x06, 0xe5, 0x1b, 0x2b, + 0x85, 0x3c, 0x9a, 0xa5, 0xaf, 0x6f, 0xac, 0xea, 0x99, 0xcc, 0xaa, 0x72, 0xe2, 0x4b, 0x8b, 0xfa, + 0x02, 0xfa, 0xa3, 0x3c, 0x93, 0x62, 0x56, 0x53, 0x5f, 0x06, 0xea, 0xb6, 0x37, 0x37, 0xe1, 0xf5, + 0x09, 0xd8, 0x3a, 0xe8, 0xae, 0x74, 0x58, 0x7d, 0x3e, 0x93, 0xf2, 0x43, 0xe8, 0x8e, 0x2e, 0x93, + 0x79, 0x14, 0x8c, 0x64, 0x76, 0x2d, 0x59, 0xad, 0x66, 0xbe, 0x51, 0x6b, 0x7b, 0x77, 0xd8, 0x16, + 0x80, 0x8e, 0x2b, 0x67, 0x61, 0xa0, 0x58, 0x07, 0x65, 0xc7, 0xf3, 0x99, 0x1e, 0xb4, 0x16, 0x70, + 0xb4, 0x66, 0x2d, 0xf6, 0xbe, 0x4e, 0xf3, 0x73, 0x58, 0xdb, 0xa3, 0xbb, 0xe9, 0x24, 0xdb, 0x3d, + 0x4f, 0xb2, 0x9c, 0xad, 0xd6, 0xcd, 0x37, 0x56, 0x19, 0xde, 0x1d, 0xf6, 0x04, 0x9c, 0x71, 0x76, + 0xa3, 0xf5, 0xdf, 0x30, 0x57, 0x56, 0x35, 0xdf, 0x2d, 0xbb, 0xdc, 0xf9, 0x75, 0x13, 0xec, 0x9f, + 0x27, 0xd9, 0x95, 0xcc, 0xd8, 0x23, 0xb0, 0xa9, 0xce, 0x61, 0xdc, 0xa8, 0xac, 0x79, 0xdc, 0x36, + 0xd1, 0x07, 0xe0, 0x12, 0x28, 0x63, 0xa1, 0xae, 0xb4, 0xa9, 0xe8, 0xe3, 0xba, 0xc6, 0x45, 0x27, + 0xb3, 0x64, 0xd7, 0x75, 0x6d, 0xa8, 0xb2, 0xec, 0xb3, 0x54, 0x7c, 0xd8, 0xe8, 0xe8, 0x4a, 0xc2, + 0x08, 0x5d, 0xf3, 0x89, 0xc5, 0x3e, 0x86, 0xd6, 0x48, 0xef, 0x14, 0x95, 0xaa, 0x4f, 0x8d, 0x1b, + 0xeb, 0x05, 0xa3, 0x1c, 0xf9, 0x31, 0xd8, 0x3a, 0xd3, 0xd1, 0xdb, 0x5c, 0x4a, 0xdf, 0x37, 0xfa, + 0x75, 0x96, 0xe9, 0xf0, 0x11, 0xd8, 0x3a, 0x4a, 0xea, 0x0e, 0x4b, 0x11, 0x73, 0xa3, 0xb0, 0x83, + 0x77, 0x87, 0x7d, 0x0c, 0xb6, 0x3e, 0xfc, 0x5a, 0x6f, 0x29, 0x10, 0xe8, 0xdd, 0xe9, 0xe8, 0xac, + 0xbd, 0x96, 0x4b, 0x5f, 0x86, 0xb5, 0x04, 0x88, 0x15, 0x3b, 0xba, 0xe5, 0xe8, 0x7d, 0x01, 0x6b, + 0x4b, 0xc9, 0x12, 0x1b, 0x10, 0xca, 0xb7, 0xe4, 0x4f, 0xab, 0x9d, 0x9f, 0xf6, 0xff, 0xf9, 0xdb, + 0xfb, 0xd6, 0xbf, 0x7e, 0x7b, 0xdf, 0xfa, 0xf7, 0x6f, 0xef, 0x5b, 0xbf, 0xfa, 0x8f, 0xfb, 0x77, + 0xce, 0x6d, 0xfa, 0x47, 0xc6, 0xe7, 0xff, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x91, 0xd7, 0x6c, 0x46, + 0xd5, 0x21, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -7671,6 +7683,16 @@ func (m *BackupRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } + if m.ForceFull { + dAtA[i] = 0x58 + i++ + if m.ForceFull { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -9097,6 +9119,9 @@ func (m *BackupRequest) Size() (n int) { if m.Anonymous { n += 2 } + if m.ForceFull { + n += 2 + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -17409,6 +17434,26 @@ func (m *BackupRequest) Unmarshal(dAtA []byte) error { } } m.Anonymous = bool(v != 0) + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ForceFull", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPb + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ForceFull = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPb(dAtA[iNdEx:]) diff --git a/worker/backup_ee.go b/worker/backup_ee.go index 862fec095be..8167f63ba1f 100644 --- a/worker/backup_ee.go +++ b/worker/backup_ee.go @@ -33,25 +33,17 @@ func backupProcess(ctx context.Context, req *pb.BackupRequest) error { glog.Errorf("Context error during backup: %v\n", err) return err } + g := groups() - // sanity, make sure this is our group. if g.groupId() != req.GroupId { return x.Errorf("Backup request group mismatch. Mine: %d. Requested: %d\n", g.groupId(), req.GroupId) } - // wait for this node to catch-up. + if err := posting.Oracle().WaitForTs(ctx, req.ReadTs); err != nil { return err } - // Get snapshot to fill any gaps. - snap, err := g.Node.Snapshot() - if err != nil { - return err - } - glog.V(3).Infof("Backup group %d snapshot: %+v", req.GroupId, snap) - // Attach snapshot readTs to request to compare with any previous version. - req.SnapshotTs = snap.ReadTs - // create backup request and process it. + br := &backup.Request{DB: pstore, Backup: req} return br.Process(ctx) } @@ -103,6 +95,7 @@ func BackupOverNetwork(ctx context.Context, r *http.Request) error { secretKey := r.FormValue("secret_key") sessionToken := r.FormValue("session_token") anonymous := r.FormValue("anonymous") == "true" + forceFull := r.FormValue("force_full") == "true" // Check that this node can accept requests. if err := x.HealthCheck(); err != nil { @@ -125,6 +118,8 @@ func BackupOverNetwork(ctx context.Context, r *http.Request) error { SecretKey: secretKey, SessionToken: sessionToken, Anonymous: anonymous, + // TODO(martinmr): Check if this field can be removed. + ForceFull: forceFull, } m := backup.Manifest{Groups: groups().KnownGroups()} glog.Infof("Created backup request: %s. Groups=%v\n", &req, m.Groups) @@ -151,10 +146,6 @@ func BackupOverNetwork(ctx context.Context, r *http.Request) error { for range m.Groups { if err := <-errCh; err != nil { - // No changes, nothing was done. - if err == backup.ErrBackupNoChanges { - return nil - } glog.Errorf("Error received during backup: %v", err) return err }