Add support for encrypted backups in online restores. (#5226)

graphql/admin/endpoints_ee.go

@@ -68,6 +68,13 @@ const adminTypes = `
 		"""
 		backupId: String!
 
+		"""
+		Path to the key file needed to decrypt the backup. This file should be accessible
+		by all alphas in the group. The backup will be written using the encryption key
+		with which the cluster was started, which might be different than this key.
+		"""
+		keyFile: String!
+
 		"""
 		Access key credential for the destination.
 		"""

graphql/admin/restore.go

@@ -29,6 +29,7 @@ import (
 type restoreInput struct {
 	Location     string
 	BackupId     string
+	KeyFile      string
 	AccessKey    string
 	SecretKey    string
 	SessionToken string
@@ -45,6 +46,7 @@ func resolveRestore(ctx context.Context, m schema.Mutation) (*resolve.Resolved,
 	req := pb.RestoreRequest{
 		Location:     input.Location,
 		BackupId:     input.BackupId,
+		KeyFile:      input.KeyFile,
 		AccessKey:    input.AccessKey,
 		SecretKey:    input.SecretKey,
 		SessionToken: input.SessionToken,

protos/pb.proto

@@ -275,6 +275,9 @@ message RestoreRequest {
 	string secret_key = 6;
 	string session_token = 7;
 	bool anonymous = 8;
+
+	// Info needed to process encrypted backups.
+	string key_file = 9;
 }
 
 message Proposal {

systest/online-restore/backup/dgraph.20200416.215532.278/manifest.json

@@ -0,0 +1 @@
+{"type":"full","since":7,"groups":{"1":["dgraph.graphql.xid","dgraph.graphql.schema","dgraph.type"]},"backup_id":"heuristic_sammet9","backup_num":1,"encrypted":true}

systest/online-restore/backup/dgraph.20200416.220003.130/manifest.json

@@ -0,0 +1 @@
+{"type":"incremental","since":2094,"groups":{"1":["dgraph.graphql.xid","genre","name","performance.film","performance.character","tagline","loc","starring","director.film","performance.character_note","rating","rated","actor.film","email","cut.note","performance.actor","initial_release_date","country","dgraph.type","dgraph.graphql.schema"]},"backup_id":"heuristic_sammet9","backup_num":2,"encrypted":true}

systest/online-restore/docker-compose.yml

@@ -16,12 +16,17 @@ services:
       source: $GOPATH/bin
       target: /gobin
       read_only: true
+    - type: bind
+      source: ./keys
+      target: /data/keys
+      read_only: true
     - type: bind
       source: ./backup
-      target: /data/alpha1/backup
+      target: /data/backup
       read_only: true
     command: /gobin/dgraph alpha -o 100 --my=alpha1:7180 --lru_mb=1024 --zero=zero1:5180
       --logtostderr -v=2 --idx=1 --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
+      --encryption_key_file /data/keys/enc_key
   ratel:
     image: dgraph/dgraph:latest
     container_name: ratel

systest/online-restore/online_restore_test.go

@@ -37,7 +37,8 @@ import (
 
 func sendRestoreRequest(t *testing.T) {
 	restoreRequest := `mutation restore() {
-		 restore(input: {location: "/data/alpha1/backup", backupId: "compassionate_antonelli7"}) {
+		 restore(input: {location: "/data/backup", backupId: "heuristic_sammet9",
+			keyFile: "/data/keys/enc_key"}) {
 			response {
 				code
 				message
@@ -138,7 +139,8 @@ func TestBasicRestore(t *testing.T) {
 
 func TestInvalidBackupId(t *testing.T) {
 	restoreRequest := `mutation restore() {
-		 restore(input: {location: "/data/alpha1/backup", backupId: "bad-backup-id"}) {
+		 restore(input: {location: "/data/backup", backupId: "bad-backup-id",
+			keyFile: "/data/keys/enc_key"}) {
 			response {
 				code
 				message

worker/online_restore_ee.go

@@ -19,6 +19,7 @@ import (
 	"net/url"
 
 	"github.com/dgraph-io/dgraph/conn"
+	"github.com/dgraph-io/dgraph/ee/enc"
 	"github.com/dgraph-io/dgraph/posting"
 	"github.com/dgraph-io/dgraph/protos/pb"
 	"github.com/dgraph-io/dgraph/schema"
@@ -117,7 +118,6 @@ func (w *grpcWorker) Restore(ctx context.Context, req *pb.RestoreRequest) (*pb.S
 // TODO(DGRAPH-1230): Track restore operations.
 // TODO(DGRAPH-1231): Use draining mode during restores.
 // TODO(DGRAPH-1232): Ensure all groups receive the restore proposal.
-// TODO(DGRAPH-1233): Online restores support encrypted backups.
 func handleRestoreProposal(ctx context.Context, req *pb.RestoreRequest) error {
 	if req == nil {
 		return errors.Errorf("nil restore request")
@@ -198,6 +198,10 @@ func handleRestoreProposal(ctx context.Context, req *pb.RestoreRequest) error {
 func writeBackup(ctx context.Context, req *pb.RestoreRequest) error {
 	res := LoadBackup(req.Location, req.BackupId,
 		func(r io.Reader, groupId int, preds predicateSet) (uint64, error) {
+			r, err := enc.GetReader(req.GetKeyFile(), r)
+			if err != nil {
+				return 0, errors.Wrapf(err, "cannot get encrypted reader")
+			}
 			gzReader, err := gzip.NewReader(r)
 			if err != nil {
 				return 0, errors.Wrapf(err, "cannot create gzip reader")