From 3dc892cec00a66e308539d0bec606589eb2fb21a Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Wed, 25 Nov 2020 23:34:01 +0530 Subject: [PATCH 1/8] Add test for old restore to a cluster --- systest/backup/filesystem/backup_test.go | 102 ++++++++++++++++-- .../dgraph.20201125.173944.587/manifest.json | 1 + .../dgraph.20201125.173944.587/r68-g1.backup | Bin 0 -> 1026 bytes .../dgraph.20201125.173944.587/r68-g2.backup | Bin 0 -> 204 bytes .../dgraph.20201125.173944.587/r68-g3.backup | Bin 0 -> 205 bytes 5 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json create mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup create mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup create mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 74f38269e1b..7fd795fd33c 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -24,6 +24,7 @@ import ( "math" "net/http" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -40,10 +41,10 @@ import ( ) var ( - copyBackupDir = "./data/backups_copy" - restoreDir = "./data/restore" - testDirs = []string{restoreDir} - + copyBackupDir = "./data/backups_copy" + restoreDir = "./data/restore" + testDirs = []string{restoreDir, exporterDir} + exporterDir = "data/exporter" alphaBackupDir = "/data/backups" alphaContainers = []string{ @@ -53,6 +54,90 @@ var ( } ) +func sendRestoreRequest(t *testing.T, location string) int { + if location == "" { + location = "/data/backup" + } + params := testutil.GraphQLParams{ + Query: `mutation restore($location: String!) { + restore(input: {location: $location}) { + code + message + restoreId + } + }`, + Variables: map[string]interface{}{ + "location": location, + }, + } + resp := testutil.MakeGQLRequest(t, ¶ms) + resp.RequireNoGraphQLErrors(t) + + var restoreResp struct { + Restore struct { + Code string + Message string + RestoreId int + } + } + + require.NoError(t, json.Unmarshal(resp.Data, &restoreResp)) + require.Equal(t, restoreResp.Restore.Code, "Success") + require.Greater(t, restoreResp.Restore.RestoreId, 0) + return restoreResp.Restore.RestoreId +} + +func TestBackupOfOldRestore(t *testing.T) { + + dirSetup(t) + conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure()) + require.NoError(t, err) + dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) + + ctx := context.Background() + + require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) + time.Sleep(2 * time.Second) + + dirs1 := runBackup(t, 3, 1) + + oldBackupDir := "/data/backups/to_restore" + _ = sendRestoreRequest(t, oldBackupDir) + + time.Sleep(10 * time.Second) + + // TODO: Check the response of the following query + _, err = dg.NewTxn().Query(context.Background(), `authors(func: has(Author.name)) { count(uid) } `) + //require.True(t, strings.Contains(string(resp.Json), "1")) + + dirs2 := runBackupInternal(t, false, 6, 2) + var recentDir string + for _, dir := range dirs2 { + if dir == dirs1[0] { + continue + } + recentDir = dir + } + + exporter := worker.BackupExporter{} + err = exporter.ExportBackup(recentDir, exporterDir, "rdf", nil) + x.Check(err) + backupExportDirs := x.WalkPathFunc(exporterDir, func(path string, isdir bool) bool { + return isdir && strings.HasPrefix(path, exporterDir+"/dgraph.r") + }) + fmt.Printf("Dirs found %v \n", backupExportDirs) + foundAuthor := false + for i := 1; i < 4; i++ { + out, err := exec.CommandContext(context.Background(), + "zcat", filepath.Join(backupExportDirs[0], fmt.Sprintf("g0%d.rdf.gz", i))).Output() + x.Check(err) + if strings.Contains(string(out), "myname") { + foundAuthor = true + } + } + require.True(t, foundAuthor) +} + func TestBackupFilesystem(t *testing.T) { conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure()) require.NoError(t, err) @@ -330,7 +415,7 @@ func runBackupInternal(t *testing.T, forceFull bool, numExpectedFiles, copyToLocalFs(t) files := x.WalkPathFunc(copyBackupDir, func(path string, isdir bool) bool { - return !isdir && strings.HasSuffix(path, ".backup") + return !isdir && strings.HasSuffix(path, ".backup") && strings.HasPrefix(path, "data/backups_copy/dgraph.") }) require.Equal(t, numExpectedFiles, len(files)) @@ -340,7 +425,7 @@ func runBackupInternal(t *testing.T, forceFull bool, numExpectedFiles, require.Equal(t, numExpectedDirs, len(dirs)) manifests := x.WalkPathFunc(copyBackupDir, func(path string, isdir bool) bool { - return !isdir && strings.Contains(path, "manifest.json") + return !isdir && strings.Contains(path, "manifest.json") && strings.HasPrefix(path, "data/backups_copy/dgraph.") }) require.Equal(t, numExpectedDirs, len(manifests)) @@ -353,6 +438,7 @@ func runRestore(t *testing.T, backupLocation, lastDir string, commitTs uint64) m require.NoError(t, os.RemoveAll(restoreDir)) t.Logf("--- Restoring from: %q", backupLocation) + result := worker.RunRestore("./data/restore", backupLocation, lastDir, x.SensitiveByteSlice(nil)) require.NoError(t, result.Err) @@ -400,9 +486,13 @@ func dirSetup(t *testing.T) { } func dirCleanup(t *testing.T) { + if err := os.RemoveAll(restoreDir); err != nil { t.Fatalf("Error removing directory: %s", err.Error()) } + if err := os.RemoveAll(exporterDir); err != nil { + t.Fatalf("Error removing directory: %s", err.Error()) + } if err := os.RemoveAll(copyBackupDir); err != nil { t.Fatalf("Error removing directory: %s", err.Error()) } diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json new file mode 100644 index 00000000000..4c3c669b8e0 --- /dev/null +++ b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json @@ -0,0 +1 @@ +{"type":"full","since":68,"groups":{"1":["dgraph.graphql.p_sha256hash","Author.pwd","dgraph.graphql.xid","dgraph.drop.op","dgraph.graphql.schema_history","dgraph.graphql.schema_created_at","dgraph.type","dgraph.graphql.schema","dgraph.cors","dgraph.graphql.p_query","Author.name","Person.name"],"2":[],"3":[]},"backup_id":"loving_jones5","backup_num":1,"encrypted":false,"drop_operations":null} diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup new file mode 100644 index 0000000000000000000000000000000000000000..5542b2215d5bfbca1f2022d166beb812deae87e2 GIT binary patch literal 1026 zcmV+d1pWITiwFP!00000|HM{LY}-^E?sHP-`8SqkKh(@kVRhON+JLeq&@#$U1ErG~ z`{T6~4ioB3V(QQ~acxg+ii%PsCNyb68c0ZRq!Jup5=b1-cGzv>GQ?r*fIu9$Oqv8L zLP%VOT<6Vyk~DCd50Rh!{l52ozxTc8-0X5Ln1+3n;GXBsT3Xe}R+ME`4C4sEU3L-d z7uY@W%e5VdAdxdf6Lm2`s3gi96d+GIA5A8?=QH7zi|@So$2ULSzVKdF{%HK~?e4DS z-{*e(e)XA4FRgzvgdred#xUs!DJ>X^u4Ri=!xRY)fpF`e+kgIiZu{y0#)Vfm)_)nh zzBzy-(~uC=qc%ZQ=H%;}^?JR2^@F>lAT-RaLyQuHhAMrvB!+R`=k1j1?Qkluq;jLF znes}-sGgiYazZPc^6Q$F&p&nI)%;MaU+Ob@&ID34$xTsd9mgaI#kFCnpqiZ#h z`a2SWBs^$$#N#K!aT(h(aTF6c=Cf?obXDYV6ojv?+_~_>$G?7P;gFC!aCzU&`iOkjwyah*t*TLL zu=eV`Hfyg;;Du(^Fb;#Tb>q9g?rdFrcM|)B)WX*{(l5d!@=m9N@gR@}(bEi@B@^Ks zuqOlrvLFRvjpvx>aTpHb_j9eS*>xf2>NJRyAO?_Y<&wzZAzwf;Za{hiaI>hX z%S!V?9EL3AT9IesL2V9&&(aoweoER+#^KvOdAcj_v~6FKRl}@Sq_;ql>|Hn^&00pe zvXF!gfE|F-b*W&gMMJfQ-zcj~r3q@TgsegaSGj6 zxt<$b3m_JMa86GSo?))Z4K5LPte6geGpih961j0p@n?m#_f4?YCm z;SPVdx~3GZQQ}VVq_58IKPJFC^wqJdgMgI3cRrPjs-CUth!A+<%6lH-ly9{`FbOT5 zdM0(HJ_~)6Z?j9V4_d!)B}_ts@=d3pKEHBSHENiEHg8;l85p2E_BX*yP05dq+bf5| zu-j{oE5>;ol9cbTr7#J*16+x-kfMC&%wq~VVq)ElbOL}z*g=nv0K41k%P&6Q4L+-V z+^KaUi~E7y?e)c&q0a!)Nbs|VQxJU8ez7s$I!d8{`x^Yerk*D- wNF#V47#Gzr%cc!>V)+_$LhKS;9Ow|oh8X$Uz5h}F1^@v6|FaO)lO+lO04&}7lmGw# literal 0 HcmV?d00001 diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup new file mode 100644 index 0000000000000000000000000000000000000000..e204447fdfb6b2b5f078d3753696ca4a5b88ef26 GIT binary patch literal 204 zcmV;-05ks|iwFP!00000|7v1n00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#@La{vSY0RR6PDG6qZ G0RRA$NK4%S literal 0 HcmV?d00001 diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup new file mode 100644 index 0000000000000000000000000000000000000000..0ce8592f8810bcd656418ec359f8ffb9d75d5cf7 GIT binary patch literal 205 zcmV;;05bm{iwFP!00000|7vDr00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#>lILZhJ00960S5uMT Hiva)t=haPD literal 0 HcmV?d00001 From 30dbe32496d199278db53d396bcb185be6bb91ae Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Fri, 27 Nov 2020 17:14:18 +0530 Subject: [PATCH 2/8] Fix author query --- systest/backup/filesystem/backup_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 7fd795fd33c..99f66bcbc7b 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -107,9 +107,12 @@ func TestBackupOfOldRestore(t *testing.T) { time.Sleep(10 * time.Second) // TODO: Check the response of the following query - _, err = dg.NewTxn().Query(context.Background(), `authors(func: has(Author.name)) { count(uid) } `) + resp, err := dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) + if err != nil { + x.Check(err) + } //require.True(t, strings.Contains(string(resp.Json), "1")) - + fmt.Printf("Json : %v\n", resp) dirs2 := runBackupInternal(t, false, 6, 2) var recentDir string for _, dir := range dirs2 { From 994984d82720dad5f71400649c3e8798f76c1733 Mon Sep 17 00:00:00 2001 From: Ibrahim Jarif Date: Mon, 30 Nov 2020 13:57:01 +0530 Subject: [PATCH 3/8] fix(restore): Use correct version after marshalling The marshalPostingList function returns a KV without the version set and this causes issues if you restore a backup on top of an existing data. This PR fixes this issue by explicitly setting the version for the KV returned by the MarshalPostingList function. --- posting/list.go | 2 ++ worker/restore.go | 10 +++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/posting/list.go b/posting/list.go index e03aca21f44..e8d7eb0e83a 100644 --- a/posting/list.go +++ b/posting/list.go @@ -918,6 +918,8 @@ func (out *rollupOutput) marshalPostingListPart(alloc *z.Allocator, return kv, nil } +// MarshalPostingList returns a KV with the marshalled posting list. The caller +// SHOULD SET the Key and Version for the returned KV. func MarshalPostingList(plist *pb.PostingList, alloc *z.Allocator) *bpb.KV { kv := y.NewKV(alloc) if isPlistEmpty(plist) { diff --git a/worker/restore.go b/worker/restore.go index 8526f2790e6..fdc179954fc 100644 --- a/worker/restore.go +++ b/worker/restore.go @@ -183,10 +183,14 @@ func loadFromBackup(db *badger.DB, r io.Reader, restoreTs uint64, preds predicat // part without rolling the key first. This part is here for backwards // compatibility. New backups are not affected because there was a change // to roll up lists into a single one. - kv := posting.MarshalPostingList(pl, nil) + newKv := posting.MarshalPostingList(pl, nil) codec.FreePack(pl.Pack) - kv.Key = restoreKey - if err := loader.Set(kv); err != nil { + newKv.Key = restoreKey + // Use the version of the KV before we marshalled the + // posting list. The MarshalPostingList function returns KV + // with a zero version. + newKv.Version = kv.Version + if err := loader.Set(newKv); err != nil { return 0, err } } else { From 4172438f35c5670cd4470edab12776a1a44f32c1 Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Mon, 30 Nov 2020 19:33:22 +0530 Subject: [PATCH 4/8] Fix test for restoring old backup incremental --- systest/backup/filesystem/backup_test.go | 75 ++++++++---------- .../dgraph.20201125.173944.587/manifest.json | 1 - .../dgraph.20201125.173944.587/r68-g1.backup | Bin 1026 -> 0 bytes .../dgraph.20201125.173944.587/r68-g2.backup | Bin 204 -> 0 bytes .../dgraph.20201125.173944.587/r68-g3.backup | Bin 205 -> 0 bytes 5 files changed, 31 insertions(+), 45 deletions(-) delete mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json delete mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup delete mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup delete mode 100644 systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 99f66bcbc7b..48c017a301c 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -24,7 +24,6 @@ import ( "math" "net/http" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -41,12 +40,12 @@ import ( ) var ( - copyBackupDir = "./data/backups_copy" - restoreDir = "./data/restore" - testDirs = []string{restoreDir, exporterDir} - exporterDir = "data/exporter" - alphaBackupDir = "/data/backups" - + copyBackupDir = "./data/backups_copy" + restoreDir = "./data/restore" + testDirs = []string{restoreDir, exporterDir} + exporterDir = "data/exporter" + alphaBackupDir = "/data/backups" + oldBackupDir = "/data/to_restore" alphaContainers = []string{ "alpha1", "alpha2", @@ -87,9 +86,11 @@ func sendRestoreRequest(t *testing.T, location string) int { return restoreResp.Restore.RestoreId } +// This test takes a backup and then restores an old backup in a cluster incrementally. +// Next, cleans up the cluster and tries restoring the backups above. func TestBackupOfOldRestore(t *testing.T) { - dirSetup(t) + copyOldBackupDir(t) conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure()) require.NoError(t, err) dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) @@ -99,46 +100,26 @@ func TestBackupOfOldRestore(t *testing.T) { require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) time.Sleep(2 * time.Second) - dirs1 := runBackup(t, 3, 1) + _ = runBackup(t, 3, 1) - oldBackupDir := "/data/backups/to_restore" _ = sendRestoreRequest(t, oldBackupDir) + time.Sleep(5 * time.Second) - time.Sleep(10 * time.Second) - - // TODO: Check the response of the following query resp, err := dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) - if err != nil { - x.Check(err) - } - //require.True(t, strings.Contains(string(resp.Json), "1")) - fmt.Printf("Json : %v\n", resp) - dirs2 := runBackupInternal(t, false, 6, 2) - var recentDir string - for _, dir := range dirs2 { - if dir == dirs1[0] { - continue - } - recentDir = dir - } + x.Check(err) + require.Equal(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) + + _ = runBackup(t, 6, 2) + + // Clean the cluster and try restoring the backups created above. + require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) + time.Sleep(2 * time.Second) + _ = sendRestoreRequest(t, alphaBackupDir) + time.Sleep(5 * time.Second) - exporter := worker.BackupExporter{} - err = exporter.ExportBackup(recentDir, exporterDir, "rdf", nil) + resp, err = dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) x.Check(err) - backupExportDirs := x.WalkPathFunc(exporterDir, func(path string, isdir bool) bool { - return isdir && strings.HasPrefix(path, exporterDir+"/dgraph.r") - }) - fmt.Printf("Dirs found %v \n", backupExportDirs) - foundAuthor := false - for i := 1; i < 4; i++ { - out, err := exec.CommandContext(context.Background(), - "zcat", filepath.Join(backupExportDirs[0], fmt.Sprintf("g0%d.rdf.gz", i))).Output() - x.Check(err) - if strings.Contains(string(out), "myname") { - foundAuthor = true - } - } - require.True(t, foundAuthor) + require.Equal(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) } func TestBackupFilesystem(t *testing.T) { @@ -441,7 +422,6 @@ func runRestore(t *testing.T, backupLocation, lastDir string, commitTs uint64) m require.NoError(t, os.RemoveAll(restoreDir)) t.Logf("--- Restoring from: %q", backupLocation) - result := worker.RunRestore("./data/restore", backupLocation, lastDir, x.SensitiveByteSlice(nil)) require.NoError(t, result.Err) @@ -489,7 +469,6 @@ func dirSetup(t *testing.T) { } func dirCleanup(t *testing.T) { - if err := os.RemoveAll(restoreDir); err != nil { t.Fatalf("Error removing directory: %s", err.Error()) } @@ -506,6 +485,14 @@ func dirCleanup(t *testing.T) { } } +func copyOldBackupDir(t *testing.T) { + destPath := testutil.DockerPrefix + "_alpha1_1:/data" + srchPath := "." + oldBackupDir + if err := testutil.DockerCp(srchPath, destPath); err != nil { + t.Fatalf("Error copying files from docker container: %s", err.Error()) + } +} + func copyToLocalFs(t *testing.T) { // The original backup files are not accessible because docker creates all files in // the shared volume as the root user. This restriction is circumvented by using diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json deleted file mode 100644 index 4c3c669b8e0..00000000000 --- a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/manifest.json +++ /dev/null @@ -1 +0,0 @@ -{"type":"full","since":68,"groups":{"1":["dgraph.graphql.p_sha256hash","Author.pwd","dgraph.graphql.xid","dgraph.drop.op","dgraph.graphql.schema_history","dgraph.graphql.schema_created_at","dgraph.type","dgraph.graphql.schema","dgraph.cors","dgraph.graphql.p_query","Author.name","Person.name"],"2":[],"3":[]},"backup_id":"loving_jones5","backup_num":1,"encrypted":false,"drop_operations":null} diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g1.backup deleted file mode 100644 index 5542b2215d5bfbca1f2022d166beb812deae87e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1026 zcmV+d1pWITiwFP!00000|HM{LY}-^E?sHP-`8SqkKh(@kVRhON+JLeq&@#$U1ErG~ z`{T6~4ioB3V(QQ~acxg+ii%PsCNyb68c0ZRq!Jup5=b1-cGzv>GQ?r*fIu9$Oqv8L zLP%VOT<6Vyk~DCd50Rh!{l52ozxTc8-0X5Ln1+3n;GXBsT3Xe}R+ME`4C4sEU3L-d z7uY@W%e5VdAdxdf6Lm2`s3gi96d+GIA5A8?=QH7zi|@So$2ULSzVKdF{%HK~?e4DS z-{*e(e)XA4FRgzvgdred#xUs!DJ>X^u4Ri=!xRY)fpF`e+kgIiZu{y0#)Vfm)_)nh zzBzy-(~uC=qc%ZQ=H%;}^?JR2^@F>lAT-RaLyQuHhAMrvB!+R`=k1j1?Qkluq;jLF znes}-sGgiYazZPc^6Q$F&p&nI)%;MaU+Ob@&ID34$xTsd9mgaI#kFCnpqiZ#h z`a2SWBs^$$#N#K!aT(h(aTF6c=Cf?obXDYV6ojv?+_~_>$G?7P;gFC!aCzU&`iOkjwyah*t*TLL zu=eV`Hfyg;;Du(^Fb;#Tb>q9g?rdFrcM|)B)WX*{(l5d!@=m9N@gR@}(bEi@B@^Ks zuqOlrvLFRvjpvx>aTpHb_j9eS*>xf2>NJRyAO?_Y<&wzZAzwf;Za{hiaI>hX z%S!V?9EL3AT9IesL2V9&&(aoweoER+#^KvOdAcj_v~6FKRl}@Sq_;ql>|Hn^&00pe zvXF!gfE|F-b*W&gMMJfQ-zcj~r3q@TgsegaSGj6 zxt<$b3m_JMa86GSo?))Z4K5LPte6geGpih961j0p@n?m#_f4?YCm z;SPVdx~3GZQQ}VVq_58IKPJFC^wqJdgMgI3cRrPjs-CUth!A+<%6lH-ly9{`FbOT5 zdM0(HJ_~)6Z?j9V4_d!)B}_ts@=d3pKEHBSHENiEHg8;l85p2E_BX*yP05dq+bf5| zu-j{oE5>;ol9cbTr7#J*16+x-kfMC&%wq~VVq)ElbOL}z*g=nv0K41k%P&6Q4L+-V z+^KaUi~E7y?e)c&q0a!)Nbs|VQxJU8ez7s$I!d8{`x^Yerk*D- wNF#V47#Gzr%cc!>V)+_$LhKS;9Ow|oh8X$Uz5h}F1^@v6|FaO)lO+lO04&}7lmGw# diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g2.backup deleted file mode 100644 index e204447fdfb6b2b5f078d3753696ca4a5b88ef26..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 204 zcmV;-05ks|iwFP!00000|7v1n00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#@La{vSY0RR6PDG6qZ G0RRA$NK4%S diff --git a/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup b/systest/backup/filesystem/data/backups/to_restore/dgraph.20201125.173944.587/r68-g3.backup deleted file mode 100644 index 0ce8592f8810bcd656418ec359f8ffb9d75d5cf7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 205 zcmV;;05bm{iwFP!00000|7vDr00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#>lILZhJ00960S5uMT Hiva)t=haPD From 387fb20c45fbc38d7a14bb8bb2cd0160370ecc5f Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Mon, 30 Nov 2020 20:53:12 +0530 Subject: [PATCH 5/8] Address review comments --- systest/backup/filesystem/backup_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 48c017a301c..4ed209ffe58 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -91,10 +91,9 @@ func sendRestoreRequest(t *testing.T, location string) int { func TestBackupOfOldRestore(t *testing.T) { dirSetup(t) copyOldBackupDir(t) - conn, err := grpc.Dial(testutil.SockAddr, grpc.WithInsecure()) - require.NoError(t, err) - dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) + dg, err := testutil.DgraphClient(testutil.SockAddr) + x.Check(err) ctx := context.Background() require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) @@ -103,11 +102,11 @@ func TestBackupOfOldRestore(t *testing.T) { _ = runBackup(t, 3, 1) _ = sendRestoreRequest(t, oldBackupDir) - time.Sleep(5 * time.Second) + time.Sleep(3 * time.Second) resp, err := dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) x.Check(err) - require.Equal(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) + require.JSONEq(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) _ = runBackup(t, 6, 2) @@ -115,11 +114,11 @@ func TestBackupOfOldRestore(t *testing.T) { require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) time.Sleep(2 * time.Second) _ = sendRestoreRequest(t, alphaBackupDir) - time.Sleep(5 * time.Second) + time.Sleep(3 * time.Second) resp, err = dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) x.Check(err) - require.Equal(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) + require.JSONEq(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) } func TestBackupFilesystem(t *testing.T) { From 44a14823559751ce9cf9095feabc5ef2c58d6620 Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Mon, 30 Nov 2020 21:04:45 +0530 Subject: [PATCH 6/8] Add old backup dir --- .../dgraph.20201125.173944.587/manifest.json | 1 + .../dgraph.20201125.173944.587/r68-g1.backup | Bin 0 -> 1026 bytes .../dgraph.20201125.173944.587/r68-g2.backup | Bin 0 -> 204 bytes .../dgraph.20201125.173944.587/r68-g3.backup | Bin 0 -> 205 bytes 4 files changed, 1 insertion(+) create mode 100644 systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/manifest.json create mode 100644 systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g1.backup create mode 100644 systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g2.backup create mode 100644 systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g3.backup diff --git a/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/manifest.json b/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/manifest.json new file mode 100644 index 00000000000..4c3c669b8e0 --- /dev/null +++ b/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/manifest.json @@ -0,0 +1 @@ +{"type":"full","since":68,"groups":{"1":["dgraph.graphql.p_sha256hash","Author.pwd","dgraph.graphql.xid","dgraph.drop.op","dgraph.graphql.schema_history","dgraph.graphql.schema_created_at","dgraph.type","dgraph.graphql.schema","dgraph.cors","dgraph.graphql.p_query","Author.name","Person.name"],"2":[],"3":[]},"backup_id":"loving_jones5","backup_num":1,"encrypted":false,"drop_operations":null} diff --git a/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g1.backup b/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g1.backup new file mode 100644 index 0000000000000000000000000000000000000000..5542b2215d5bfbca1f2022d166beb812deae87e2 GIT binary patch literal 1026 zcmV+d1pWITiwFP!00000|HM{LY}-^E?sHP-`8SqkKh(@kVRhON+JLeq&@#$U1ErG~ z`{T6~4ioB3V(QQ~acxg+ii%PsCNyb68c0ZRq!Jup5=b1-cGzv>GQ?r*fIu9$Oqv8L zLP%VOT<6Vyk~DCd50Rh!{l52ozxTc8-0X5Ln1+3n;GXBsT3Xe}R+ME`4C4sEU3L-d z7uY@W%e5VdAdxdf6Lm2`s3gi96d+GIA5A8?=QH7zi|@So$2ULSzVKdF{%HK~?e4DS z-{*e(e)XA4FRgzvgdred#xUs!DJ>X^u4Ri=!xRY)fpF`e+kgIiZu{y0#)Vfm)_)nh zzBzy-(~uC=qc%ZQ=H%;}^?JR2^@F>lAT-RaLyQuHhAMrvB!+R`=k1j1?Qkluq;jLF znes}-sGgiYazZPc^6Q$F&p&nI)%;MaU+Ob@&ID34$xTsd9mgaI#kFCnpqiZ#h z`a2SWBs^$$#N#K!aT(h(aTF6c=Cf?obXDYV6ojv?+_~_>$G?7P;gFC!aCzU&`iOkjwyah*t*TLL zu=eV`Hfyg;;Du(^Fb;#Tb>q9g?rdFrcM|)B)WX*{(l5d!@=m9N@gR@}(bEi@B@^Ks zuqOlrvLFRvjpvx>aTpHb_j9eS*>xf2>NJRyAO?_Y<&wzZAzwf;Za{hiaI>hX z%S!V?9EL3AT9IesL2V9&&(aoweoER+#^KvOdAcj_v~6FKRl}@Sq_;ql>|Hn^&00pe zvXF!gfE|F-b*W&gMMJfQ-zcj~r3q@TgsegaSGj6 zxt<$b3m_JMa86GSo?))Z4K5LPte6geGpih961j0p@n?m#_f4?YCm z;SPVdx~3GZQQ}VVq_58IKPJFC^wqJdgMgI3cRrPjs-CUth!A+<%6lH-ly9{`FbOT5 zdM0(HJ_~)6Z?j9V4_d!)B}_ts@=d3pKEHBSHENiEHg8;l85p2E_BX*yP05dq+bf5| zu-j{oE5>;ol9cbTr7#J*16+x-kfMC&%wq~VVq)ElbOL}z*g=nv0K41k%P&6Q4L+-V z+^KaUi~E7y?e)c&q0a!)Nbs|VQxJU8ez7s$I!d8{`x^Yerk*D- wNF#V47#Gzr%cc!>V)+_$LhKS;9Ow|oh8X$Uz5h}F1^@v6|FaO)lO+lO04&}7lmGw# literal 0 HcmV?d00001 diff --git a/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g2.backup b/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g2.backup new file mode 100644 index 0000000000000000000000000000000000000000..e204447fdfb6b2b5f078d3753696ca4a5b88ef26 GIT binary patch literal 204 zcmV;-05ks|iwFP!00000|7v1n00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#@La{vSY0RR6PDG6qZ G0RRA$NK4%S literal 0 HcmV?d00001 diff --git a/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g3.backup b/systest/backup/filesystem/data/to_restore/dgraph.20201125.173944.587/r68-g3.backup new file mode 100644 index 0000000000000000000000000000000000000000..0ce8592f8810bcd656418ec359f8ffb9d75d5cf7 GIT binary patch literal 205 zcmV;;05bm{iwFP!00000|7vDr00XWBE+GzfA-Juq3ABjn75E+WY#iYlU4 zoSc!Gn<&63B*rC#CRLG{BETud$f&>=z{u6c$R)+WE+mGgCL^=BB)_OqD3J@7j3SpT zRx9J73IsTXRJatdD@iU&O)N=GiBBvMV1+n@iK~l|ONoPBNDfU$L26NPW-&-{VQFen zrBEyvAr(>>PA-7R2yhC?b4g>!6=x(GnVMxJ7H7ahg(;w=0R#>lILZhJ00960S5uMT Hiva)t=haPD literal 0 HcmV?d00001 From 314ac551d6a7494d308e96e69d38e2a59fee5a52 Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Tue, 1 Dec 2020 18:09:21 +0530 Subject: [PATCH 7/8] Add wait for restore, address review comments --- systest/backup/filesystem/backup_test.go | 39 +++++++------- testutil/backup.go | 65 ++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 20 deletions(-) diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 4ed209ffe58..4d564589869 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -42,8 +42,7 @@ import ( var ( copyBackupDir = "./data/backups_copy" restoreDir = "./data/restore" - testDirs = []string{restoreDir, exporterDir} - exporterDir = "data/exporter" + testDirs = []string{restoreDir} alphaBackupDir = "/data/backups" oldBackupDir = "/data/to_restore" alphaContainers = []string{ @@ -88,36 +87,36 @@ func sendRestoreRequest(t *testing.T, location string) int { // This test takes a backup and then restores an old backup in a cluster incrementally. // Next, cleans up the cluster and tries restoring the backups above. +// Regression test for DGRAPH-2775 func TestBackupOfOldRestore(t *testing.T) { dirSetup(t) copyOldBackupDir(t) dg, err := testutil.DgraphClient(testutil.SockAddr) - x.Check(err) - ctx := context.Background() + require.NoError(t, err) - require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) + testutil.DropAll(t, dg) time.Sleep(2 * time.Second) _ = runBackup(t, 3, 1) - _ = sendRestoreRequest(t, oldBackupDir) - time.Sleep(3 * time.Second) + restoreId := sendRestoreRequest(t, oldBackupDir) + testutil.WaitForRestore(t, restoreId, dg) resp, err := dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) - x.Check(err) + require.NoError(t, err) require.JSONEq(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) _ = runBackup(t, 6, 2) // Clean the cluster and try restoring the backups created above. - require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) + testutil.DropAll(t, dg) time.Sleep(2 * time.Second) - _ = sendRestoreRequest(t, alphaBackupDir) - time.Sleep(3 * time.Second) + restoreId = sendRestoreRequest(t, alphaBackupDir) + testutil.WaitForRestore(t, restoreId, dg) resp, err = dg.NewTxn().Query(context.Background(), `{ authors(func: has(Author.name)) { count(uid) } }`) - x.Check(err) + require.NoError(t, err) require.JSONEq(t, "{\"authors\":[{\"count\":1}]}", string(resp.Json)) } @@ -177,8 +176,8 @@ func TestBackupFilesystem(t *testing.T) { break } } - require.True(t, moveOk) + require.True(t, moveOk) // Setup test directories. dirSetup(t) @@ -471,9 +470,7 @@ func dirCleanup(t *testing.T) { if err := os.RemoveAll(restoreDir); err != nil { t.Fatalf("Error removing directory: %s", err.Error()) } - if err := os.RemoveAll(exporterDir); err != nil { - t.Fatalf("Error removing directory: %s", err.Error()) - } + if err := os.RemoveAll(copyBackupDir); err != nil { t.Fatalf("Error removing directory: %s", err.Error()) } @@ -485,10 +482,12 @@ func dirCleanup(t *testing.T) { } func copyOldBackupDir(t *testing.T) { - destPath := testutil.DockerPrefix + "_alpha1_1:/data" - srchPath := "." + oldBackupDir - if err := testutil.DockerCp(srchPath, destPath); err != nil { - t.Fatalf("Error copying files from docker container: %s", err.Error()) + for i := 1; i < 4; i++ { + destPath := fmt.Sprintf("%s_alpha%d_1:/data", testutil.DockerPrefix, i) + srchPath := "." + oldBackupDir + if err := testutil.DockerCp(srchPath, destPath); err != nil { + t.Fatalf("Error copying files from docker container: %s", err.Error()) + } } } diff --git a/testutil/backup.go b/testutil/backup.go index ea35c195548..90695cc09e5 100644 --- a/testutil/backup.go +++ b/testutil/backup.go @@ -17,10 +17,18 @@ package testutil import ( + "bytes" + "context" + "encoding/json" "fmt" + "io/ioutil" + "net/http" + "strings" "testing" + "time" "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/dgo/v200" "github.com/dgraph-io/dgraph/ee/enc" "github.com/dgraph-io/dgraph/posting" "github.com/dgraph-io/dgraph/protos/pb" @@ -56,6 +64,63 @@ func openDgraph(pdir string) (*badger.DB, error) { return badger.OpenManaged(opt) } +func WaitForRestore(t *testing.T, restoreId int, dg *dgo.Dgraph) { + query := fmt.Sprintf(`query status() { + restoreStatus(restoreId: %d) { + status + errors + } + }`, restoreId) + params := GraphQLParams{ + Query: query, + } + b, err := json.Marshal(params) + require.NoError(t, err) + + restoreDone := false + for i := 0; i < 15; i++ { + resp, err := http.Post(AdminUrl(), "application/json", bytes.NewBuffer(b)) + require.NoError(t, err) + buf, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + sbuf := string(buf) + println(sbuf) + if strings.Contains(sbuf, "OK") { + restoreDone = true + break + } + time.Sleep(4 * time.Second) + } + require.True(t, restoreDone) + + // Wait for the client to exit draining mode. This is needed because the client might + // be connected to a follower and might be behind the leader in applying the restore. + // Waiting for three consecutive successful queries is done to prevent a situation in + // which the query succeeds at the first attempt because the follower is behind and + // has not started to apply the restore proposal. + numSuccess := 0 + for { + // This is a dummy query that returns no results. + _, err = dg.NewTxn().Query(context.Background(), `{ + q(func: has(invalid_pred)) { + invalid_pred + }}`) + + if err == nil { + numSuccess += 1 + } else { + require.Contains(t, err.Error(), "the server is in draining mode") + numSuccess = 0 + } + + if numSuccess == 3 { + // The server has been responsive three times in a row. + break + } + time.Sleep(1 * time.Second) + } +} + // GetPredicateValues reads the specified p directory and returns the values for the given // attribute in a map. func GetPredicateValues(pdir, attr string, readTs uint64) (map[string]string, error) { From 6232050454581efee9b360fe617e57c900c04456 Mon Sep 17 00:00:00 2001 From: rahulgurnani Date: Wed, 2 Dec 2020 09:48:18 +0530 Subject: [PATCH 8/8] Make test use tls, refactor out waitforrestore --- systest/backup/filesystem/backup_test.go | 6 +- .../backup/filesystem/data/backups/.gitkeep | 0 systest/online-restore/online_restore_test.go | 67 ++----------------- testutil/backup.go | 21 +++--- 4 files changed, 19 insertions(+), 75 deletions(-) delete mode 100644 systest/backup/filesystem/data/backups/.gitkeep diff --git a/systest/backup/filesystem/backup_test.go b/systest/backup/filesystem/backup_test.go index 2063977e3d1..ac306ba888d 100644 --- a/systest/backup/filesystem/backup_test.go +++ b/systest/backup/filesystem/backup_test.go @@ -70,7 +70,7 @@ func sendRestoreRequest(t *testing.T, location string) int { "location": location, }, } - resp := testutil.MakeGQLRequest(t, ¶ms) + resp := testutil.MakeGQLRequestWithTLS(t, ¶ms, testutil.GetAlphaClientConfig(t)) resp.RequireNoGraphQLErrors(t) var restoreResp struct { @@ -94,7 +94,9 @@ func TestBackupOfOldRestore(t *testing.T) { dirSetup(t) copyOldBackupDir(t) - dg, err := testutil.DgraphClient(testutil.SockAddr) + conn, err := grpc.Dial(testutil.SockAddr, grpc.WithTransportCredentials(credentials.NewTLS(testutil.GetAlphaClientConfig(t)))) + require.NoError(t, err) + dg := dgo.NewDgraphClient(api.NewDgraphClient(conn)) require.NoError(t, err) testutil.DropAll(t, dg) diff --git a/systest/backup/filesystem/data/backups/.gitkeep b/systest/backup/filesystem/data/backups/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/systest/online-restore/online_restore_test.go b/systest/online-restore/online_restore_test.go index 4380f9542eb..7625e98c855 100644 --- a/systest/online-restore/online_restore_test.go +++ b/systest/online-restore/online_restore_test.go @@ -74,63 +74,6 @@ func sendRestoreRequest(t *testing.T, location, backupId string, backupNum int) return restoreResp.Restore.RestoreId } -func waitForRestore(t *testing.T, restoreId int, dg *dgo.Dgraph) { - query := fmt.Sprintf(`query status() { - restoreStatus(restoreId: %d) { - status - errors - } - }`, restoreId) - params := testutil.GraphQLParams{ - Query: query, - } - b, err := json.Marshal(params) - require.NoError(t, err) - - restoreDone := false - client := testutil.GetHttpsClient(t) - for i := 0; i < 15; i++ { - resp, err := client.Post(testutil.AdminUrlHttps(), "application/json", bytes.NewBuffer(b)) - require.NoError(t, err) - buf, err := ioutil.ReadAll(resp.Body) - require.NoError(t, err) - sbuf := string(buf) - if strings.Contains(sbuf, "OK") { - restoreDone = true - break - } - time.Sleep(4 * time.Second) - } - require.True(t, restoreDone) - - // Wait for the client to exit draining mode. This is needed because the client might - // be connected to a follower and might be behind the leader in applying the restore. - // Waiting for three consecutive successful queries is done to prevent a situation in - // which the query succeeds at the first attempt because the follower is behind and - // has not started to apply the restore proposal. - numSuccess := 0 - for { - // This is a dummy query that returns no results. - _, err = dg.NewTxn().Query(context.Background(), `{ - q(func: has(invalid_pred)) { - invalid_pred - }}`) - - if err == nil { - numSuccess += 1 - } else { - require.Contains(t, err.Error(), "the server is in draining mode") - numSuccess = 0 - } - - if numSuccess == 3 { - // The server has been responsive three times in a row. - break - } - time.Sleep(1 * time.Second) - } -} - // disableDraining disables draining mode before each test for increased reliability. func disableDraining(t *testing.T) { drainRequest := `mutation draining { @@ -237,7 +180,7 @@ func TestBasicRestore(t *testing.T) { require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) restoreId := sendRestoreRequest(t, "", "youthful_rhodes3", 0) - waitForRestore(t, restoreId, dg) + testutil.WaitForRestore(t, restoreId, dg) runQueries(t, dg, false) runMutations(t, dg) } @@ -254,7 +197,7 @@ func TestRestoreBackupNum(t *testing.T) { runQueries(t, dg, true) restoreId := sendRestoreRequest(t, "", "youthful_rhodes3", 1) - waitForRestore(t, restoreId, dg) + testutil.WaitForRestore(t, restoreId, dg) runQueries(t, dg, true) runMutations(t, dg) } @@ -329,13 +272,13 @@ func TestMoveTablets(t *testing.T) { require.NoError(t, dg.Alter(ctx, &api.Operation{DropAll: true})) restoreId := sendRestoreRequest(t, "", "youthful_rhodes3", 0) - waitForRestore(t, restoreId, dg) + testutil.WaitForRestore(t, restoreId, dg) runQueries(t, dg, false) // Send another restore request with a different backup. This backup has some of the // same predicates as the previous one but they are stored in different groups. restoreId = sendRestoreRequest(t, "", "blissful_hermann1", 0) - waitForRestore(t, restoreId, dg) + testutil.WaitForRestore(t, restoreId, dg) resp, err := dg.NewTxn().Query(context.Background(), `{ q(func: has(name), orderasc: name) { @@ -628,7 +571,7 @@ func backupRestoreAndVerify(t *testing.T, dg *dgo.Dgraph, backupDir, queryToVeri expectedResponse string, schemaVerificationOpts testutil.SchemaOptions) { schemaVerificationOpts.ExcludeAclSchema = true backup(t, backupDir) - waitForRestore(t, sendRestoreRequest(t, backupDir, "", 0), dg) + testutil.WaitForRestore(t, sendRestoreRequest(t, backupDir, "", 0), dg) testutil.VerifyQueryResponse(t, dg, queryToVerify, expectedResponse) testutil.VerifySchema(t, dg, schemaVerificationOpts) } diff --git a/testutil/backup.go b/testutil/backup.go index 90695cc09e5..7f8a18f9c71 100644 --- a/testutil/backup.go +++ b/testutil/backup.go @@ -22,7 +22,6 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net/http" "strings" "testing" "time" @@ -66,11 +65,11 @@ func openDgraph(pdir string) (*badger.DB, error) { func WaitForRestore(t *testing.T, restoreId int, dg *dgo.Dgraph) { query := fmt.Sprintf(`query status() { - restoreStatus(restoreId: %d) { - status - errors - } - }`, restoreId) + restoreStatus(restoreId: %d) { + status + errors + } + }`, restoreId) params := GraphQLParams{ Query: query, } @@ -78,13 +77,13 @@ func WaitForRestore(t *testing.T, restoreId int, dg *dgo.Dgraph) { require.NoError(t, err) restoreDone := false + client := GetHttpsClient(t) for i := 0; i < 15; i++ { - resp, err := http.Post(AdminUrl(), "application/json", bytes.NewBuffer(b)) + resp, err := client.Post(AdminUrlHttps(), "application/json", bytes.NewBuffer(b)) require.NoError(t, err) buf, err := ioutil.ReadAll(resp.Body) require.NoError(t, err) sbuf := string(buf) - println(sbuf) if strings.Contains(sbuf, "OK") { restoreDone = true break @@ -102,9 +101,9 @@ func WaitForRestore(t *testing.T, restoreId int, dg *dgo.Dgraph) { for { // This is a dummy query that returns no results. _, err = dg.NewTxn().Query(context.Background(), `{ - q(func: has(invalid_pred)) { - invalid_pred - }}`) + q(func: has(invalid_pred)) { + invalid_pred + }}`) if err == nil { numSuccess += 1