log-backup: truncate metadata streamingly (pingcap#38870)

close pingcap#38869, ref pingcap#38939, ref pingcap#38940
Leavrth · Nov 8, 2022 · cae68c5 · cae68c5
1 parent da8ffc4
commit cae68c5
Show file tree

Hide file tree

Showing 5 changed files with 2,212 additions and 267 deletions.
diff --git a/br/pkg/restore/BUILD.bazel b/br/pkg/restore/BUILD.bazel
@@ -141,6 +141,7 @@ go_test(
         "//types",
         "//util/codec",
         "//util/mathutil",
+        "@com_github_fsouza_fake_gcs_server//fakestorage",
         "@com_github_golang_protobuf//proto",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_failpoint//:failpoint",

diff --git a/br/pkg/restore/log_client_test.go b/br/pkg/restore/log_client_test.go
@@ -370,12 +370,19 @@ func TestReadFromMetadata(t *testing.T) {
 
 		meta := new(StreamMetadataSet)
 		meta.Helper = stream.NewMetadataHelper()
-		meta.LoadUntil(ctx, loc, c.untilTS)
+		meta.LoadUntilAndCalculateShiftTS(ctx, loc, c.untilTS)
 
 		var metas []*backuppb.Metadata
-		for _, m := range meta.metadata {
+		for path := range meta.metadataInfos {
+			data, err := loc.ReadFile(ctx, path)
+			require.NoError(t, err)
+
+			m, err := meta.Helper.ParseToMetadataHard(data)
+			require.NoError(t, err)
+
 			metas = append(metas, m)
 		}
+
 		actualStoreIDs := make([]int64, 0, len(metas))
 		for _, meta := range metas {
 			actualStoreIDs = append(actualStoreIDs, meta.StoreId)
@@ -436,10 +443,16 @@ func TestReadFromMetadataV2(t *testing.T) {
 
 		meta := new(StreamMetadataSet)
 		meta.Helper = stream.NewMetadataHelper()
-		meta.LoadUntil(ctx, loc, c.untilTS)
+		meta.LoadUntilAndCalculateShiftTS(ctx, loc, c.untilTS)
 
 		var metas []*backuppb.Metadata
-		for _, m := range meta.metadata {
+		for path := range meta.metadataInfos {
+			data, err := loc.ReadFile(ctx, path)
+			require.NoError(t, err)
+
+			m, err := meta.Helper.ParseToMetadataHard(data)
+			require.NoError(t, err)
+
 			metas = append(metas, m)
 		}
 		actualStoreIDs := make([]int64, 0, len(metas))