Merge branch 'master' into typectx_clip_zero

pingcap · Oct 12, 2023 · 7f0d336 · 7f0d336
2 parents ca41ccd + 0fd232f
commit 7f0d336
Show file tree

Hide file tree

Showing 134 changed files with 7,411 additions and 5,967 deletions.
diff --git a/DEPS.bzl b/DEPS.bzl
@@ -7054,13 +7054,13 @@ def go_deps():
         name = "com_github_tikv_client_go_v2",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/tikv/client-go/v2",
-        sha256 = "2a345685fc8d9c23349ebc5c7f6aba495a286df5ab57a35575597edfccdba7eb",
-        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20230925032502-44b0cf7aba2b",
+        sha256 = "43c1cc06bfb94431d4e78c6d961080e9a4e8ceaba22bee60617b7c009640e601",
+        strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20231009020855-652de4de2da7",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230925032502-44b0cf7aba2b.zip",
-            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230925032502-44b0cf7aba2b.zip",
-            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230925032502-44b0cf7aba2b.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20230925032502-44b0cf7aba2b.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231009020855-652de4de2da7.zip",
+            "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231009020855-652de4de2da7.zip",
+            "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231009020855-652de4de2da7.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20231009020855-652de4de2da7.zip",
         ],
     )
     go_repository(
@@ -10152,13 +10152,13 @@ def go_deps():
         name = "org_golang_x_crypto",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/crypto",
-        sha256 = "b58d902f48a7f595a28589b6ed4be8b5e2ee1a3496eca477039e80eb6e7eba57",
-        strip_prefix = "golang.org/x/crypto@v0.13.0",
+        sha256 = "a72c57e71d0fbd460d8493c958fb2841357e3babbb32d7fe2cd533564b1d66b7",
+        strip_prefix = "golang.org/x/crypto@v0.14.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.13.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.13.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.13.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.13.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.14.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.14.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.14.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/crypto/org_golang_x_crypto-v0.14.0.zip",
         ],
     )
     go_repository(
@@ -10243,13 +10243,13 @@ def go_deps():
         name = "org_golang_x_net",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/net",
-        sha256 = "e891941f0a83dfc85f82990e29cbf1939dca5952d04241666c8a227d419fded3",
-        strip_prefix = "golang.org/x/net@v0.15.0",
+        sha256 = "d99428e95af6fd8fdc23285d69b8f6e7c52c964c990295e45183718e33ebeb5a",
+        strip_prefix = "golang.org/x/net@v0.17.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.15.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.15.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.15.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.15.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/net/org_golang_x_net-v0.17.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/net/org_golang_x_net-v0.17.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/net/org_golang_x_net-v0.17.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/net/org_golang_x_net-v0.17.0.zip",
         ],
     )
     go_repository(
@@ -10282,26 +10282,26 @@ def go_deps():
         name = "org_golang_x_sys",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/sys",
-        sha256 = "89225d9e6603c090ffd93286b7ca124849fadfe4320c3b18a6bdccc4ac08672c",
-        strip_prefix = "golang.org/x/sys@v0.12.0",
+        sha256 = "3d149afc9939980354374c0d47461dcdbd1d3980eb2f77e60304b23fee6f3d37",
+        strip_prefix = "golang.org/x/sys@v0.13.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.12.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.12.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.12.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.12.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/sys/org_golang_x_sys-v0.13.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/sys/org_golang_x_sys-v0.13.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/sys/org_golang_x_sys-v0.13.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/sys/org_golang_x_sys-v0.13.0.zip",
         ],
     )
     go_repository(
         name = "org_golang_x_term",
         build_file_proto_mode = "disable_global",
         importpath = "golang.org/x/term",
-        sha256 = "f4bbc4baa0c9b053f7d252b06e4e8baabd686a9a87d82025b341796e29f39c60",
-        strip_prefix = "golang.org/x/term@v0.12.0",
+        sha256 = "a2140133ab5edea486ec9239da01995e606e6261d66d1a673cd2add33d080fdd",
+        strip_prefix = "golang.org/x/term@v0.13.0",
         urls = [
-            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.12.0.zip",
-            "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.12.0.zip",
-            "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.12.0.zip",
-            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.12.0.zip",
+            "http://bazel-cache.pingcap.net:8080/gomod/golang.org/x/term/org_golang_x_term-v0.13.0.zip",
+            "http://ats.apps.svc/gomod/golang.org/x/term/org_golang_x_term-v0.13.0.zip",
+            "https://cache.hawkingrei.com/gomod/golang.org/x/term/org_golang_x_term-v0.13.0.zip",
+            "https://storage.googleapis.com/pingcapmirror/gomod/golang.org/x/term/org_golang_x_term-v0.13.0.zip",
         ],
     )
     go_repository(

diff --git a/br/cmd/br/debug.go b/br/cmd/br/debug.go
@@ -85,61 +85,52 @@ func newCheckSumCommand() *cobra.Command {
 				return errors.Trace(err)
 			}
 
-			for _, schema := range backupMeta.Schemas {
-				dbInfo := &model.DBInfo{}
-				err = json.Unmarshal(schema.Db, dbInfo)
-				if err != nil {
-					return errors.Trace(err)
-				}
-				if schema.Table == nil {
-					continue
-				}
-				tblInfo := &model.TableInfo{}
-				err = json.Unmarshal(schema.Table, tblInfo)
-				if err != nil {
-					return errors.Trace(err)
-				}
-				tbl := dbs[dbInfo.Name.String()].GetTable(tblInfo.Name.String())
-
-				var calCRC64 uint64
-				var totalKVs uint64
-				var totalBytes uint64
-				for _, file := range tbl.Files {
-					calCRC64 ^= file.Crc64Xor
-					totalKVs += file.GetTotalKvs()
-					totalBytes += file.GetTotalBytes()
-					log.Info("file info", zap.Stringer("table", tblInfo.Name),
-						zap.String("file", file.GetName()),
-						zap.Uint64("crc64xor", file.GetCrc64Xor()),
-						zap.Uint64("totalKvs", file.GetTotalKvs()),
-						zap.Uint64("totalBytes", file.GetTotalBytes()),
-						zap.Uint64("startVersion", file.GetStartVersion()),
-						zap.Uint64("endVersion", file.GetEndVersion()),
-						logutil.Key("startKey", file.GetStartKey()),
-						logutil.Key("endKey", file.GetEndKey()),
-					)
-
-					var data []byte
-					data, err = s.ReadFile(ctx, file.Name)
-					if err != nil {
-						return errors.Trace(err)
-					}
-					s := sha256.Sum256(data)
-					if !bytes.Equal(s[:], file.Sha256) {
-						return errors.Annotatef(berrors.ErrBackupChecksumMismatch, `
+			for _, db := range dbs {
+				for _, tbl := range db.Tables {
+					var calCRC64 uint64
+					var totalKVs uint64
+					var totalBytes uint64
+					for _, file := range tbl.Files {
+						calCRC64 ^= file.Crc64Xor
+						totalKVs += file.GetTotalKvs()
+						totalBytes += file.GetTotalBytes()
+						log.Info("file info", zap.Stringer("table", tbl.Info.Name),
+							zap.String("file", file.GetName()),
+							zap.Uint64("crc64xor", file.GetCrc64Xor()),
+							zap.Uint64("totalKvs", file.GetTotalKvs()),
+							zap.Uint64("totalBytes", file.GetTotalBytes()),
+							zap.Uint64("startVersion", file.GetStartVersion()),
+							zap.Uint64("endVersion", file.GetEndVersion()),
+							logutil.Key("startKey", file.GetStartKey()),
+							logutil.Key("endKey", file.GetEndKey()),
+						)
+
+						var data []byte
+						data, err = s.ReadFile(ctx, file.Name)
+						if err != nil {
+							return errors.Trace(err)
+						}
+						s := sha256.Sum256(data)
+						if !bytes.Equal(s[:], file.Sha256) {
+							return errors.Annotatef(berrors.ErrBackupChecksumMismatch, `
 backup data checksum failed: %s may be changed
 calculated sha256 is %s,
 origin sha256 is %s`,
-							file.Name, hex.EncodeToString(s[:]), hex.EncodeToString(file.Sha256))
+								file.Name, hex.EncodeToString(s[:]), hex.EncodeToString(file.Sha256))
+						}
+					}
+					if tbl.Info == nil {
+						log.Info("table info(empty)", zap.Stringer("db", db.Info.Name))
+					} else {
+						log.Info("table info", zap.Stringer("table", tbl.Info.Name),
+							zap.Uint64("CRC64", calCRC64),
+							zap.Uint64("totalKvs", totalKVs),
+							zap.Uint64("totalBytes", totalBytes),
+							zap.Uint64("schemaTotalKvs", tbl.TotalKvs),
+							zap.Uint64("schemaTotalBytes", tbl.TotalBytes),
+							zap.Uint64("schemaCRC64", tbl.Crc64Xor))
 					}
 				}
-				log.Info("table info", zap.Stringer("table", tblInfo.Name),
-					zap.Uint64("CRC64", calCRC64),
-					zap.Uint64("totalKvs", totalKVs),
-					zap.Uint64("totalBytes", totalBytes),
-					zap.Uint64("schemaTotalKvs", schema.TotalKvs),
-					zap.Uint64("schemaTotalBytes", schema.TotalBytes),
-					zap.Uint64("schemaCRC64", schema.Crc64Xor))
 			}
 			cmd.Println("backup data checksum succeed!")
 			return nil

diff --git a/br/pkg/lightning/backend/backend.go b/br/pkg/lightning/backend/backend.go
@@ -103,12 +103,13 @@ type LocalEngineConfig struct {
 
 // ExternalEngineConfig is the configuration used for local backend external engine.
 type ExternalEngineConfig struct {
-	StorageURI string
-	DataFiles  []string
-	StatFiles  []string
-	MinKey     []byte
-	MaxKey     []byte
-	SplitKeys  [][]byte
+	StorageURI      string
+	DataFiles       []string
+	StatFiles       []string
+	MinKey          []byte
+	MaxKey          []byte
+	SplitKeys       [][]byte
+	RegionSplitSize int64
 	// TotalFileSize can be an estimated value.
 	TotalFileSize int64
 	// TotalKVCount can be an estimated value.

diff --git a/br/pkg/lightning/backend/external/BUILD.bazel b/br/pkg/lightning/backend/external/BUILD.bazel
@@ -23,6 +23,7 @@ go_library(
         "//br/pkg/lightning/backend/encode",
         "//br/pkg/lightning/backend/kv",
         "//br/pkg/lightning/common",
+        "//br/pkg/lightning/config",
         "//br/pkg/lightning/log",
         "//br/pkg/membuf",
         "//br/pkg/storage",
@@ -57,7 +58,7 @@ go_test(
     ],
     embed = [":external"],
     flaky = True,
-    shard_count = 39,
+    shard_count = 40,
     deps = [
         "//br/pkg/lightning/backend/kv",
         "//br/pkg/lightning/common",