Merge branch 'master' into rename-explaintest

Author: Defined2014

.codecov.yml

@@ -8,15 +8,23 @@ coverage:
   round: down
   range: "65...90"
 
+  # status:
+  #   project:
+  #     default:
+  #       threshold: 0.2 #Allow the coverage to drop by threshold%, and posting a success status.
+  #   patch:
+  #     default:
+  #       target: 0% # trial operation
+
+  # Ref: https://docs.codecov.com/docs/commit-status#disabling-a-status
   status:
-    project: 
-      default:
-        threshold: 0.2 #Allow the coverage to drop by threshold%, and posting a success status.
-    patch: 
-      default:
-        target: 0% # trial operation
+    project: off # disable it
+    patch: off # disable it
     changes: no
 
+# Ref: https://docs.codecov.com/docs/github-checks#disabling-github-checks-completely-via-yaml
+github_checks: false
+
 parsers:
   gcov:
     branch_detection:
@@ -43,7 +51,7 @@ component_management:
     - component_id: component_parser
       name: parser
       paths:
-        - parser/**      
+        - parser/**
     - component_id: component_br
       name: br
       paths:
@@ -52,7 +60,7 @@ component_management:
 
 flag_management:
   default_rules: # the rules that will be followed for any flag added, generally
-    carryforward: false
+    carryforward: true
     statuses:
       - type: project
         target: auto
@@ -61,11 +69,11 @@ flag_management:
 
 ignore:
   - "LICENSES"
-  - "*_test.go"       
+  - "*_test.go"
   - ".git"
   - "*.yml"
   - "*.md"
-  - "cmd/.*"       
+  - "cmd/.*"
   - "docs/.*"
   - "vendor/.*"
   - "ddl/failtest/.*"

Makefile

@@ -493,7 +493,7 @@ bazel_test: failpoint-enable bazel_prepare
 		-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...
 
 
-bazel_coverage_test: check-bazel-prepare failpoint-enable bazel_ci_simple_prepare
+bazel_coverage_test: failpoint-enable bazel_ci_simple_prepare
 	bazel $(BAZEL_GLOBAL_CONFIG) --nohome_rc coverage $(BAZEL_CMD_CONFIG) --jobs=35 --build_tests_only --test_keep_going=false \
 		--@io_bazel_rules_go//go/config:cover_format=go_cover --define gotags=deadlock,intest \
 		-- //... -//cmd/... -//tests/graceshutdown/... \

bindinfo/internal/testutil.go

@@ -35,6 +35,6 @@ func UtilNormalizeWithDefaultDB(t *testing.T, sql string) (normalized, digest st
 	testParser := parser.New()
 	stmt, err := testParser.ParseOneStmt(sql, "", "")
 	require.NoError(t, err)
-	normalized, digestResult := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(stmt, "test", ""))
+	normalized, digestResult := parser.NormalizeDigestForBinding(utilparser.RestoreWithDefaultDB(stmt, "test", ""))
 	return normalized, digestResult.String()
 }

bindinfo/tests/BUILD.bazel

@@ -9,7 +9,7 @@ go_test(
     ],
     flaky = True,
     race = "on",
-    shard_count = 34,
+    shard_count = 35,
     deps = [
         "//bindinfo",
         "//bindinfo/internal",

bindinfo/tests/bind_test.go

@@ -414,7 +414,7 @@ func TestBindingSymbolList(t *testing.T) {
 	require.True(t, tk.MustUseIndex("select a, b from t where a = 3 limit 1, 100", "ib(b)"))
 
 	// Normalize
-	sql, hash := parser.NormalizeDigest("select a, b from test . t where a = 1 limit 0, 1")
+	sql, hash := parser.NormalizeDigestForBinding("select a, b from test . t where a = 1 limit 0, 1")
 
 	bindData := dom.BindHandle().GetBindRecord(hash.String(), sql, "test")
 	require.NotNil(t, bindData)
@@ -429,6 +429,50 @@ func TestBindingSymbolList(t *testing.T) {
 	require.NotNil(t, bind.UpdateTime)
 }
 
+// TestBindingInListWithSingleLiteral tests sql with "IN (Lit)", fixes #44298
+func TestBindingInListWithSingleLiteral(t *testing.T) {
+	store, dom := testkit.CreateMockStoreAndDomain(t)
+
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b int, INDEX ia (a), INDEX ib (b));")
+	tk.MustExec("insert into t value(1, 1);")
+
+	// GIVEN
+	sqlcmd := "select a, b from t where a in (1)"
+	binding := `create global binding for select a, b from t where a in (1, 2, 3) using select a, b from t use index (ib) where a in (1, 2, 3)`
+
+	// before binding
+	tk.MustQuery(sqlcmd)
+	require.Equal(t, "t:ia", tk.Session().GetSessionVars().StmtCtx.IndexNames[0])
+	require.True(t, tk.MustUseIndex(sqlcmd, "ia(a)"))
+
+	tk.MustExec(binding)
+
+	// after binding
+	tk.MustQuery(sqlcmd)
+	require.Equal(t, "t:ib", tk.Session().GetSessionVars().StmtCtx.IndexNames[0])
+	require.True(t, tk.MustUseIndex(sqlcmd, "ib(b)"))
+
+	tk.MustQuery("select @@last_plan_from_binding").Check(testkit.Rows("1"))
+
+	// Normalize
+	sql, hash := parser.NormalizeDigestForBinding("select a, b from test . t where a in (1)")
+
+	bindData := dom.BindHandle().GetBindRecord(hash.String(), sql, "test")
+	require.NotNil(t, bindData)
+	require.Equal(t, "select `a` , `b` from `test` . `t` where `a` in ( ... )", bindData.OriginalSQL)
+	bind := bindData.Bindings[0]
+	require.Equal(t, "SELECT `a`,`b` FROM `test`.`t` USE INDEX (`ib`) WHERE `a` IN (1,2,3)", bind.BindSQL)
+	require.Equal(t, "test", bindData.Db)
+	require.Equal(t, bindinfo.Enabled, bind.Status)
+	require.NotNil(t, bind.Charset)
+	require.NotNil(t, bind.Collation)
+	require.NotNil(t, bind.CreateTime)
+	require.NotNil(t, bind.UpdateTime)
+}
+
 func TestDMLSQLBind(t *testing.T) {
 	store := testkit.CreateMockStore(t)
 
@@ -538,7 +582,7 @@ func TestErrorBind(t *testing.T) {
 	_, err := tk.Exec("create global binding for select * from t where i>100 using select * from t use index(index_t) where i>100")
 	require.NoError(t, err, "err %v", err)
 
-	sql, hash := parser.NormalizeDigest("select * from test . t where i > ?")
+	sql, hash := parser.NormalizeDigestForBinding("select * from test . t where i > ?")
 	bindData := dom.BindHandle().GetBindRecord(hash.String(), sql, "test")
 	require.NotNil(t, bindData)
 	require.Equal(t, "select * from `test` . `t` where `i` > ?", bindData.OriginalSQL)
@@ -1304,7 +1348,7 @@ func TestBindSQLDigest(t *testing.T) {
 		parser4binding := parser.New()
 		originNode, err := parser4binding.ParseOneStmt(c.origin, "utf8mb4", "utf8mb4_general_ci")
 		require.NoError(t, err)
-		_, sqlDigestWithDB := parser.NormalizeDigest(utilparser.RestoreWithDefaultDB(originNode, "test", c.origin))
+		_, sqlDigestWithDB := parser.NormalizeDigestForBinding(utilparser.RestoreWithDefaultDB(originNode, "test", c.origin))
 		require.Equal(t, res[0][9], sqlDigestWithDB.String())
 	}
 }

br/pkg/lightning/backend/backend.go

@@ -84,6 +84,8 @@ type EngineConfig struct {
 	TableInfo *checkpoints.TidbTableInfo
 	// local backend specified configuration
 	Local LocalEngineConfig
+	// local backend external engine specified configuration
+	External *ExternalEngineConfig
 	// KeepSortDir indicates whether to keep the temporary sort directory
 	// when opening the engine, instead of removing it.
 	KeepSortDir bool
@@ -99,6 +101,20 @@ type LocalEngineConfig struct {
 	CompactConcurrency int
 }
 
+// ExternalEngineConfig is the configuration used for local backend external engine.
+type ExternalEngineConfig struct {
+	StorageURI string
+	DataFiles  []string
+	StatFiles  []string
+	MinKey     []byte
+	MaxKey     []byte
+	SplitKeys  [][]byte
+	// TotalFileSize can be an estimated value.
+	TotalFileSize int64
+	// TotalKVCount can be an estimated value.
+	TotalKVCount int64
+}
+
 // CheckCtx contains all parameters used in CheckRequirements
 type CheckCtx struct {
 	DBMetas []*mydump.MDDatabaseMeta

br/pkg/lightning/backend/external/BUILD.bazel

@@ -53,7 +53,7 @@ go_test(
     ],
     embed = [":external"],
     flaky = True,
-    shard_count = 28,
+    shard_count = 31,
     deps = [
         "//br/pkg/lightning/backend/kv",
         "//br/pkg/lightning/common",

br/pkg/lightning/backend/external/engine.go

@@ -38,6 +38,8 @@ type Engine struct {
 	storage    storage.ExternalStorage
 	dataFiles  []string
 	statsFiles []string
+	minKey     []byte
+	maxKey     []byte
 	splitKeys  [][]byte
 	bufPool    *membuf.Pool
 
@@ -49,8 +51,8 @@ type Engine struct {
 	dupDetectOpt       common.DupDetectOpt
 	ts                 uint64
 
-	totalKVSize   int64
-	totalKVLength int64
+	totalKVSize  int64
+	totalKVCount int64
 
 	importedKVSize  *atomic.Int64
 	importedKVCount *atomic.Int64
@@ -61,26 +63,32 @@ func NewExternalEngine(
 	storage storage.ExternalStorage,
 	dataFiles []string,
 	statsFiles []string,
+	minKey []byte,
+	maxKey []byte,
+	splitKeys [][]byte,
 	keyAdapter common.KeyAdapter,
 	duplicateDetection bool,
 	duplicateDB *pebble.DB,
 	dupDetectOpt common.DupDetectOpt,
 	ts uint64,
 	totalKVSize int64,
-	totakKVLength int64,
+	totalKVCount int64,
 ) common.Engine {
 	return &Engine{
 		storage:            storage,
 		dataFiles:          dataFiles,
 		statsFiles:         statsFiles,
+		minKey:             minKey,
+		maxKey:             maxKey,
+		splitKeys:          splitKeys,
 		bufPool:            membuf.NewPool(),
 		keyAdapter:         keyAdapter,
 		duplicateDetection: duplicateDetection,
 		duplicateDB:        duplicateDB,
 		dupDetectOpt:       dupDetectOpt,
 		ts:                 ts,
 		totalKVSize:        totalKVSize,
-		totalKVLength:      totakKVLength,
+		totalKVCount:       totalKVCount,
 		importedKVSize:     atomic.NewInt64(0),
 		importedKVCount:    atomic.NewInt64(0),
 	}
@@ -177,13 +185,13 @@ func (e *Engine) createMergeIter(ctx context.Context, start kv.Key) (*MergeKVIte
 	return iter, nil
 }
 
-// KVStatistics returns the total kv size and total kv length.
-func (e *Engine) KVStatistics() (totalKVSize int64, totalKVLength int64) {
-	return e.totalKVSize, e.totalKVLength
+// KVStatistics returns the total kv size and total kv count.
+func (e *Engine) KVStatistics() (totalKVSize int64, totalKVCount int64) {
+	return e.totalKVSize, e.totalKVCount
 }
 
-// ImportedStatistics returns the imported kv size and imported kv length.
-func (e *Engine) ImportedStatistics() (importedKVSize int64, importedKVLength int64) {
+// ImportedStatistics returns the imported kv size and imported kv count.
+func (e *Engine) ImportedStatistics() (importedSize int64, importedKVCount int64) {
 	return e.importedKVSize.Load(), e.importedKVCount.Load()
 }
 
@@ -192,6 +200,11 @@ func (e *Engine) ID() string {
 	return "external"
 }
 
+// GetKeyRange implements common.Engine.
+func (e *Engine) GetKeyRange() (firstKey []byte, lastKey []byte, err error) {
+	return e.minKey, e.maxKey, nil
+}
+
 // SplitRanges split the ranges by split keys provided by external engine.
 func (e *Engine) SplitRanges(
 	startKey, endKey []byte,

br/pkg/lightning/backend/external/util.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"slices"
 	"sort"
 	"strings"
 
@@ -199,3 +200,50 @@ func MockExternalEngineWithWriter(
 	}
 	return GetAllFileNames(ctx, storage, subDir)
 }
+
+// EndpointTp is the type of Endpoint.Key.
+type EndpointTp int
+
+const (
+	// ExclusiveEnd represents "..., Endpoint.Key)".
+	ExclusiveEnd EndpointTp = iota
+	// InclusiveStart represents "[Endpoint.Key, ...".
+	InclusiveStart
+	// InclusiveEnd represents "..., Endpoint.Key]".
+	InclusiveEnd
+)
+
+// Endpoint represents an endpoint of an interval which can be used by GetMaxOverlapping.
+type Endpoint struct {
+	Key    []byte
+	Tp     EndpointTp
+	Weight uint64 // all EndpointTp use positive weight
+}
+
+// GetMaxOverlapping returns the maximum overlapping weight treating given
+// `points` as endpoints of intervals. `points` are not required to be sorted,
+// and will be sorted in-place in this function.
+func GetMaxOverlapping(points []Endpoint) int {
+	slices.SortFunc(points, func(i, j Endpoint) int {
+		if cmp := bytes.Compare(i.Key, j.Key); cmp != 0 {
+			return cmp
+		}
+		return int(i.Tp) - int(j.Tp)
+	})
+	var maxWeight uint64
+	var curWeight uint64
+	for _, p := range points {
+		switch p.Tp {
+		case InclusiveStart:
+			curWeight += p.Weight
+		case ExclusiveEnd:
+			curWeight -= p.Weight
+		case InclusiveEnd:
+			curWeight -= p.Weight
+		}
+		if curWeight > maxWeight {
+			maxWeight = curWeight
+		}
+	}
+	return int(maxWeight)
+}

br/pkg/lightning/backend/external/util_test.go

@@ -209,3 +209,34 @@ func TestCleanUpFiles(t *testing.T) {
 	require.Equal(t, []string(nil), statFiles)
 	require.Equal(t, []string(nil), dataFiles)
 }
+
+func TestGetMaxOverlapping(t *testing.T) {
+	// [1, 3), [2, 4)
+	points := []Endpoint{
+		{Key: []byte{1}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{3}, Tp: ExclusiveEnd, Weight: 1},
+		{Key: []byte{2}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{4}, Tp: ExclusiveEnd, Weight: 1},
+	}
+	require.Equal(t, 2, GetMaxOverlapping(points))
+	// [1, 3), [2, 4), [3, 5)
+	points = []Endpoint{
+		{Key: []byte{1}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{3}, Tp: ExclusiveEnd, Weight: 1},
+		{Key: []byte{2}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{4}, Tp: ExclusiveEnd, Weight: 1},
+		{Key: []byte{3}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{5}, Tp: ExclusiveEnd, Weight: 1},
+	}
+	require.Equal(t, 2, GetMaxOverlapping(points))
+	// [1, 3], [2, 4], [3, 5]
+	points = []Endpoint{
+		{Key: []byte{1}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{3}, Tp: InclusiveEnd, Weight: 1},
+		{Key: []byte{2}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{4}, Tp: InclusiveEnd, Weight: 1},
+		{Key: []byte{3}, Tp: InclusiveStart, Weight: 1},
+		{Key: []byte{5}, Tp: InclusiveEnd, Weight: 1},
+	}
+	require.Equal(t, 3, GetMaxOverlapping(points))
+}