Merge branch 'master' into cluster_info__start_time__native

.cilinter.yaml

@@ -3,12 +3,9 @@ run:
 linters:
   disable-all: true
   enable:
-    - typecheck
     - varcheck
-    - unused
     - structcheck
     - deadcode
-    - bodyclose
     - rowserrcheck
     - prealloc
 

bindinfo/bind_test.go

@@ -27,21 +27,15 @@ import (
 	"github.com/pingcap/tidb/parser/auth"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/parser/terror"
-	plannercore "github.com/pingcap/tidb/planner/core"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/util"
 	"github.com/stretchr/testify/require"
 )
 
 func TestPrepareCacheWithBinding(t *testing.T) {
 	store := testkit.CreateMockStore(t)
-	orgEnable := plannercore.PreparedPlanCacheEnabled()
-	defer func() {
-		plannercore.SetPreparedPlanCache(orgEnable)
-	}()
-	plannercore.SetPreparedPlanCache(true)
-
 	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(`set tidb_enable_prepared_plan_cache=1`)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t1, t2")
 	tk.MustExec("create table t1(a int, b int, c int, key idx_b(b), key idx_c(c))")

bindinfo/session_handle_test.go

@@ -26,7 +26,6 @@ import (
 	"github.com/pingcap/tidb/metrics"
 	"github.com/pingcap/tidb/parser"
 	"github.com/pingcap/tidb/parser/auth"
-	plannercore "github.com/pingcap/tidb/planner/core"
 	"github.com/pingcap/tidb/session/txninfo"
 	"github.com/pingcap/tidb/testkit"
 	"github.com/pingcap/tidb/util"
@@ -508,15 +507,8 @@ func TestDropSingleBindings(t *testing.T) {
 
 func TestPreparedStmt(t *testing.T) {
 	store := testkit.CreateMockStore(t)
-
 	tk := testkit.NewTestKit(t, store)
-
-	orgEnable := plannercore.PreparedPlanCacheEnabled()
-	defer func() {
-		plannercore.SetPreparedPlanCache(orgEnable)
-	}()
-	plannercore.SetPreparedPlanCache(false) // requires plan cache disabled, or the IndexNames = 1 on first test.
-
+	tk.MustExec(`set tidb_enable_prepared_plan_cache=1`)
 	tk.MustExec("use test")
 	tk.MustExec("drop table if exists t")
 	tk.MustExec("create table t(a int, b int, index idx(a))")

br/pkg/lightning/restore/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
         "check_template.go",
         "checksum.go",
         "get_pre_info.go",
+        "get_pre_info_opts.go",
         "meta_manager.go",
         "precheck.go",
         "precheck_impl.go",
@@ -57,6 +58,7 @@ go_library(
         "//table/tables",
         "//types",
         "//util/collate",
+        "//util/dbterror",
         "//util/engine",
         "//util/mathutil",
         "//util/mock",

br/pkg/lightning/restore/get_pre_info.go

@@ -49,6 +49,7 @@ import (
 	"github.com/pingcap/tidb/store/pdtypes"
 	"github.com/pingcap/tidb/table/tables"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/dbterror"
 	"github.com/pingcap/tidb/util/mock"
 	"go.uber.org/zap"
 	"golang.org/x/exp/maps"
@@ -272,6 +273,7 @@ func (g *TargetInfoGetterImpl) GetEmptyRegionsInfo(ctx context.Context) (*pdtype
 // PreRestoreInfoGetterImpl implements the operations to get information used in importing preparation.
 type PreRestoreInfoGetterImpl struct {
 	cfg              *config.Config
+	getPreInfoCfg    *GetPreInfoConfig
 	srcStorage       storage.ExternalStorage
 	ioWorkers        *worker.Pool
 	encBuilder       backend.EncodingBuilder
@@ -290,6 +292,7 @@ func NewPreRestoreInfoGetter(
 	targetInfoGetter TargetInfoGetter,
 	ioWorkers *worker.Pool,
 	encBuilder backend.EncodingBuilder,
+	opts ...GetPreInfoOption,
 ) (*PreRestoreInfoGetterImpl, error) {
 	if ioWorkers == nil {
 		ioWorkers = worker.NewPool(context.Background(), cfg.App.IOConcurrency, "pre_info_getter_io")
@@ -305,8 +308,13 @@ func NewPreRestoreInfoGetter(
 		}
 	}
 
+	getPreInfoCfg := NewDefaultGetPreInfoConfig()
+	for _, o := range opts {
+		o.Apply(getPreInfoCfg)
+	}
 	result := &PreRestoreInfoGetterImpl{
 		cfg:              cfg,
+		getPreInfoCfg:    getPreInfoCfg,
 		dbMetas:          dbMetas,
 		srcStorage:       srcStorage,
 		ioWorkers:        ioWorkers,
@@ -368,8 +376,20 @@ func (p *PreRestoreInfoGetterImpl) getTableStructuresByFileMeta(ctx context.Cont
 	dbName := dbSrcFileMeta.Name
 	currentTableInfosFromDB, err := p.targetInfoGetter.FetchRemoteTableModels(ctx, dbName)
 	if err != nil {
+		if p.getPreInfoCfg != nil && p.getPreInfoCfg.IgnoreDBNotExist {
+			dbNotExistErr := dbterror.ClassSchema.NewStd(errno.ErrBadDB).FastGenByArgs(dbName)
+			// The returned error is an error showing get info request error,
+			// and attaches the detailed error response as a string.
+			// So we cannot get the error chain and use error comparison,
+			// and instead, we use the string comparison on error messages.
+			if strings.Contains(err.Error(), dbNotExistErr.Error()) {
+				log.L().Warn("DB not exists.  But ignore it", zap.Error(err))
+				goto get_struct_from_src
+			}
+		}
 		return nil, errors.Trace(err)
 	}
+get_struct_from_src:
 	currentTableInfosMap := make(map[string]*model.TableInfo)
 	for _, tblInfo := range currentTableInfosFromDB {
 		currentTableInfosMap[tblInfo.Name.L] = tblInfo

br/pkg/lightning/restore/get_pre_info_opts.go

@@ -0,0 +1,42 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package restore
+
+type GetPreInfoConfig struct {
+	IgnoreDBNotExist bool
+}
+
+func NewDefaultGetPreInfoConfig() *GetPreInfoConfig {
+	return &GetPreInfoConfig{
+		IgnoreDBNotExist: false,
+	}
+}
+
+type GetPreInfoOption interface {
+	Apply(c *GetPreInfoConfig)
+}
+
+type ignoreDBNotExistOption struct {
+	ignoreDBNotExist bool
+}
+
+func (o *ignoreDBNotExistOption) Apply(c *GetPreInfoConfig) {
+	c.IgnoreDBNotExist = o.ignoreDBNotExist
+}
+
+func WithIgnoreDBNotExist(ignoreDBNotExist bool) GetPreInfoOption {
+	return &ignoreDBNotExistOption{
+		ignoreDBNotExist: ignoreDBNotExist,
+	}
+}

br/pkg/lightning/restore/get_pre_info_test.go

@@ -227,7 +227,7 @@ func TestGetPreInfoGetAllTableStructures(t *testing.T) {
 
 	cfg := config.NewConfig()
 	cfg.TikvImporter.Backend = config.BackendLocal
-	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil)
+	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil, WithIgnoreDBNotExist(true))
 	require.NoError(t, err)
 	tblStructMap, err := ig.GetAllTableStructures(ctx)
 	require.Nil(t, err)
@@ -400,7 +400,7 @@ func TestGetPreInfoSampleSource(t *testing.T) {
 	mockTarget := mock.NewMockTargetInfo()
 	cfg := config.NewConfig()
 	cfg.TikvImporter.Backend = config.BackendLocal
-	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil)
+	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil, WithIgnoreDBNotExist(true))
 	require.NoError(t, err)
 
 	mdDBMeta := mockSrc.GetAllDBFileMetas()[0]
@@ -490,7 +490,7 @@ func TestGetPreInfoEstimateSourceSize(t *testing.T) {
 	mockTarget := mock.NewMockTargetInfo()
 	cfg := config.NewConfig()
 	cfg.TikvImporter.Backend = config.BackendLocal
-	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil)
+	ig, err := NewPreRestoreInfoGetter(cfg, mockSrc.GetAllDBFileMetas(), mockSrc.GetStorage(), mockTarget, nil, nil, WithIgnoreDBNotExist(true))
 	require.NoError(t, err)
 
 	sizeResult, err := ig.EstimateSourceDataSize(ctx)

br/pkg/lightning/restore/mock/BUILD.bazel

@@ -8,8 +8,10 @@ go_library(
     deps = [
         "//br/pkg/lightning/mydump",
         "//br/pkg/storage",
+        "//errno",
         "//parser/model",
         "//store/pdtypes",
+        "//util/dbterror",
         "//util/filter",
         "@com_github_pingcap_errors//:errors",
         "@com_github_pingcap_kvproto//pkg/metapb",

br/pkg/lightning/restore/mock/mock.go

@@ -21,8 +21,10 @@ import (
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/tidb/br/pkg/lightning/mydump"
 	"github.com/pingcap/tidb/br/pkg/storage"
+	"github.com/pingcap/tidb/errno"
 	"github.com/pingcap/tidb/parser/model"
 	"github.com/pingcap/tidb/store/pdtypes"
+	"github.com/pingcap/tidb/util/dbterror"
 	"github.com/pingcap/tidb/util/filter"
 )
 
@@ -202,7 +204,8 @@ func (t *MockTargetInfo) FetchRemoteTableModels(ctx context.Context, schemaName
 	resultInfos := []*model.TableInfo{}
 	tblMap, ok := t.dbTblInfoMap[schemaName]
 	if !ok {
-		return resultInfos, nil
+		dbNotExistErr := dbterror.ClassSchema.NewStd(errno.ErrBadDB).FastGenByArgs(schemaName)
+		return nil, errors.Errorf("get xxxxxx http status code != 200, message %s", dbNotExistErr.Error())
 	}
 	for _, tblInfo := range tblMap {
 		resultInfos = append(resultInfos, tblInfo.TableModel)

br/pkg/lightning/restore/precheck.go

@@ -48,14 +48,40 @@ func WithPrecheckKey(ctx context.Context, key precheckContextKey, val any) conte
 	return context.WithValue(ctx, key, val)
 }
 
+type PrecheckItemBuilderConfig struct {
+	PreInfoGetterOptions []GetPreInfoOption
+}
+
+type PrecheckItemBuilderOption interface {
+	Apply(c *PrecheckItemBuilderConfig)
+}
+
+type preInfoGetterOptsForBuilder struct {
+	opts []GetPreInfoOption
+}
+
+func (o *preInfoGetterOptsForBuilder) Apply(c *PrecheckItemBuilderConfig) {
+	c.PreInfoGetterOptions = append([]GetPreInfoOption{}, o.opts...)
+}
+
+func WithPreInfoGetterOptions(opts ...GetPreInfoOption) PrecheckItemBuilderOption {
+	return &preInfoGetterOptsForBuilder{
+		opts: opts,
+	}
+}
+
 type PrecheckItemBuilder struct {
 	cfg           *config.Config
 	dbMetas       []*mydump.MDDatabaseMeta
 	preInfoGetter PreRestoreInfoGetter
 	checkpointsDB checkpoints.DB
 }
 
-func NewPrecheckItemBuilderFromConfig(ctx context.Context, cfg *config.Config) (*PrecheckItemBuilder, error) {
+func NewPrecheckItemBuilderFromConfig(ctx context.Context, cfg *config.Config, opts ...PrecheckItemBuilderOption) (*PrecheckItemBuilder, error) {
+	builderCfg := new(PrecheckItemBuilderConfig)
+	for _, o := range opts {
+		o.Apply(builderCfg)
+	}
 	targetDB, err := DBFromConfig(ctx, cfg.TiDB)
 	if err != nil {
 		return nil, errors.Trace(err)
@@ -77,6 +103,7 @@ func NewPrecheckItemBuilderFromConfig(ctx context.Context, cfg *config.Config) (
 		targetInfoGetter,
 		nil, // ioWorkers
 		nil, // encBuilder
+		builderCfg.PreInfoGetterOptions...,
 	)
 	if err != nil {
 		return nil, errors.Trace(err)

br/pkg/lightning/restore/precheck_impl_test.go

@@ -59,7 +59,7 @@ func (s *precheckImplSuite) setMockImportData(mockDataMap map[string]*mock.MockD
 	if err != nil {
 		return err
 	}
-	s.preInfoGetter, err = NewPreRestoreInfoGetter(s.cfg, s.mockSrc.GetAllDBFileMetas(), s.mockSrc.GetStorage(), s.mockTarget, nil, nil)
+	s.preInfoGetter, err = NewPreRestoreInfoGetter(s.cfg, s.mockSrc.GetAllDBFileMetas(), s.mockSrc.GetStorage(), s.mockTarget, nil, nil, WithIgnoreDBNotExist(true))
 	if err != nil {
 		return err
 	}

build/BUILD.bazel

@@ -54,6 +54,7 @@ STATICHECK_ANALYZERS = [
     "S1038",
     "S1039",
     "S1040",
+    "SA1019",
     "SA2000",
     "SA2001",
     "SA2003",
@@ -121,6 +122,7 @@ nogo(
                "@org_golang_x_tools//go/analysis/passes/unsafeptr:go_default_library",
                "@org_golang_x_tools//go/analysis/passes/unusedresult:go_default_library",
                "//build/linter/asciicheck:asciicheck",
+               "//build/linter/bodyclose:bodyclose",
                "//build/linter/durationcheck:durationcheck",
                "//build/linter/exportloopref:exportloopref",
                "//build/linter/gofmt:gofmt",

build/linter/bodyclose/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "bodyclose",
+    srcs = ["analysis.go"],
+    importpath = "github.com/pingcap/tidb/build/linter/bodyclose",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//build/linter/util",
+        "@com_github_timakin_bodyclose//passes/bodyclose",
+    ],
+)

build/linter/bodyclose/analysis.go

@@ -0,0 +1,27 @@
+// Copyright 2022 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bodyclose
+
+import (
+	"github.com/pingcap/tidb/build/linter/util"
+	"github.com/timakin/bodyclose/passes/bodyclose"
+)
+
+// Analyzer is the analyzer struct of bodyclose.
+var Analyzer = bodyclose.Analyzer
+
+func init() {
+	util.SkipAnalyzer(Analyzer)
+}

build/nogo_config.json

@@ -40,7 +40,8 @@
   },
   "bodyclose": {
     "exclude_files": {
-      "external/": "no need to vet third party code"
+      "external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
     }
   },
   "bools": {
@@ -605,6 +606,16 @@
       "parser/parser.go": "ignore generated code"
     }
   },
+  "SA1019": {
+    "exclude_files": {
+      "/build/": "no need to linter code",
+      "/external/": "no need to vet third party code",
+      ".*_generated\\.go$": "ignore generated code"
+    },
+     "only_files": {
+       "ddl/": "enable to ddl"
+     }
+  },
   "SA2000": {
     "exclude_files": {
       "/external/": "no need to vet third party code",