Merge branch 'master' into gotestsum-verifyci

.golangci.yml

@@ -28,9 +28,6 @@ linters-settings:
  checks: ["S1002","S1004","S1007","S1009","S1010","S1012","S1019","S1020","S1021","S1024","S1030","SA2*","SA3*","SA4009","SA5*","SA6000","SA6001","SA6005", "-SA2002"]
  stylecheck:
  checks: ["-ST1003"]
- gosec:
- excludes:
- - G601
 issues:
  exclude-rules:
  - path: _test\.go

bindinfo/bind_serial_test.go

@@ -555,7 +555,7 @@ func TestHintsSetID(t *testing.T) {
 
  utilCleanBindingEnv(tk, dom)
  err := tk.ExecToErr("create global binding for select * from t using select /*+ non_exist_hint() */ * from t")
- require.True(t, terror.ErrorEqual(err, parser.ErrWarnOptimizerHintParseError))
+ require.True(t, terror.ErrorEqual(err, parser.ErrParse))
  tk.MustExec("create global binding for select * from t where a > 10 using select * from t where a > 10")
  bindData = bindHandle.GetBindRecord(hash, sql, "test")
  require.NotNil(t, bindData)
@@ -826,11 +826,11 @@ func TestForbidEvolvePlanBaseLinesBeforeGA(t *testing.T) {
  err := tk.ExecToErr("set @@tidb_evolve_plan_baselines=0")
  require.Equal(t, nil, err)
  err = tk.ExecToErr("set @@TiDB_Evolve_pLan_baselines=1")
- require.Regexp(t, "Cannot enable baseline evolution feature, it is not generally available now", err)
+ require.EqualError(t, err, "Cannot enable baseline evolution feature, it is not generally available now")
  err = tk.ExecToErr("set @@TiDB_Evolve_pLan_baselines=oN")
- require.Regexp(t, "Cannot enable baseline evolution feature, it is not generally available now", err)
+ require.EqualError(t, err, "Cannot enable baseline evolution feature, it is not generally available now")
  err = tk.ExecToErr("admin evolve bindings")
- require.Regexp(t, "Cannot enable baseline evolution feature, it is not generally available now", err)
+ require.EqualError(t, err, "Cannot enable baseline evolution feature, it is not generally available now")
 }
 
 func TestExplainTableStmts(t *testing.T) {
@@ -861,7 +861,8 @@ func TestSPMWithoutUseDatabase(t *testing.T) {
  tk.MustExec("create global binding for select * from t using select * from t force index(a)")
 
  err := tk1.ExecToErr("select * from t")
- require.Regexp(t, ".*No database selected", err)
+ require.Error(t, err)
+ require.Regexp(t, "No database selected$", err)
  tk1.MustQuery(`select @@last_plan_from_binding;`).Check(testkit.Rows("0"))
  require.True(t, tk1.MustUseIndex("select * from test.t", "a"))
  tk1.MustExec("select * from test.t")
@@ -883,6 +884,22 @@ func TestBindingWithoutCharset(t *testing.T) {
  require.Equal(t, "SELECT * FROM `test`.`t` WHERE `a` = 'aa'", rows[0][1])
 }
 
+func TestBindingWithMultiParenthesis(t *testing.T) {
+ store, clean := testkit.CreateMockStore(t)
+ defer clean()
+
+ tk := testkit.NewTestKit(t, store)
+ tk.MustExec("use test")
+ tk.MustExec("drop table if exists t")
+ tk.MustExec("create table t (a int)")
+ tk.MustExec("create global binding for select * from (select * from t where a = 1) tt using select * from (select * from t where a = 1) tt")
+ tk.MustExec("create global binding for select * from ((select * from t where a = 1)) tt using select * from (select * from t where a = 1) tt")
+ rows := tk.MustQuery("show global bindings").Rows()
+ require.Len(t, rows, 1)
+ require.Equal(t, "select * from ( select * from `test` . `t` where `a` = ? ) as `tt`", rows[0][0])
+ require.Equal(t, "SELECT * FROM (SELECT * FROM `test`.`t` WHERE `a` = 1) AS `tt`", rows[0][1])
+}
+
 func TestGCBindRecord(t *testing.T) {
  // set lease for gc tests
  originLease := bindinfo.Lease

bindinfo/cache.go

@@ -105,7 +105,8 @@ func (br *BindRecord) HasUsingBinding() bool {
 
 // FindBinding find bindings in BindRecord.
 func (br *BindRecord) FindBinding(hint string) *Binding {
- for _, binding := range br.Bindings {
+ for i := range br.Bindings {
+ binding := br.Bindings[i]
  if binding.ID == hint {
  return &binding
  }
@@ -160,7 +161,8 @@ func merge(lBindRecord, rBindRecord *BindRecord) *BindRecord {
  return lBindRecord
  }
  result := lBindRecord.shallowCopy()
- for _, rbind := range rBindRecord.Bindings {
+ for i := range rBindRecord.Bindings {
+ rbind := rBindRecord.Bindings[i]
  found := false
  for j, lbind := range lBindRecord.Bindings {
  if lbind.isSame(&rbind) {
@@ -184,7 +186,8 @@ func (br *BindRecord) remove(deleted *BindRecord) *BindRecord {
  return &BindRecord{OriginalSQL: br.OriginalSQL, Db: br.Db}
  }
  result := br.shallowCopy()
- for _, deletedBind := range deleted.Bindings {
+ for j := range deleted.Bindings {
+ deletedBind := deleted.Bindings[j]
  for i, bind := range result.Bindings {
  if bind.isSame(&deletedBind) {
  result.Bindings = append(result.Bindings[:i], result.Bindings[i+1:]...)

br/cmd/tidb-lightning-ctl/main.go

@@ -229,8 +229,8 @@ func checkpointErrorDestroy(ctx context.Context, cfg *config.Config, tls *common
  for engineID := table.MinEngineID; engineID <= table.MaxEngineID; engineID++ {
  fmt.Fprintln(os.Stderr, "Closing and cleaning up engine:", table.TableName, engineID)
  _, eID := backend.MakeUUID(table.TableName, engineID)
- file := local.File{UUID: eID}
- err := file.Cleanup(cfg.TikvImporter.SortedKVDir)
+ engine := local.Engine{UUID: eID}
+ err := engine.Cleanup(cfg.TikvImporter.SortedKVDir)
  if err != nil {
  fmt.Fprintln(os.Stderr, "* Encountered error while cleanup engine:", err)
  lastErr = err

br/cmd/tidb-lightning/main.go

@@ -30,7 +30,10 @@ import (
 
 func main() {
  globalCfg := config.Must(config.LoadGlobalConfig(os.Args[1:], nil))
- fmt.Fprintf(os.Stdout, "Verbose debug logs will be written to %s\n\n", globalCfg.App.Config.File)
+ logToFile := globalCfg.App.File != "" && globalCfg.App.File != "-"
+ if logToFile {
+ fmt.Fprintf(os.Stdout, "Verbose debug logs will be written to %s\n\n", globalCfg.App.Config.File)
+ }
 
  app := lightning.New(globalCfg)
 
@@ -95,7 +98,7 @@ func main() {
  }
 
  // call Sync() with log to stdout may return error in some case, so just skip it
- if globalCfg.App.File != "" {
+ if logToFile {
  syncErr := logger.Sync()
  if syncErr != nil {
  fmt.Fprintln(os.Stderr, "sync log failed", syncErr)

br/pkg/backup/client.go

@@ -180,14 +180,9 @@ func (bc *Client) SetStorage(ctx context.Context, backend *backuppb.StorageBacke
  "there may be some backup files in the path already, "+
  "please specify a correct backup directory!", bc.storage.URI()+"/"+metautil.MetaFile)
  }
- exist, err = bc.storage.FileExists(ctx, metautil.LockFile)
+ err = CheckBackupStorageIsLocked(ctx, bc.storage)
  if err != nil {
- return errors.Annotatef(err, "error occurred when checking %s file", metautil.LockFile)
- }
- if exist {
- return errors.Annotatef(berrors.ErrInvalidArgument, "backup lock file exists in %v, "+
- "there may be some backup files in the path already, "+
- "please specify a correct backup directory!", bc.storage.URI()+"/"+metautil.LockFile)
+ return err
  }
  bc.backend = backend
  return nil
@@ -198,6 +193,29 @@ func (bc *Client) GetClusterID() uint64 {
  return bc.clusterID
 }
 
+// CheckBackupStorageIsLocked checks whether backups is locked.
+// which means we found other backup progress already write
+// some data files into the same backup directory or cloud prefix.
+func CheckBackupStorageIsLocked(ctx context.Context, s storage.ExternalStorage) error {
+ exist, err := s.FileExists(ctx, metautil.LockFile)
+ if err != nil {
+ return errors.Annotatef(err, "error occurred when checking %s file", metautil.LockFile)
+ }
+ if exist {
+ err = s.WalkDir(ctx, &storage.WalkOption{}, func(path string, size int64) error {
+ // should return error to break the walkDir when found lock file and other .sst files.
+ if strings.HasSuffix(path, ".sst") {
+ return errors.Annotatef(berrors.ErrInvalidArgument, "backup lock file and sst file exist in %v, "+
+ "there are some backup files in the path already, "+
+ "please specify a correct backup directory!", s.URI()+"/"+metautil.LockFile)
+ }
+ return nil
+ })
+ return err
+ }
+ return nil
+}
+
 // BuildTableRanges returns the key ranges encompassing the entire table,
 // and its partitions if exists.
 func BuildTableRanges(tbl *model.TableInfo) ([]kv.KeyRange, error) {
@@ -719,7 +737,7 @@ func OnBackupResponse(
  if lockErr := v.KvError.Locked; lockErr != nil {
  // Try to resolve lock.
  log.Warn("backup occur kv error", zap.Reflect("error", v))
- msBeforeExpired, _, err1 := lockResolver.ResolveLocks(
+ msBeforeExpired, err1 := lockResolver.ResolveLocks(
  bo, backupTS, []*txnlock.Lock{txnlock.NewLock(lockErr)})
  if err1 != nil {
  return nil, 0, errors.Trace(err1)

br/pkg/backup/client_test.go

@@ -70,6 +70,13 @@ func (r *testBackup) SetUpSuite(c *C) {
 
 }
 
+func (r *testBackup) resetStorage(c *C) {
+ var err error
+ base := c.MkDir()
+ r.storage, err = storage.NewLocalStorage(base)
+ c.Assert(err, IsNil)
+}
+
 func (r *testBackup) TestGetTS(c *C) {
  var (
  err error
@@ -335,3 +342,30 @@ func (r *testBackup) TestskipUnsupportedDDLJob(c *C) {
  c.Assert(err, IsNil)
  c.Assert(len(allDDLJobs), Equals, 8)
 }
+
+func (r *testBackup) TestCheckBackupIsLocked(c *C) {
+ ctx := context.Background()
+
+ r.resetStorage(c)
+ // check passed with an empty storage
+ err := backup.CheckBackupStorageIsLocked(ctx, r.storage)
+ c.Assert(err, IsNil)
+
+ // check passed with only a lock file
+ err = r.storage.WriteFile(ctx, metautil.LockFile, nil)
+ c.Assert(err, IsNil)
+ err = backup.CheckBackupStorageIsLocked(ctx, r.storage)
+ c.Assert(err, IsNil)
+
+ // check passed with a lock file and other non-sst files.
+ err = r.storage.WriteFile(ctx, "1.txt", nil)
+ c.Assert(err, IsNil)
+ err = backup.CheckBackupStorageIsLocked(ctx, r.storage)
+ c.Assert(err, IsNil)
+
+ // check failed
+ err = r.storage.WriteFile(ctx, "1.sst", nil)
+ c.Assert(err, IsNil)
+ err = backup.CheckBackupStorageIsLocked(ctx, r.storage)
+ c.Assert(err, ErrorMatches, "backup lock file and sst file exist in(.+)")
+}

br/pkg/conn/conn_test.go

@@ -201,7 +201,7 @@ func TestGetAllTiKVStores(t *testing.T) {
  {Id: 2, Labels: []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}}},
  },
  storeBehavior: ErrorOnTiFlash,
- expectedError: "cannot restore to a cluster with active TiFlash stores.*",
+ expectedError: "^cannot restore to a cluster with active TiFlash stores",
  },
  {
  stores: []*metapb.Store{
@@ -225,7 +225,7 @@ func TestGetAllTiKVStores(t *testing.T) {
  {Id: 6, Labels: []*metapb.StoreLabel{{Key: "else", Value: "tiflash"}, {Key: "engine", Value: "tikv"}}},
  },
  storeBehavior: ErrorOnTiFlash,
- expectedError: "cannot restore to a cluster with active TiFlash stores.*",
+ expectedError: "^cannot restore to a cluster with active TiFlash stores",
  },
  {
  stores: []*metapb.Store{
@@ -267,9 +267,9 @@ func TestGetConnOnCanceledContext(t *testing.T) {
 
  _, err := mgr.GetBackupClient(ctx, 42)
  require.Error(t, err)
- require.Regexp(t, ".*context canceled.*", err.Error())
+ require.Contains(t, err.Error(), "context canceled")
 
  _, err = mgr.ResetBackupClient(ctx, 42)
  require.Error(t, err)
- require.Regexp(t, ".*context canceled.*", err.Error())
+ require.Contains(t, err.Error(), "context canceled")
 }

br/pkg/gluetikv/glue_test.go

@@ -22,5 +22,5 @@ import (
 
 func TestGetVersion(t *testing.T) {
  g := Glue{}
- require.Regexp(t, "BR(.|\n)*Release Version(.|\n)*Git Commit Hash(.|\n)*", g.GetVersion())
+ require.Regexp(t, "^BR(.|\n)*Release Version(.|\n)*Git Commit Hash(.|\n)*$", g.GetVersion())
 }