diff --git a/cmd/explaintest/main.go b/cmd/explaintest/main.go index be897ec594fdc..5216ed25ff561 100644 --- a/cmd/explaintest/main.go +++ b/cmd/explaintest/main.go @@ -644,6 +644,24 @@ func main() { if _, err = mdb.Exec("set @@tidb_hash_join_concurrency=1"); err != nil { log.Fatal("set @@tidb_hash_join_concurrency=1 failed", zap.Error(err)) } +<<<<<<< HEAD +======= + resets := []string{ + "set @@tidb_index_lookup_concurrency=4", + "set @@tidb_index_lookup_join_concurrency=4", + "set @@tidb_hashagg_final_concurrency=4", + "set @@tidb_hashagg_partial_concurrency=4", + "set @@tidb_window_concurrency=4", + "set @@tidb_projection_concurrency=4", + "set @@tidb_distsql_scan_concurrency=15", + "set @@global.tidb_enable_clustered_index=0;", + } + for _, sql := range resets { + if _, err = mdb.Exec(sql); err != nil { + log.Fatal(fmt.Sprintf("%s failed", sql), zap.Error(err)) + } + } +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) if _, err = mdb.Exec("set sql_mode='STRICT_TRANS_TABLES'"); err != nil { log.Fatal("set sql_mode='STRICT_TRANS_TABLES' failed", zap.Error(err)) diff --git a/ddl/db_change_test.go b/ddl/db_change_test.go index 7b38eea190270..cd8a164379cf5 100644 --- a/ddl/db_change_test.go +++ b/ddl/db_change_test.go @@ -744,6 +744,47 @@ func (s *testStateChangeSuite) TestParallelAlterModifyColumn(c *C) { s.testControlParallelExecSQL(c, sql, sql, f) } +<<<<<<< HEAD +======= +func (s *testStateChangeSuite) TestParallelAddGeneratedColumnAndAlterModifyColumn(c *C) { + _, err := s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 1") + c.Assert(err, IsNil) + defer func() { + _, err = s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 0") + c.Assert(err, IsNil) + }() + + sql1 := "ALTER TABLE t ADD COLUMN f INT GENERATED ALWAYS AS(a+1);" + sql2 := "ALTER TABLE t MODIFY COLUMN a tinyint;" + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true, oldCol is a dependent column 'a' for generated column") + _, err := s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + } + s.testControlParallelExecSQL(c, sql1, sql2, f) +} + +func (s *testStateChangeSuite) TestParallelAlterModifyColumnAndAddPK(c *C) { + _, err := s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 1") + c.Assert(err, IsNil) + defer func() { + _, err = s.se.Execute(context.Background(), "set global tidb_enable_change_column_type = 0") + c.Assert(err, IsNil) + }() + + sql1 := "ALTER TABLE t ADD PRIMARY KEY (b) NONCLUSTERED;" + sql2 := "ALTER TABLE t MODIFY COLUMN b tinyint;" + f := func(c *C, err1, err2 error) { + c.Assert(err1, IsNil) + c.Assert(err2.Error(), Equals, "[ddl:8200]Unsupported modify column: tidb_enable_change_column_type is true and this column has primary key flag") + _, err := s.se.Execute(context.Background(), "select * from t") + c.Assert(err, IsNil) + } + s.testControlParallelExecSQL(c, sql1, sql2, f) +} + +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) // TODO: This test is not a test that performs two DDLs in parallel. // So we should not use the function of testControlParallelExecSQL. We will handle this test in the next PR. // func (s *testStateChangeSuite) TestParallelColumnModifyingDefinition(c *C) { diff --git a/domain/domain.go b/domain/domain.go index c581ea037fcc7..071fface91bd3 100644 --- a/domain/domain.go +++ b/domain/domain.go @@ -73,7 +73,7 @@ type Domain struct { sysSessionPool *sessionPool exit chan struct{} etcdClient *clientv3.Client - gvc GlobalVariableCache + sysVarCache SysVarCache // replaces GlobalVariableCache slowQuery *topNSlowQueries expensiveQueryHandle *expensivequery.Handle wg sync.WaitGroup @@ -896,6 +896,55 @@ func (do *Domain) LoadPrivilegeLoop(ctx sessionctx.Context) error { return nil } +// LoadSysVarCacheLoop create a goroutine loads sysvar cache in a loop, +// it should be called only once in BootstrapSession. +func (do *Domain) LoadSysVarCacheLoop(ctx sessionctx.Context) error { + err := do.sysVarCache.RebuildSysVarCache(ctx) + if err != nil { + return err + } + var watchCh clientv3.WatchChan + duration := 30 * time.Second + if do.etcdClient != nil { + watchCh = do.etcdClient.Watch(context.Background(), sysVarCacheKey) + } + do.wg.Add(1) + go func() { + defer func() { + do.wg.Done() + logutil.BgLogger().Info("LoadSysVarCacheLoop exited.") + util.Recover(metrics.LabelDomain, "LoadSysVarCacheLoop", nil, false) + }() + var count int + for { + ok := true + select { + case <-do.exit: + return + case _, ok = <-watchCh: + case <-time.After(duration): + } + if !ok { + logutil.BgLogger().Error("LoadSysVarCacheLoop loop watch channel closed") + watchCh = do.etcdClient.Watch(context.Background(), sysVarCacheKey) + count++ + if count > 10 { + time.Sleep(time.Duration(count) * time.Second) + } + continue + } + count = 0 + logutil.BgLogger().Debug("Rebuilding sysvar cache from etcd watch event.") + err := do.sysVarCache.RebuildSysVarCache(ctx) + metrics.LoadSysVarCacheCounter.WithLabelValues(metrics.RetLabel(err)).Inc() + if err != nil { + logutil.BgLogger().Error("LoadSysVarCacheLoop failed", zap.Error(err)) + } + } + }() + return nil +} + // PrivilegeHandle returns the MySQLPrivilege. func (do *Domain) PrivilegeHandle() *privileges.Handle { return do.privHandle @@ -1198,12 +1247,19 @@ func (do *Domain) ExpensiveQueryHandle() *expensivequery.Handle { return do.expensiveQueryHandle } +<<<<<<< HEAD // InitExpensiveQueryHandle init the expensive query handler. func (do *Domain) InitExpensiveQueryHandle() { do.expensiveQueryHandle = expensivequery.NewExpensiveQueryHandle(do.exit) } const privilegeKey = "/tidb/privilege" +======= +const ( + privilegeKey = "/tidb/privilege" + sysVarCacheKey = "/tidb/sysvars" +) +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) // NotifyUpdatePrivilege updates privilege key in etcd, TiDB client that watches // the key will get notification. @@ -1225,6 +1281,262 @@ func (do *Domain) NotifyUpdatePrivilege(ctx sessionctx.Context) { } } +<<<<<<< HEAD +======= +// NotifyUpdateSysVarCache updates the sysvar cache key in etcd, which other TiDB +// clients are subscribed to for updates. For the caller, the cache is also built +// synchronously so that the effect is immediate. +func (do *Domain) NotifyUpdateSysVarCache(ctx sessionctx.Context) { + if do.etcdClient != nil { + row := do.etcdClient.KV + _, err := row.Put(context.Background(), sysVarCacheKey, "") + if err != nil { + logutil.BgLogger().Warn("notify update sysvar cache failed", zap.Error(err)) + } + } + // update locally + if err := do.sysVarCache.RebuildSysVarCache(ctx); err != nil { + logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err)) + } +} + +// ServerID gets serverID. +func (do *Domain) ServerID() uint64 { + return atomic.LoadUint64(&do.serverID) +} + +// IsLostConnectionToPD indicates lost connection to PD or not. +func (do *Domain) IsLostConnectionToPD() bool { + return do.isLostConnectionToPD.Get() != 0 +} + +const ( + serverIDEtcdPath = "/tidb/server_id" + refreshServerIDRetryCnt = 3 + acquireServerIDRetryInterval = 300 * time.Millisecond + acquireServerIDTimeout = 10 * time.Second + retrieveServerIDSessionTimeout = 10 * time.Second +) + +var ( + // serverIDTTL should be LONG ENOUGH to avoid barbarically killing an on-going long-run SQL. + serverIDTTL = 12 * time.Hour + // serverIDTimeToKeepAlive is the interval that we keep serverID TTL alive periodically. + serverIDTimeToKeepAlive = 5 * time.Minute + // serverIDTimeToCheckPDConnectionRestored is the interval that we check connection to PD restored (after broken) periodically. + serverIDTimeToCheckPDConnectionRestored = 10 * time.Second + // lostConnectionToPDTimeout is the duration that when TiDB cannot connect to PD excceeds this limit, + // we realize the connection to PD is lost utterly, and server ID acquired before should be released. + // Must be SHORTER than `serverIDTTL`. + lostConnectionToPDTimeout = 6 * time.Hour +) + +var ( + ldflagIsGlobalKillTest = "0" // 1:Yes, otherwise:No. + ldflagServerIDTTL = "10" // in seconds. + ldflagServerIDTimeToKeepAlive = "1" // in seconds. + ldflagServerIDTimeToCheckPDConnectionRestored = "1" // in seconds. + ldflagLostConnectionToPDTimeout = "5" // in seconds. +) + +func initByLDFlagsForGlobalKill() { + if ldflagIsGlobalKillTest == "1" { + var ( + i int + err error + ) + + if i, err = strconv.Atoi(ldflagServerIDTTL); err != nil { + panic("invalid ldflagServerIDTTL") + } + serverIDTTL = time.Duration(i) * time.Second + + if i, err = strconv.Atoi(ldflagServerIDTimeToKeepAlive); err != nil { + panic("invalid ldflagServerIDTimeToKeepAlive") + } + serverIDTimeToKeepAlive = time.Duration(i) * time.Second + + if i, err = strconv.Atoi(ldflagServerIDTimeToCheckPDConnectionRestored); err != nil { + panic("invalid ldflagServerIDTimeToCheckPDConnectionRestored") + } + serverIDTimeToCheckPDConnectionRestored = time.Duration(i) * time.Second + + if i, err = strconv.Atoi(ldflagLostConnectionToPDTimeout); err != nil { + panic("invalid ldflagLostConnectionToPDTimeout") + } + lostConnectionToPDTimeout = time.Duration(i) * time.Second + + logutil.BgLogger().Info("global_kill_test is enabled", zap.Duration("serverIDTTL", serverIDTTL), + zap.Duration("serverIDTimeToKeepAlive", serverIDTimeToKeepAlive), + zap.Duration("serverIDTimeToCheckPDConnectionRestored", serverIDTimeToCheckPDConnectionRestored), + zap.Duration("lostConnectionToPDTimeout", lostConnectionToPDTimeout)) + } +} + +func (do *Domain) retrieveServerIDSession(ctx context.Context) (*concurrency.Session, error) { + if do.serverIDSession != nil { + return do.serverIDSession, nil + } + + // `etcdClient.Grant` needs a shortterm timeout, to avoid blocking if connection to PD lost, + // while `etcdClient.KeepAlive` should be longterm. + // So we separately invoke `etcdClient.Grant` and `concurrency.NewSession` with leaseID. + childCtx, cancel := context.WithTimeout(ctx, retrieveServerIDSessionTimeout) + resp, err := do.etcdClient.Grant(childCtx, int64(serverIDTTL.Seconds())) + cancel() + if err != nil { + logutil.BgLogger().Error("retrieveServerIDSession.Grant fail", zap.Error(err)) + return nil, err + } + leaseID := resp.ID + + session, err := concurrency.NewSession(do.etcdClient, + concurrency.WithLease(leaseID), concurrency.WithContext(context.Background())) + if err != nil { + logutil.BgLogger().Error("retrieveServerIDSession.NewSession fail", zap.Error(err)) + return nil, err + } + do.serverIDSession = session + return session, nil +} + +func (do *Domain) acquireServerID(ctx context.Context) error { + atomic.StoreUint64(&do.serverID, 0) + + session, err := do.retrieveServerIDSession(ctx) + if err != nil { + return err + } + + for { + randServerID := rand.Int63n(int64(util.MaxServerID)) + 1 // get a random serverID: [1, MaxServerID] + key := fmt.Sprintf("%s/%v", serverIDEtcdPath, randServerID) + cmp := clientv3.Compare(clientv3.CreateRevision(key), "=", 0) + value := "0" + + childCtx, cancel := context.WithTimeout(ctx, acquireServerIDTimeout) + txn := do.etcdClient.Txn(childCtx) + t := txn.If(cmp) + resp, err := t.Then(clientv3.OpPut(key, value, clientv3.WithLease(session.Lease()))).Commit() + cancel() + if err != nil { + return err + } + if !resp.Succeeded { + logutil.BgLogger().Info("proposed random serverID exists, will randomize again", zap.Int64("randServerID", randServerID)) + time.Sleep(acquireServerIDRetryInterval) + continue + } + + atomic.StoreUint64(&do.serverID, uint64(randServerID)) + logutil.BgLogger().Info("acquireServerID", zap.Uint64("serverID", do.ServerID()), + zap.String("lease id", strconv.FormatInt(int64(session.Lease()), 16))) + return nil + } +} + +func (do *Domain) refreshServerIDTTL(ctx context.Context) error { + session, err := do.retrieveServerIDSession(ctx) + if err != nil { + return err + } + + key := fmt.Sprintf("%s/%v", serverIDEtcdPath, do.ServerID()) + value := "0" + err = ddlutil.PutKVToEtcd(ctx, do.etcdClient, refreshServerIDRetryCnt, key, value, clientv3.WithLease(session.Lease())) + if err != nil { + logutil.BgLogger().Error("refreshServerIDTTL fail", zap.Uint64("serverID", do.ServerID()), zap.Error(err)) + } else { + logutil.BgLogger().Info("refreshServerIDTTL succeed", zap.Uint64("serverID", do.ServerID()), + zap.String("lease id", strconv.FormatInt(int64(session.Lease()), 16))) + } + return err +} + +func (do *Domain) serverIDKeeper() { + defer func() { + do.wg.Done() + logutil.BgLogger().Info("serverIDKeeper exited.") + }() + defer util.Recover(metrics.LabelDomain, "serverIDKeeper", func() { + logutil.BgLogger().Info("recover serverIDKeeper.") + // should be called before `do.wg.Done()`, to ensure that Domain.Close() waits for the new `serverIDKeeper()` routine. + do.wg.Add(1) + go do.serverIDKeeper() + }, false) + + tickerKeepAlive := time.NewTicker(serverIDTimeToKeepAlive) + tickerCheckRestored := time.NewTicker(serverIDTimeToCheckPDConnectionRestored) + defer func() { + tickerKeepAlive.Stop() + tickerCheckRestored.Stop() + }() + + blocker := make(chan struct{}) // just used for blocking the sessionDone() when session is nil. + sessionDone := func() <-chan struct{} { + if do.serverIDSession == nil { + return blocker + } + return do.serverIDSession.Done() + } + + var lastSucceedTimestamp time.Time + + onConnectionToPDRestored := func() { + logutil.BgLogger().Info("restored connection to PD") + do.isLostConnectionToPD.Set(0) + lastSucceedTimestamp = time.Now() + + if err := do.info.StoreServerInfo(context.Background()); err != nil { + logutil.BgLogger().Error("StoreServerInfo failed", zap.Error(err)) + } + } + + onConnectionToPDLost := func() { + logutil.BgLogger().Warn("lost connection to PD") + do.isLostConnectionToPD.Set(1) + + // Kill all connections when lost connection to PD, + // to avoid the possibility that another TiDB instance acquires the same serverID and generates a same connection ID, + // which will lead to a wrong connection killed. + do.InfoSyncer().GetSessionManager().KillAllConnections() + } + + for { + select { + case <-tickerKeepAlive.C: + if !do.IsLostConnectionToPD() { + if err := do.refreshServerIDTTL(context.Background()); err == nil { + lastSucceedTimestamp = time.Now() + } else { + if lostConnectionToPDTimeout > 0 && time.Since(lastSucceedTimestamp) > lostConnectionToPDTimeout { + onConnectionToPDLost() + } + } + } + case <-tickerCheckRestored.C: + if do.IsLostConnectionToPD() { + if err := do.acquireServerID(context.Background()); err == nil { + onConnectionToPDRestored() + } + } + case <-sessionDone(): + // inform that TTL of `serverID` is expired. See https://godoc.org/github.com/coreos/etcd/clientv3/concurrency#Session.Done + // Should be in `IsLostConnectionToPD` state, as `lostConnectionToPDTimeout` is shorter than `serverIDTTL`. + // So just set `do.serverIDSession = nil` to restart `serverID` session in `retrieveServerIDSession()`. + logutil.BgLogger().Info("serverIDSession need restart") + do.serverIDSession = nil + case <-do.exit: + return + } + } +} + +func init() { + initByLDFlagsForGlobalKill() +} + +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) var ( // ErrInfoSchemaExpired returns the error that information schema is out of date. ErrInfoSchemaExpired = dbterror.ClassDomain.NewStd(errno.ErrInfoSchemaExpired) diff --git a/domain/sysvar_cache.go b/domain/sysvar_cache.go new file mode 100644 index 0000000000000..23c9688ea2f81 --- /dev/null +++ b/domain/sysvar_cache.go @@ -0,0 +1,167 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package domain + +import ( + "context" + "fmt" + "sync" + + "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/sessionctx/variable" + "github.com/pingcap/tidb/util/logutil" + "github.com/pingcap/tidb/util/sqlexec" + "github.com/pingcap/tidb/util/stmtsummary" + "go.uber.org/zap" +) + +// The sysvar cache replaces the GlobalVariableCache. +// It is an improvement because it operates similar to privilege cache, +// where it caches for 5 minutes instead of 2 seconds, plus it listens on etcd +// for updates from other servers. + +// SysVarCache represents the cache of system variables broken up into session and global scope. +type SysVarCache struct { + sync.RWMutex + global map[string]string + session map[string]string +} + +// GetSysVarCache gets the global variable cache. +func (do *Domain) GetSysVarCache() *SysVarCache { + return &do.sysVarCache +} + +func (svc *SysVarCache) rebuildCacheIfNeeded(ctx sessionctx.Context) (err error) { + svc.RLock() + cacheNeedsRebuild := len(svc.session) == 0 || len(svc.global) == 0 + svc.RUnlock() + if cacheNeedsRebuild { + logutil.BgLogger().Warn("sysvar cache is empty, triggering rebuild") + if err = svc.RebuildSysVarCache(ctx); err != nil { + logutil.BgLogger().Error("rebuilding sysvar cache failed", zap.Error(err)) + } + } + return err +} + +// GetSessionCache gets a copy of the session sysvar cache. +// The intention is to copy it directly to the systems[] map +// on creating a new session. +func (svc *SysVarCache) GetSessionCache(ctx sessionctx.Context) (map[string]string, error) { + if err := svc.rebuildCacheIfNeeded(ctx); err != nil { + return nil, err + } + svc.RLock() + defer svc.RUnlock() + // Perform a deep copy since this will be assigned directly to the session + newMap := make(map[string]string, len(svc.session)) + for k, v := range svc.session { + newMap[k] = v + } + return newMap, nil +} + +// GetGlobalVar gets an individual global var from the sysvar cache. +func (svc *SysVarCache) GetGlobalVar(ctx sessionctx.Context, name string) (string, error) { + if err := svc.rebuildCacheIfNeeded(ctx); err != nil { + return "", err + } + svc.RLock() + defer svc.RUnlock() + + if val, ok := svc.global[name]; ok { + return val, nil + } + logutil.BgLogger().Warn("could not find key in global cache", zap.String("name", name)) + return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) +} + +func (svc *SysVarCache) fetchTableValues(ctx sessionctx.Context) (map[string]string, error) { + tableContents := make(map[string]string) + // Copy all variables from the table to tableContents + exec := ctx.(sqlexec.RestrictedSQLExecutor) + stmt, err := exec.ParseWithParams(context.Background(), `SELECT variable_name, variable_value FROM mysql.global_variables`) + if err != nil { + return tableContents, err + } + rows, _, err := exec.ExecRestrictedStmt(context.TODO(), stmt) + if err != nil { + return nil, err + } + for _, row := range rows { + name := row.GetString(0) + val := row.GetString(1) + tableContents[name] = val + } + return tableContents, nil +} + +// RebuildSysVarCache rebuilds the sysvar cache both globally and for session vars. +// It needs to be called when sysvars are added or removed. +func (svc *SysVarCache) RebuildSysVarCache(ctx sessionctx.Context) error { + newSessionCache := make(map[string]string) + newGlobalCache := make(map[string]string) + tableContents, err := svc.fetchTableValues(ctx) + if err != nil { + return err + } + + for _, sv := range variable.GetSysVars() { + sVal := sv.Value + if _, ok := tableContents[sv.Name]; ok { + sVal = tableContents[sv.Name] + } + if sv.HasSessionScope() { + newSessionCache[sv.Name] = sVal + } + if sv.HasGlobalScope() { + newGlobalCache[sv.Name] = sVal + } + // Propagate any changes to the server scoped variables + checkEnableServerGlobalVar(sv.Name, sVal) + } + + logutil.BgLogger().Debug("rebuilding sysvar cache") + + svc.Lock() + defer svc.Unlock() + svc.session = newSessionCache + svc.global = newGlobalCache + return nil +} + +// checkEnableServerGlobalVar processes variables that acts in server and global level. +func checkEnableServerGlobalVar(name, sVal string) { + var err error + switch name { + case variable.TiDBEnableStmtSummary: + err = stmtsummary.StmtSummaryByDigestMap.SetEnabled(sVal, false) + case variable.TiDBStmtSummaryInternalQuery: + err = stmtsummary.StmtSummaryByDigestMap.SetEnabledInternalQuery(sVal, false) + case variable.TiDBStmtSummaryRefreshInterval: + err = stmtsummary.StmtSummaryByDigestMap.SetRefreshInterval(sVal, false) + case variable.TiDBStmtSummaryHistorySize: + err = stmtsummary.StmtSummaryByDigestMap.SetHistorySize(sVal, false) + case variable.TiDBStmtSummaryMaxStmtCount: + err = stmtsummary.StmtSummaryByDigestMap.SetMaxStmtCount(sVal, false) + case variable.TiDBStmtSummaryMaxSQLLength: + err = stmtsummary.StmtSummaryByDigestMap.SetMaxSQLLength(sVal, false) + case variable.TiDBCapturePlanBaseline: + variable.CapturePlanBaseline.Set(sVal, false) + } + if err != nil { + logutil.BgLogger().Error(fmt.Sprintf("load global variable %s error", name), zap.Error(err)) + } +} diff --git a/executor/executor_test.go b/executor/executor_test.go index 741fcde0945c5..dbb57697259c7 100644 --- a/executor/executor_test.go +++ b/executor/executor_test.go @@ -2135,8 +2135,6 @@ func (s *testSuiteP2) TestSQLMode(c *C) { tk.MustExec("set sql_mode = 'STRICT_TRANS_TABLES'") tk.MustExec("set @@global.sql_mode = ''") - // Disable global variable cache, so load global session variable take effect immediate. - s.domain.GetGlobalVarsCache().Disable() tk2 := testkit.NewTestKit(c, s.store) tk2.MustExec("use test") tk2.MustExec("drop table if exists t2") diff --git a/executor/seqtest/seq_executor_test.go b/executor/seqtest/seq_executor_test.go index 738a54122e1ba..4f36ec57f425d 100644 --- a/executor/seqtest/seq_executor_test.go +++ b/executor/seqtest/seq_executor_test.go @@ -1463,8 +1463,6 @@ func (s *seqTestSuite) TestMaxDeltaSchemaCount(c *C) { tk := testkit.NewTestKit(c, s.store) tk.MustExec("use test") c.Assert(variable.GetMaxDeltaSchemaCount(), Equals, int64(variable.DefTiDBMaxDeltaSchemaCount)) - gvc := domain.GetDomain(tk.Se).GetGlobalVarsCache() - gvc.Disable() tk.MustExec("set @@global.tidb_max_delta_schema_count= -1") tk.MustQuery("show warnings;").Check(testkit.Rows("Warning 1292 Truncated incorrect tidb_max_delta_schema_count value: '-1'")) diff --git a/infoschema/tables_test.go b/infoschema/tables_test.go index aec469458a6a8..8468d9bf5a097 100644 --- a/infoschema/tables_test.go +++ b/infoschema/tables_test.go @@ -915,8 +915,6 @@ func (s *testTableSuite) TestStmtSummaryTable(c *C) { tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) @@ -1176,8 +1174,6 @@ func (s *testTableSuite) TestStmtSummaryHistoryTable(c *C) { tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) @@ -1218,8 +1214,6 @@ func (s *testTableSuite) TestStmtSummaryInternalQuery(c *C) { tk.MustExec("create global binding for select * from t where t.a = 1 using select * from t ignore index(k) where t.a = 1") tk.MustExec("set global tidb_enable_stmt_summary = 1") tk.MustQuery("select @@global.tidb_enable_stmt_summary").Check(testkit.Rows("1")) - // Invalidate the cache manually so that tidb_enable_stmt_summary works immediately. - s.dom.GetGlobalVarsCache().Disable() // Disable refreshing summary. tk.MustExec("set global tidb_stmt_summary_refresh_interval = 999999999") tk.MustQuery("select @@global.tidb_stmt_summary_refresh_interval").Check(testkit.Rows("999999999")) diff --git a/metrics/domain.go b/metrics/domain.go index dd3912555d59c..5397290f33993 100644 --- a/metrics/domain.go +++ b/metrics/domain.go @@ -47,6 +47,15 @@ var ( Help: "Counter of load privilege", }, []string{LblType}) + // LoadSysVarCacheCounter records the counter of loading sysvars + LoadSysVarCacheCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "domain", + Name: "load_sysvarcache_total", + Help: "Counter of load sysvar cache", + }, []string{LblType}) + SchemaValidatorStop = "stop" SchemaValidatorRestart = "restart" SchemaValidatorReset = "reset" diff --git a/metrics/metrics.go b/metrics/metrics.go index 75125d664d8d5..e6b9a57ca4e6e 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -175,4 +175,12 @@ func RegisterMetrics() { prometheus.MustRegister(ConfigStatus) prometheus.MustRegister(SmallTxnWriteDuration) prometheus.MustRegister(TxnWriteThroughput) +<<<<<<< HEAD +======= + prometheus.MustRegister(LoadSysVarCacheCounter) + + tikvmetrics.InitMetrics(TiDB, TiKVClient) + tikvmetrics.RegisterMetrics() + tikvmetrics.TiKVPanicCounter = PanicCounter // reset tidb metrics for tikv metrics +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) } diff --git a/planner/core/prepare_test.go b/planner/core/prepare_test.go index ca1ddb8424768..3fe6df6cebb5d 100644 --- a/planner/core/prepare_test.go +++ b/planner/core/prepare_test.go @@ -280,9 +280,6 @@ func (s *testPrepareSuite) TestPrepareOverMaxPreparedStmtCount(c *C) { tk.MustExec("set @@global.max_prepared_stmt_count = 2") tk.MustQuery("select @@global.max_prepared_stmt_count").Check(testkit.Rows("2")) - // Disable global variable cache, so load global session variable take effect immediate. - dom.GetGlobalVarsCache().Disable() - // test close session to give up all prepared stmt tk.MustExec(`prepare stmt2 from "select 1"`) prePrepared = readGaugeInt(metrics.PreparedStmtGauge) diff --git a/session/session.go b/session/session.go index 3dd6227307d45..a6d7eb60fabf6 100644 --- a/session/session.go +++ b/session/session.go @@ -965,9 +965,21 @@ func (s *session) GetAllSysVars() (map[string]string, error) { if err != nil { return nil, err } +<<<<<<< HEAD rows, _, err := s.ExecRestrictedStmt(context.TODO(), stmt) if err != nil { return nil, err +======= + _, _, err = s.ExecRestrictedStmt(ctx, stmt) + domain.GetDomain(s).NotifyUpdateSysVarCache(s) + return err +} + +func (s *session) varFromTiDBTable(name string) bool { + switch name { + case variable.TiDBGCConcurrency, variable.TiDBGCEnable, variable.TiDBGCRunInterval, variable.TiDBGCLifetime, variable.TiDBGCScanLockMode: + return true +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) } ret := make(map[string]string, len(rows)) for _, r := range rows { @@ -986,15 +998,35 @@ func (s *session) GetGlobalSysVar(name string) (string, error) { // When running bootstrap or upgrade, we should not access global storage. return "", nil } - sysVar, err := s.getTableValue(context.TODO(), mysql.GlobalVariablesTable, name) + + sv := variable.GetSysVar(name) + if sv == nil { + // It might be a recently unregistered sysvar. We should return unknown + // since GetSysVar is the canonical version, but we can update the cache + // so the next request doesn't attempt to load this. + logutil.BgLogger().Info("sysvar does not exist. sysvar cache may be stale", zap.String("name", name)) + return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) + } + + sysVar, err := domain.GetDomain(s).GetSysVarCache().GetGlobalVar(s, name) if err != nil { +<<<<<<< HEAD if errResultIsEmpty.Equal(err) { if sv, ok := variable.SysVars[name]; ok { return sv.Value, nil } return "", variable.ErrUnknownSystemVar.GenWithStackByArgs(name) +======= + // The sysvar exists, but there is no cache entry yet. + // This might be because the sysvar was only recently registered. + // In which case it is safe to return the default, but we can also + // update the cache for the future. + logutil.BgLogger().Info("sysvar not in cache yet. sysvar cache may be stale", zap.String("name", name)) + sysVar, err = s.getTableValue(context.TODO(), mysql.GlobalVariablesTable, name) + if err != nil { + return sv.Value, nil +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) } - return "", err } return sysVar, nil } @@ -1010,6 +1042,7 @@ func (s *session) SetGlobalSysVar(name, value string) error { return err } } +<<<<<<< HEAD var sVal string var err error sVal, err = variable.ValidateSetSystemVar(s.sessionVars, name, value, variable.ScopeGlobal) @@ -1018,6 +1051,69 @@ func (s *session) SetGlobalSysVar(name, value string) error { } name = strings.ToLower(name) stmt, err := s.ParseWithParams(context.TODO(), "REPLACE %n.%n VALUES (%?, %?)", mysql.SystemDB, mysql.GlobalVariablesTable, name, sVal) +======= + return s.replaceTableValue(context.TODO(), mysql.GlobalVariablesTable, sv.Name, value) +} + +// setTiDBTableValue handles tikv_* sysvars which need to update mysql.tidb +// for backwards compatibility. Validation has already been performed. +func (s *session) setTiDBTableValue(name, val string) error { + if name == variable.TiDBGCConcurrency { + autoConcurrency := "false" + if val == "-1" { + autoConcurrency = "true" + } + err := s.replaceTableValue(context.TODO(), mysql.TiDBTable, tiKVGCAutoConcurrency, autoConcurrency) + if err != nil { + return err + } + } + val = onOffToTrueFalse(val) + err := s.replaceTableValue(context.TODO(), mysql.TiDBTable, gcVariableMap[name], val) + return err +} + +// In mysql.tidb the convention has been to store the string value "true"/"false", +// but sysvars use the convention ON/OFF. +func trueFalseToOnOff(str string) string { + if strings.EqualFold("true", str) { + return variable.On + } else if strings.EqualFold("false", str) { + return variable.Off + } + return str +} + +// In mysql.tidb the convention has been to store the string value "true"/"false", +// but sysvars use the convention ON/OFF. +func onOffToTrueFalse(str string) string { + if strings.EqualFold("ON", str) { + return "true" + } else if strings.EqualFold("OFF", str) { + return "false" + } + return str +} + +// getTiDBTableValue handles tikv_* sysvars which need +// to read from mysql.tidb for backwards compatibility. +func (s *session) getTiDBTableValue(name, val string) (string, error) { + if name == variable.TiDBGCConcurrency { + // Check if autoconcurrency is set + autoConcurrencyVal, err := s.getTableValue(context.TODO(), mysql.TiDBTable, tiKVGCAutoConcurrency) + if err == nil && strings.EqualFold(autoConcurrencyVal, "true") { + return "-1", nil // convention for "AUTO" + } + } + tblValue, err := s.getTableValue(context.TODO(), mysql.TiDBTable, gcVariableMap[name]) + if err != nil { + return val, nil // mysql.tidb value does not exist. + } + // Run validation on the tblValue. This will return an error if it can't be validated, + // but will also make it more consistent: disTribuTeD -> DISTRIBUTED etc + tblValue = trueFalseToOnOff(tblValue) + validatedVal, err := variable.GetSysVar(name).Validate(s.sessionVars, tblValue, variable.ScopeGlobal) +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) if err != nil { return err } @@ -1951,14 +2047,28 @@ func BootstrapSession(store kv.Storage) (*domain.Domain, error) { } } + // Rebuild sysvar cache in a loop + err = dom.LoadSysVarCacheLoop(se) + if err != nil { + return nil, err + } + if len(cfg.Plugin.Load) > 0 { err := plugin.Init(context.Background(), plugin.Config{EtcdClient: dom.GetEtcdClient()}) if err != nil { return nil, err } } +<<<<<<< HEAD err = executor.LoadExprPushdownBlacklist(se) +======= + se4, err := createSession(store) + if err != nil { + return nil, err + } + err = executor.LoadExprPushdownBlacklist(se4) +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) if err != nil { return nil, err } @@ -2054,7 +2164,7 @@ func createSessionWithOpt(store kv.Storage, opt *Opt) (*session, error) { // CreateSessionWithDomain creates a new Session and binds it with a Domain. // We need this because when we start DDL in Domain, the DDL need a session // to change some system tables. But at that time, we have been already in -// a lock context, which cause we can't call createSesion directly. +// a lock context, which cause we can't call createSession directly. func CreateSessionWithDomain(store kv.Storage, dom *domain.Domain) (*session, error) { s := &session{ store: store, @@ -2250,6 +2360,7 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { return nil } +<<<<<<< HEAD var err error // Use GlobalVariableCache if TiDB just loaded global variables within 2 second ago. // When a lot of connections connect to TiDB simultaneously, it can protect TiKV meta region from overload. @@ -2258,11 +2369,17 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { return s.ExecRestrictedSQL(loadCommonGlobalVarsSQL) } rows, fields, err := gvc.LoadGlobalVariables(loadFunc) +======= + vars.CommonGlobalLoaded = true + + // Deep copy sessionvar cache + // Eventually this whole map will be applied to systems[], which is a MySQL behavior. + sessionCache, err := domain.GetDomain(s).GetSysVarCache().GetSessionCache(s) +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) if err != nil { - logutil.BgLogger().Warn("failed to load global variables", - zap.Uint64("conn", s.sessionVars.ConnectionID), zap.Error(err)) return err } +<<<<<<< HEAD vars.CommonGlobalLoaded = true for _, row := range rows { @@ -2270,6 +2387,24 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { varVal := row.GetDatum(1, &fields[1].Column.FieldType) if _, ok := vars.GetSystemVar(varName); !ok { err = variable.SetSessionSystemVar(s.sessionVars, varName, varVal) +======= + for _, varName := range builtinGlobalVariable { + // The item should be in the sessionCache, but due to a strange current behavior there are some Global-only + // vars that are in builtinGlobalVariable. For compatibility we need to fall back to the Global cache on these items. + // TODO: don't load these globals into the session! + var varVal string + var ok bool + if varVal, ok = sessionCache[varName]; !ok { + varVal, err = s.GetGlobalSysVar(varName) + if err != nil { + continue // skip variables that are not loaded. + } + } + // `collation_server` is related to `character_set_server`, set `character_set_server` will also set `collation_server`. + // We have to make sure we set the `collation_server` with right value. + if _, ok := vars.GetSystemVar(varName); !ok || varName == variable.CollationServer { + err = vars.SetSystemVarWithRelaxedValidation(varName, varVal) +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) if err != nil { return err } @@ -2284,8 +2419,6 @@ func (s *session) loadCommonGlobalVariablesIfNeeded() error { } } } - - vars.CommonGlobalLoaded = true return nil } diff --git a/session/session_test.go b/session/session_test.go index 6b2d0efd7d06c..a50e586199db9 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -202,7 +202,6 @@ func (s *testSessionSuiteBase) SetUpSuite(c *C) { var err error s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) - s.dom.GetGlobalVarsCache().Disable() } func (s *testSessionSuiteBase) TearDownSuite(c *C) { @@ -627,7 +626,6 @@ func (s *testSessionSuite) TestGlobalVarAccessor(c *C) { c.Assert(v, Equals, varValue2) // For issue 10955, make sure the new session load `max_execution_time` into sessionVars. - s.dom.GetGlobalVarsCache().Disable() tk1.MustExec("set @@global.max_execution_time = 100") tk2 := testkit.NewTestKitWithInit(c, s.store) c.Assert(tk2.Se.GetSessionVars().MaxExecutionTime, Equals, uint64(100)) @@ -2567,8 +2565,6 @@ func (s *testSessionSuite) TestSetGlobalTZ(c *C) { tk.MustQuery("show variables like 'time_zone'").Check(testkit.Rows("time_zone +08:00")) - // Disable global variable cache, so load global session variable take effect immediate. - s.dom.GetGlobalVarsCache().Disable() tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustQuery("show variables like 'time_zone'").Check(testkit.Rows("time_zone +00:00")) } @@ -2701,8 +2697,6 @@ func (s *testSessionSuite3) TestEnablePartition(c *C) { tk.MustQuery("show variables like 'tidb_enable_table_partition'").Check(testkit.Rows("tidb_enable_table_partition off")) tk.MustQuery("show global variables like 'tidb_enable_table_partition'").Check(testkit.Rows("tidb_enable_table_partition on")) - // Disable global variable cache, so load global session variable take effect immediate. - s.dom.GetGlobalVarsCache().Disable() tk1 := testkit.NewTestKitWithInit(c, s.store) tk1.MustQuery("show variables like 'tidb_enable_table_partition'").Check(testkit.Rows("tidb_enable_table_partition on")) } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index c12bb2251e87d..1e590e8173576 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1417,6 +1417,15 @@ func (s *SessionVars) SetSystemVar(name string, val string) error { return nil } +// SetSystemVarWithRelaxedValidation sets the value of a system variable for session scope. +// Validation functions are called, but scope validation is skipped. +// Errors are not expected to be returned because this could cause upgrade issues. +func (s *SessionVars) SetSystemVarWithRelaxedValidation(name string, val string) error { + sv := GetSysVar(name) + val = sv.ValidateWithRelaxedValidation(s, val, ScopeSession) + return sv.SetSessionFromHook(s, val) +} + // GetReadableTxnMode returns the session variable TxnMode but rewrites it to "OPTIMISTIC" when it's empty. func (s *SessionVars) GetReadableTxnMode() string { txnMode := s.TxnMode diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index f9713ef5ed0df..e752b1230d7a2 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -54,6 +54,262 @@ type SysVar struct { // Value is the variable value. Value string +<<<<<<< HEAD +======= + // Type is the MySQL type (optional) + Type TypeFlag + // MinValue will automatically be validated when specified (optional) + MinValue int64 + // MaxValue will automatically be validated when specified (optional) + MaxValue uint64 + // AutoConvertNegativeBool applies to boolean types (optional) + AutoConvertNegativeBool bool + // AutoConvertOutOfRange applies to int and unsigned types. + AutoConvertOutOfRange bool + // ReadOnly applies to all types + ReadOnly bool + // PossibleValues applies to ENUM type + PossibleValues []string + // AllowEmpty is a special TiDB behavior which means "read value from config" (do not use) + AllowEmpty bool + // AllowEmptyAll is a special behavior that only applies to TiDBCapturePlanBaseline, TiDBTxnMode (do not use) + AllowEmptyAll bool + // AllowAutoValue means that the special value "-1" is permitted, even when outside of range. + AllowAutoValue bool + // Validation is a callback after the type validation has been performed, but before the Set function + Validation func(*SessionVars, string, string, ScopeFlag) (string, error) + // SetSession is called after validation but before updating systems[]. It also doubles as an Init function + // and will be called on all variables in builtinGlobalVariable, regardless of their scope. + SetSession func(*SessionVars, string) error + // SetGlobal is called after validation + SetGlobal func(*SessionVars, string) error + // IsHintUpdatable indicate whether it's updatable via SET_VAR() hint (optional) + IsHintUpdatable bool + // Hidden means that it still responds to SET but doesn't show up in SHOW VARIABLES + Hidden bool + // Aliases is a list of sysvars that should also be updated when this sysvar is updated. + // Updating aliases calls the SET function of the aliases, but does not update their aliases (preventing SET recursion) + Aliases []string +} + +// SetSessionFromHook calls the SetSession func if it exists. +func (sv *SysVar) SetSessionFromHook(s *SessionVars, val string) error { + if sv.SetSession != nil { + if err := sv.SetSession(s, val); err != nil { + return err + } + } + s.systems[sv.Name] = val + + // Call the Set function on all the aliases for this sysVar + // Skipping the validation function, and not calling aliases of + // aliases. By skipping the validation function it means that things + // like duplicate warnings should not appear. + + if sv.Aliases != nil { + for _, aliasName := range sv.Aliases { + aliasSv := GetSysVar(aliasName) + if aliasSv.SetSession != nil { + if err := aliasSv.SetSession(s, val); err != nil { + return err + } + } + s.systems[aliasSv.Name] = val + } + } + return nil +} + +// SetGlobalFromHook calls the SetGlobal func if it exists. +func (sv *SysVar) SetGlobalFromHook(s *SessionVars, val string, skipAliases bool) error { + if sv.SetGlobal != nil { + return sv.SetGlobal(s, val) + } + + // Call the SetGlobalSysVarOnly function on all the aliases for this sysVar + // which skips the validation function and when SetGlobalFromHook is called again + // it will be with skipAliases=true. This helps break recursion because + // most aliases are reciprocal. + + if !skipAliases && sv.Aliases != nil { + for _, aliasName := range sv.Aliases { + if err := s.GlobalVarsAccessor.SetGlobalSysVarOnly(aliasName, val); err != nil { + return err + } + } + } + return nil +} + +// HasNoneScope returns true if the scope for the sysVar is None. +func (sv *SysVar) HasNoneScope() bool { + return sv.Scope == ScopeNone +} + +// HasSessionScope returns true if the scope for the sysVar includes session. +func (sv *SysVar) HasSessionScope() bool { + return sv.Scope&ScopeSession != 0 +} + +// HasGlobalScope returns true if the scope for the sysVar includes global. +func (sv *SysVar) HasGlobalScope() bool { + return sv.Scope&ScopeGlobal != 0 +} + +// Validate checks if system variable satisfies specific restriction. +func (sv *SysVar) Validate(vars *SessionVars, value string, scope ScopeFlag) (string, error) { + // Check that the scope is correct first. + if err := sv.validateScope(scope); err != nil { + return value, err + } + // Normalize the value and apply validation based on type. + // i.e. TypeBool converts 1/on/ON to ON. + normalizedValue, err := sv.validateFromType(vars, value, scope) + if err != nil { + return normalizedValue, err + } + // If type validation was successful, call the (optional) validation function + if sv.Validation != nil { + return sv.Validation(vars, normalizedValue, value, scope) + } + return normalizedValue, nil +} + +// validateFromType provides automatic validation based on the SysVar's type +func (sv *SysVar) validateFromType(vars *SessionVars, value string, scope ScopeFlag) (string, error) { + // The string "DEFAULT" is a special keyword in MySQL, which restores + // the compiled sysvar value. In which case we can skip further validation. + if strings.EqualFold(value, "DEFAULT") { + return sv.Value, nil + } + // Some sysvars in TiDB have a special behavior where the empty string means + // "use the config file value". This needs to be cleaned up once the behavior + // for instance variables is determined. + if value == "" && ((sv.AllowEmpty && scope == ScopeSession) || sv.AllowEmptyAll) { + return value, nil + } + // Provide validation using the SysVar struct + switch sv.Type { + case TypeUnsigned: + return sv.checkUInt64SystemVar(value, vars) + case TypeInt: + return sv.checkInt64SystemVar(value, vars) + case TypeBool: + return sv.checkBoolSystemVar(value, vars) + case TypeFloat: + return sv.checkFloatSystemVar(value, vars) + case TypeEnum: + return sv.checkEnumSystemVar(value, vars) + case TypeTime: + return sv.checkTimeSystemVar(value, vars) + case TypeDuration: + return sv.checkDurationSystemVar(value, vars) + } + return value, nil // typeString +} + +func (sv *SysVar) validateScope(scope ScopeFlag) error { + if sv.ReadOnly || sv.Scope == ScopeNone { + return ErrIncorrectScope.FastGenByArgs(sv.Name, "read only") + } + if scope == ScopeGlobal && !sv.HasGlobalScope() { + return errLocalVariable.FastGenByArgs(sv.Name) + } + if scope == ScopeSession && !sv.HasSessionScope() { + return errGlobalVariable.FastGenByArgs(sv.Name) + } + return nil +} + +// ValidateWithRelaxedValidation normalizes values but can not return errors. +// Normalization+validation needs to be applied when reading values because older versions of TiDB +// may be less sophisticated in normalizing values. But errors should be caught and handled, +// because otherwise there will be upgrade issues. +func (sv *SysVar) ValidateWithRelaxedValidation(vars *SessionVars, value string, scope ScopeFlag) string { + normalizedValue, err := sv.validateFromType(vars, value, scope) + if err != nil { + return normalizedValue + } + if sv.Validation != nil { + normalizedValue, err = sv.Validation(vars, normalizedValue, value, scope) + if err != nil { + return normalizedValue + } + } + return normalizedValue +} + +const ( + localDayTimeFormat = "15:04" + // FullDayTimeFormat is the full format of analyze start time and end time. + FullDayTimeFormat = "15:04 -0700" +) + +func (sv *SysVar) checkTimeSystemVar(value string, vars *SessionVars) (string, error) { + var t time.Time + var err error + if len(value) <= len(localDayTimeFormat) { + t, err = time.ParseInLocation(localDayTimeFormat, value, vars.TimeZone) + } else { + t, err = time.ParseInLocation(FullDayTimeFormat, value, vars.TimeZone) + } + if err != nil { + return "", err + } + return t.Format(FullDayTimeFormat), nil +} + +func (sv *SysVar) checkDurationSystemVar(value string, vars *SessionVars) (string, error) { + d, err := time.ParseDuration(value) + if err != nil { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + // Check for min/max violations + if int64(d) < sv.MinValue { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + if uint64(d) > sv.MaxValue { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + // return a string representation of the duration + return d.String(), nil +} + +func (sv *SysVar) checkUInt64SystemVar(value string, vars *SessionVars) (string, error) { + if sv.AllowAutoValue && value == "-1" { + return value, nil + } + // There are two types of validation behaviors for integer values. The default + // is to return an error saying the value is out of range. For MySQL compatibility, some + // values prefer convert the value to the min/max and return a warning. + if !sv.AutoConvertOutOfRange { + return sv.checkUint64SystemVarWithError(value) + } + if len(value) == 0 { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + if value[0] == '-' { + _, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + vars.StmtCtx.AppendWarning(ErrTruncatedWrongValue.GenWithStackByArgs(sv.Name, value)) + return fmt.Sprintf("%d", sv.MinValue), nil + } + val, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return value, ErrWrongTypeForVar.GenWithStackByArgs(sv.Name) + } + if val < uint64(sv.MinValue) { + vars.StmtCtx.AppendWarning(ErrTruncatedWrongValue.GenWithStackByArgs(sv.Name, value)) + return fmt.Sprintf("%d", sv.MinValue), nil + } + if val > sv.MaxValue { + vars.StmtCtx.AppendWarning(ErrTruncatedWrongValue.GenWithStackByArgs(sv.Name, value)) + return fmt.Sprintf("%d", sv.MaxValue), nil + } + return value, nil +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) } // SysVars is global sys vars map. @@ -691,6 +947,7 @@ var defaultSysVars = []*SysVar{ {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableRateLimitAction, Value: boolToOnOff(DefTiDBEnableRateLimitAction)}, +<<<<<<< HEAD /* The following variable is defined as session scope but is actually server scope. */ {ScopeSession, TiDBGeneralLog, strconv.Itoa(DefTiDBGeneralLog)}, {ScopeSession, TiDBPProfSQLCPU, strconv.Itoa(DefTiDBPProfSQLCPU)}, @@ -744,6 +1001,17 @@ var defaultSysVars = []*SysVar{ {ScopeGlobal, TiDBEnableTelemetry, BoolToIntStr(DefTiDBEnableTelemetry)}, {ScopeGlobal | ScopeSession, TiDBEnableAmendPessimisticTxn, boolToOnOff(DefTiDBEnableAmendPessimisticTxn)}, {ScopeGlobal | ScopeSession, TiDBMultiStatementMode, Off}, +======= +// GetSysVars deep copies the sysVars list under a RWLock +func GetSysVars() map[string]*SysVar { + sysVarsLock.RLock() + defer sysVarsLock.RUnlock() + copy := make(map[string]*SysVar, len(sysVars)) + for name, sv := range sysVars { + copy[name] = sv + } + return copy +>>>>>>> 0f10bef47... domain, session: Add new sysvarcache to replace global values cache (#24359) } // SynonymsSysVariables is synonyms of system variables.