cherry-pick upstream release 7.1 (pingcap#829)

* lightning: fix lightning failed to log encoding error (pingcap#45241) (pingcap#45365) close pingcap#44321 * *: Fix data race between getting labels and setting labels in config (pingcap#45563) (pingcap#45568) close pingcap#45561 * store: Don't set limit for GetRegionsInfoByRange function (pingcap#45556) (pingcap#45571) close pingcap#45531 * parser: support more join hints on parser (pingcap#45525) (pingcap#45574) ref pingcap#45520 * *: Fix data race in TestSetLabelsConcurrentWithGetLabel (pingcap#45579) (pingcap#45590) close pingcap#45578 * planner: support `no_hash_join` hint on optimizer (pingcap#45538) (pingcap#45635) ref pingcap#45520 * planner: support `no_merge_join` hint on optimizer (pingcap#45562) (pingcap#45705) ref pingcap#45520 * planner: refine `tryToGetIndexJoin` (pingcap#45587) (pingcap#45713) ref pingcap#45520 * ddl: create temp dir automatically for adding index (pingcap#45457) (pingcap#45475) close pingcap#45456 * statistics: kill in time when to mergeGlobalTopN with partition (pingcap#45707) (pingcap#45709) close pingcap#45706 * executor, util: make tmp-storage-quota take affect (pingcap#45549) (pingcap#45732) close pingcap#45161 * planner: refactor `tryToGetIndexJoin` (pingcap#45617) (pingcap#45841) ref pingcap#45520 * lightning: fix routes panic for csv data load (pingcap#45405) (pingcap#45537) close pingcap#43284 * planner: support `no_index_join`, `no_index_hash_join`, `no_index_merge_join` hints (pingcap#45633) (pingcap#45882) ref pingcap#45520 * parser: reset `lexer.identifierDot` correctly when reusing parsers (pingcap#45903) (pingcap#45910) close pingcap#45898 * executor, planner: add extraProj for indexLookUp with order by + limit + static prune (pingcap#45771) close pingcap#45757 * statistics: remove useless code in the MergePartTopN2GlobalTopN (pingcap#45755) (pingcap#45769) close pingcap#45706 * statistics: kill in time when to run topnStatsMergeWorker and improve memory (pingcap#45765) (pingcap#45768) close pingcap#45706 * snap_restore: resend `recover_region` while there are TiKV restarts (pingcap#45361) (pingcap#45722) close pingcap#45206 * statistics: reduce memory usage when to MergePartTopN2GlobalTopN (pingcap#45718) (pingcap#45971) close pingcap#45727 * calibrate: shorten the minimum calibrate duration (pingcap#44972) (pingcap#46007) ref pingcap#43212, close pingcap#46006 * *: set go1.20.7 (pingcap#46027) close pingcap#46026 * ttl: make ttl manually trigger stable (pingcap#45869) (pingcap#45896) close pingcap#45868 * statistics: add bench for MergeGlobalStatsTopNByConcurrency (pingcap#45998) (pingcap#46031) ref pingcap#45281 * statistics: improve memory for mergeGlobalStatsTopNByConcurrency (pingcap#45993) (pingcap#46058) close pingcap#45727 * ttl: fix some memory leak in TTL (pingcap#45512) (pingcap#45514) close pingcap#45510 * lightning: make OpLevelOptional suppress the error of DoChecksum (pingcap#45486) (pingcap#45867) close pingcap#45382 * expression: Fix different behaviors with MySQL when comparing datetime column with numeric constant | tidb-test=pr/2198 (pingcap#46129) close pingcap#38361 * br: make download metadata concurrency adjustable (pingcap#45639) (pingcap#45845) close pingcap#45511 * executor: Fix coroutine leak after kill remote connection (pingcap#46035) (pingcap#46067) close pingcap#46034 * br: Fix RangeTree.GetIncompleteRange when the rangeTree is empty. (pingcap#37086) (pingcap#46168) * *: modify headers of some source files in `privilege` (pingcap#46222) (pingcap#46236) * ddl: use the correct timezone to encode record for adding index (pingcap#46055) (pingcap#46103) close pingcap#46033 * parser: fix parser identifier with dot (pingcap#46314) (pingcap#46323) close pingcap#35031 * lightning: "no leader" should be a retryable error (pingcap#46300) (pingcap#46317) close pingcap#45673 * lightning: save NEXT_GLOBAL_ROW_ID as uint64 instead of int64 (pingcap#45614) (pingcap#45940) close pingcap#45427 * executor: optimize load data assignment expression (pingcap#46563) close pingcap#46081 * br: fix restore metakv without default cf files (pingcap#46589) (pingcap#46643) close pingcap#46578 * autoid_service,owner: change autoid service's etcd lease to 10s (pingcap#46455) (pingcap#46653) close pingcap#46454 * executor: fix hang when index join newInnerworker panic (pingcap#45890) (pingcap#46114) close pingcap#45716 * br: adjust default tidb global config for br (pingcap#45794) (pingcap#46192) close pingcap#45793 * br: configure the httpclient for external storage (pingcap#46040) (pingcap#46141) close pingcap#46011 * planner: refactor Join and Limit's ResolveIndices (pingcap#45831) (pingcap#45864) close pingcap#45758, close pingcap#45805 * expression: fix wrong result for unsigned non-const int cmp const duration (pingcap#46620) (pingcap#46703) close pingcap#45410 * executor: fix batchPoint for partition table with duplicate rows (pingcap#46808) (pingcap#46885) close pingcap#46779 * statistics: add historgram bench for MergePartitionHist2GlobalHist (pingcap#46788) (pingcap#47038) close pingcap#46805 * br: not use the custom http client for gcs (pingcap#47026) (pingcap#47058) close pingcap#47022 * planner: don't recompute the hashcode when generated column substitution doesn't happen (pingcap#46450) (pingcap#46630) close pingcap#42788 * *: add bench for analyze (pingcap#46898) (pingcap#47249) close pingcap#46914 * hint: fix panic when query block not found in prepare statement (pingcap#46818) (pingcap#46843) close pingcap#46817 * br: fix unstable test br_300_small_tables (pingcap#46560) (pingcap#46568) close pingcap#46561 * executor: fix plan replayer for sql file input wrongly fetch startTS after `OnTxnEnd` (pingcap#46201) (pingcap#46345) close pingcap#46197 * statistics, planner: use the correct `IndexInfo` in `GenerateHistCollFromColumnInfo()` | tidb-test=pr/2218 (pingcap#44441) (pingcap#46730) close pingcap#44369 * lightning: fix incorrect _tidb_rowid allocator value after import for table with AUTO_ID_CACHE=1 (pingcap#46171) (pingcap#46184) close pingcap#46100 * br: check the correct changefeed info when restore/import data (pingcap#47322) (pingcap#47370) close pingcap/tiflow#9807 * *: fix the duplicate entry error when using BR to restore a NONCLUSTERED AUTO_ID_CACHE=1 table (pingcap#46127) (pingcap#46338) close pingcap#46093 * br: wait tiflash replicas ready && fix unstable test (pingcap#46301) (pingcap#46341) close pingcap#46302 * ddl, session: using table ID instead of partition ID when calling `SplitRegions` (pingcap#46156) (pingcap#46161) close pingcap#46135 * util: change log level to debug calculating stale read ts (pingcap#47309) (pingcap#47453) close pingcap#47308 * ddl: fix DST times for adding index (pingcap#47425) (pingcap#47448) ref pingcap#46033, close pingcap#47426 * ddl: Exchange partition rollback (pingcap#45877) (pingcap#45981) close pingcap#45791, close pingcap#45920 * br: avoid possible infinite loop checking tikv gc and scheduling status (pingcap#46078) (pingcap#46196) close pingcap#46073 * br: fix the split issue in txn restore (pingcap#45441) (pingcap#45548) close pingcap#45476 * *: Exchange partition, fix LIST COLUMNs validation as well as NULL validation (pingcap#46533) (pingcap#46697) close pingcap#46492 * domain: use dedicated lock for expiredTimeStamp4PC (pingcap#45802) (pingcap#47519) close pingcap#45400 * autoid_service: fix potential 'duplicated entry' error when tidb exit for AUTO_ID_CACHE=1 tables (pingcap#46445) (pingcap#46500) close pingcap#46444 * domain: fix a bug when reloading take a long time in domain.Init (pingcap#45170) (pingcap#45391) close pingcap#45176 * autoid_service: add some debug log for rebase related operation (pingcap#46526) (pingcap#46530) ref pingcap#46444 * ddl: relax the check in ownerCheckAllVersions (pingcap#46752) (pingcap#46794) close pingcap#46751 * planner: clean the one side filters for outer join reorder which is not done in pingcap#44409 (pingcap#45565) (pingcap#45586) ref pingcap#44409, close pingcap#45564 * planner: fix the issue that TIDB_INLJ hint cannot take effect when left joining two sub-queries (pingcap#46271) (pingcap#46291) close pingcap#46160 * planner: DML queries might panic when CTE exists (pingcap#46092) (pingcap#46097) close pingcap#46083 * snap_restore: added retry for recovery (pingcap#46094) (pingcap#46225) close pingcap#46181 * planner: fix Uncertain Results caused by MERGE_JOIN (pingcap#47078) (pingcap#47081) close pingcap#46580 * br: fix add ingest index as sub job (pingcap#47484) (pingcap#47556) close pingcap#47482 * planner: Adjust the log level and returned value when `cacheableChecker` check `*ast.TableName` nodes (pingcap#46831) (pingcap#46877) close pingcap#46760 * planner: disallow split clustered index with SPLIT TABLE .. INDEX (pingcap#47412) (pingcap#47436) close pingcap#47350 * planner: fix cast(col) = range couldn't build range when cast function doesn't contain any precision loss in some cases | tidb-test=pr/2224 (pingcap#46303) (pingcap#46548) close pingcap#45199 * planner: fix group_concat function couldn't resolve the index of its order-by item (pingcap#46419) (pingcap#47121) close pingcap#41986 * planner: adjust N used in TopN cost formula based on the total number of rows (pingcap#46368) (pingcap#46571) close pingcap#43285 * planner: increase the maximum number limit of TopN when analyzing tables (pingcap#45962) (pingcap#45973) ref pingcap#45919 * ddl: fix a bug that MDL may progress unexpectedly or block forever (pingcap#46921) (pingcap#47069) close pingcap#46920 * ddl: fix duplicate inforSchema information of rename tables (pingcap#47087) (pingcap#47142) close pingcap#47064 * OWNERS: Auto Sync OWNERS files from community membership (pingcap#47615) * planner: don't force set the Column in physical proj elimination (pingcap#45824) (pingcap#46132) close pingcap#45804 * *: update dependency for uber-go/automaxprocs (pingcap#47605) close pingcap#47601 * planner: store the hints of session variable (pingcap#45814) (pingcap#46047) close pingcap#45812 * planner: consider Unspecified length when setting length for unhex (pingcap#45449) (pingcap#45451) close pingcap#45378 * planner: fix index heuristic rule will prune out hint preferred tiflash path (pingcap#46102) (pingcap#46145) close pingcap#40146 * planner: fix group concat projection elimination logic (pingcap#47612) (pingcap#47628) close pingcap#41957 * planner: log the reason why the sample-rate is chosen when analyzing table (pingcap#45938) (pingcap#45968) close pingcap#45936 * session: set request source to type of the prepared stmt for execute stmt (pingcap#46348) (pingcap#46354) close pingcap#46349 * *: add MVCC version ratio to slow log metrics (pingcap#44897) (pingcap#47645) close pingcap#44896 * extension: fix enterprise git commit hash display in `tidb-server -V` (pingcap#47647) (pingcap#47652) close pingcap#47646 * executor: Fix crash during sort spill (pingcap#47581) (pingcap#47625) close pingcap#47538 * binlog: fix the wrong fatal error when binlog is too large (pingcap#46365) (pingcap#46380) close pingcap#28659 * dumpling: fix dumpling ignore file writer close error (pingcap#45374) (pingcap#45404) close pingcap#45353 * planner: output a warning if plan rebuilding fails when reusing a cached plan (pingcap#46278) (pingcap#46281) close pingcap#46159 * executor, planner: fix plan_replayer zip format (pingcap#47474) (pingcap#47713) close pingcap#46474 * statistics: check Killed in the TableStatsFromStorage (pingcap#47568) (pingcap#47641) close pingcap#47570 * lightning/backend/local: fix `buildIndexDupTasks` (pingcap#44442) (pingcap#47894) close pingcap#44439 * executor: fix an issue that index_lookup_hash_join incorrectly estimate memory usage. (pingcap#47795) (pingcap#47925) close pingcap#47788 * br: avoid retry if it's ec2metadata (pingcap#47651) (pingcap#47790) close pingcap#47650 * br: add integration test for pitr (pingcap#47740) (pingcap#47906) ref pingcap#47738 * planner: do not convert update to point get if the expr has sub-query (pingcap#47454) (pingcap#47735) close pingcap#47445 * statistic: fix panic when building topN (pingcap#47928) (pingcap#47990) close pingcap#35948 * local backend: fix log been eaten when failed on execute region job (pingcap#47782) (pingcap#47814) close pingcap#47781 * local backend: fix worker err overriden by job generation err (pingcap#48185) (pingcap#48198) close pingcap#47992 * executor: reuse chunk for GetMatchedRowsAndPtrs calls (pingcap#48090) (pingcap#48105) close pingcap#48082 * pdutil/backend: enlarge max retry time and fix nested retriable error (pingcap#48210) (pingcap#48227) close pingcap#46950 * lightning: every HTTP retry should use its own request (pingcap#47959) (pingcap#47968) close pingcap#47930 * statistic: improve ipv6 for analyze job meta (pingcap#45177) (pingcap#45185) ref pingcap#43260 * statistics: check Killed in the GenJSONTableFromStats (pingcap#47778) (pingcap#47824) close pingcap#47779 * util/cgroup: fix InContainer for cgroup v2 (pingcap#48344) (pingcap#48345) close pingcap#48342 * *: upgrade 1.20.11 (pingcap#48385) ref pingcap#48383 * resource_control: add duration buffer for calibrate time window (pingcap#48172) (pingcap#48353) close pingcap#48162 * statistics: avoid oom when to gc large stats_history (pingcap#48430) (pingcap#48489) close pingcap#48431 * planner, CTE: Fix default inline CTE which contains agg or window function and refactor inline CTE strategy (pingcap#48436) close pingcap#47711 * copr: fix copr cache panic when `tidb_enable_collect_execution_info` is off (pingcap#48340) (pingcap#48475) close pingcap#48212 * executor: AnalyzePartitionConcurrency is not more than number of task (pingcap#47463) (pingcap#48547) close pingcap#47464 * domain: randomly trigger the DumpStatsDeltaToKV (pingcap#48692) (pingcap#48704) close pingcap#48693 * ddl: fix issue of alter last partition failed when partition column is datetime (pingcap#48815) (pingcap#48847) close pingcap#48814 * planner: fix nil pointer at expression.(*CorrelatedColumn).Eval (pingcap#42789) (pingcap#48977) close pingcap#42739 * *: handle region error for GetMvccByEncodedKey API (pingcap#47811) (pingcap#47822) close pingcap#47807 * pkg/util: refine tidb_server_memory_limit to make the cpu usage more stable (pingcap#48927) (pingcap#49061) close pingcap#48741 * session: fix select for update statement can't get stmt-count-limit error (pingcap#48412) (pingcap#48468) close pingcap#48411 * domain: add resolve lock logic for mvcc get key loading schema diff (pingcap#48330) (pingcap#48417) close pingcap#48281 * *: exclude `**/.git/**` in licenserc.yml (pingcap#49157) (pingcap#49158) close pingcap#49156 * br: add more retryable error for br_import (pingcap#48529) (pingcap#48543) close pingcap#48528 * extension: update submodule for release-7.1 (pingcap#49175) close pingcap#49133 * executor: fix index out of range panic of cte when max_chunk_size is samll (pingcap#48839) (pingcap#49004) close pingcap#48808 * executor: fix deadlock in dml statement with cte when oom panic action was triggered (pingcap#49192) (pingcap#49202) close pingcap#49096 * errmsg: fix example s3 url in error msg (pingcap#49207) (pingcap#49208) close pingcap#49196 * *: support concurrent write for S3 writer (pingcap#45723) (pingcap#49185) ref pingcap#45719, close pingcap#48607 * planner: fixing wrong result after applying predicate push down for CTEs (pingcap#47891) (pingcap#48193) close pingcap#47881 * util/ranger: don't exclude start key for range from `_` in `like` function | tidb-test=pr/2258 (pingcap#48984) (pingcap#49065) ref pingcap#48181, close pingcap#48983 * br: retry to scatter the regions if status is timeout or cancel (pingcap#46471) (pingcap#48378) close pingcap#47236 * planner/core: keep sort operator when ordered by tablesample (pingcap#48315) (pingcap#48450) close pingcap#48253 * s3: fix s3 concurrent uploader will overwrite error (pingcap#48163) (pingcap#49221) close pingcap#48164 * lightning: add timeout for "write" RPC (pingcap#48355) (pingcap#48397) close pingcap#46321, close pingcap#48352 * *: fix data race of Column.GeneratedExpr (pingcap#48888) (pingcap#48923) close pingcap#44919, close pingcap#48191 * planner: Fix the issue that may generate many plandigests when the inner table is clustered (pingcap#47952) (pingcap#48112) close pingcap#47634 * planner: fix the issue that the optimizer terminates the optimization process for `DataSource` too early (pingcap#48186) (pingcap#48266) close pingcap#46177 * executor/brie: use the default value from flags (pingcap#48025) (pingcap#48442) close pingcap#48000 * planner: fix possible inconsistent output cols among union's children (pingcap#48775) (pingcap#48895) close pingcap#48755 * executor, codec: hash join build wrong hash key for ENUM/SET value (pingcap#49031) (pingcap#49055) close pingcap#48991 * br: fix incorrect uri for the file storage (pingcap#48453) (pingcap#48722) close pingcap#48452 * BR: Support encryption for restored ebs volumes (pingcap#48900) (pingcap#48937) close pingcap#48899 * plugin: fix bug that watch loop will refresh frequently when channel closed (pingcap#49275) (pingcap#49289) close pingcap#49273 * planner: fix a panic during column pruning (pingcap#47883) (pingcap#48809) close pingcap#47331 * ddl: fix recover table by JobID bug when JobID is set to 0 tidb-server panic (pingcap#46343) (pingcap#48086) close pingcap#46296 * pkg/lightning : remove get_regions call in physical backend (pingcap#46202) (pingcap#46337) close pingcap#45507 * planner: check nil pointer in rule_result_reorder to avoid panic (pingcap#48100) (pingcap#48116) close pingcap#45044 * planner: plan cache supports Batch/PointGet converted from (primary keys) in ((...), ...) (pingcap#44838) (pingcap#49380) close pingcap#44830 * executor: fill correlated column value in late materialization filter conditions (pingcap#49244) (pingcap#49431) close pingcap#49241 * executor: set OverflowAsWarning for insert statement in non-strict sql mode (pingcap#49383) (pingcap#49459) close pingcap#49369 * planner: fix tiflash path will be identified as table dual (pingcap#49338) (pingcap#49395) close pingcap#49285 * expression: enum/set could be invalid during evaluation (pingcap#49543) (pingcap#49550) close pingcap#49487 * pkg/executor: fix the hang issue in indexHashJoin (pingcap#49218) (pingcap#49411) close pingcap#49033 * cherry-pick: mannual cherry-pick 49421 back to release 7.1 (pingcap#49592) close pingcap#49377 * executor: use EncodeKey to encode groupkey in StreamAggExec (pingcap#49974) (pingcap#50026) close pingcap#49902 * executor: avoid concurrently update for `SimpleLRUCache` in `ApplyCache` (pingcap#50356) (pingcap#50385) close pingcap#50347 * fix ut Signed-off-by: zzm <zhouzemin@pingcap.com> * fix ut Signed-off-by: zzm <zhouzemin@pingcap.com> * tag this branch image as v7.1 for e2e test Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 79a02af Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick a52c366 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 3652a93 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick d121f46 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick a1ea04c Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick e07cf40 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick e5004d8 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick fc4edaf Signed-off-by: zzm <zhouzemin@pingcap.com> * planner: fix issue 48643 that aggDesc modification will change the referrence (pingcap#48662) close pingcap#48643 * cherry-pick 72e5323 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick eac9cd0 Signed-off-by: zzm <zhouzemin@pingcap.com> * resource_control: fetch cpu quota metrics from store instead of prometheus (pingcap#49176) (pingcap#49255) close pingcap#49174 * cherry-pick 6b72ccd Signed-off-by: zzm <zhouzemin@pingcap.com> * pkg: support the TSO format for asof expression (pingcap#48371) (pingcap#48380) close pingcap#48372 * ebs br: control the snapshots batch size for fsr enable/disable (pingcap#48506) (pingcap#48526) close pingcap#48505 * cherry-pick eacc493 Signed-off-by: zzm <zhouzemin@pingcap.com> * ebs br: make sure fsr credit is full filled (pingcap#48627) (pingcap#48743) close pingcap#48629 * cherry-pick 950521e Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 36c74e6 Signed-off-by: zzm <zhouzemin@pingcap.com> * planner: fix leading hint cannot take effect in UNION ALL statements (pingcap#50277) (pingcap#50322) close pingcap#50067 * make check Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 38fb0eb Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 0312e78 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 0d9903c Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 359aea8 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick a56af1b Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 536545d Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 9da9d4d Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 14a07f2 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 126cf52 Signed-off-by: zzm <zhouzemin@pingcap.com> * make check Signed-off-by: zzm <zhouzemin@pingcap.com> * update kvproto Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick b003558 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick acdac74 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 49e9daf Signed-off-by: zzm <zhouzemin@pingcap.com> * pdutil: fix retry reusing body reader (pingcap#48312) (pingcap#48320) close pingcap#48307 * cherry-pick f249304 Signed-off-by: zzm <zhouzemin@pingcap.com> * cherry-pick 575a4e6 Signed-off-by: zzm <zhouzemin@pingcap.com> * parser: fix limit lost in SetOprStmt (pingcap#49898) (pingcap#49931) close pingcap#49874 * build br/lightning Signed-off-by: zzm <zhouzemin@pingcap.com> * fix lightning panic Signed-off-by: zzm <zhouzemin@pingcap.com> * make check Signed-off-by: zzm <zhouzemin@pingcap.com> * fix build br Signed-off-by: zzm <zhouzemin@pingcap.com> * revert cd Signed-off-by: zzm <zhouzemin@pingcap.com> --------- Signed-off-by: zzm <zhouzemin@pingcap.com> Co-authored-by: Ti Chi Robot <ti-community-prow-bot@tidb.io> Co-authored-by: Hangjie Mo <mohangjie1995@gmail.com> Co-authored-by: Weizhen Wang <wangweizhen@pingcap.com> Co-authored-by: yibin <huyibin@pingcap.com> Co-authored-by: lance6716 <lance6716@gmail.com> Co-authored-by: 王超 <cclcwangchao@hotmail.com> Co-authored-by: Yuanjia Zhang <zhangyuanjia@pingcap.com> Co-authored-by: Arenatlx <314806019@qq.com> Co-authored-by: Lynn <zimu_xia@126.com> Co-authored-by: crazycs <crazycs520@gmail.com> Co-authored-by: ShuNing <nolouch@gmail.com> Co-authored-by: BornChanger <97348524+BornChanger@users.noreply.github.com>
rleungx · Feb 22, 2024 · b5415fc · b5415fc
1 parent 5887f60
commit b5415fc
Show file tree

Hide file tree

Showing 486 changed files with 25,930 additions and 14,500 deletions.
diff --git a/.github/licenserc.yml b/.github/licenserc.yml
@@ -27,6 +27,7 @@ header:
     - "**/*.example"
     - "**/*.patch"
     - "**/*.bzl"
+    - "**/.git/**"
     - ".codecov.yml"
     - "Jenkinsfile"
     - ".editorconfig"

diff --git a/DEPS.bzl b/DEPS.bzl
@@ -3578,8 +3578,8 @@ def go_deps():
         name = "com_github_pingcap_errors",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/errors",
-        sum = "h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8=",
-        version = "v0.11.5-0.20221009092201-b66cddb77c32",
+        sum = "h1:yqyP+k0mgRPpXJQDOCrtaG2YZym0ZDD+vt5JzlBUkrw=",
+        version = "v0.11.5-0.20231212100244-799fae176cfb",
     )
     go_repository(
         name = "com_github_pingcap_failpoint",
@@ -3606,8 +3606,8 @@ def go_deps():
         name = "com_github_pingcap_kvproto",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/pingcap/kvproto",
-        sum = "h1:qbLm5cOdCWxZ0mt6SaN2aXI+KFekbPqURd6YkNI+XRI=",
-        version = "v0.0.0-20230904082117-ecdbf1f8c130",
+        sum = "h1:n3FMveYjc2VuETjo6YhmsgkDx0P/yLJTvk96BJdCq6Y=",
+        version = "v0.0.0-20240109063850-932639606bcf",
     )
     go_repository(
         name = "com_github_pingcap_log",
@@ -3707,6 +3707,13 @@ def go_deps():
         sum = "h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig=",
         version = "v0.0.0-20221212215047-62379fc7944b",
     )
+    go_repository(
+        name = "com_github_prashantv_gostub",
+        build_file_proto_mode = "disable",
+        importpath = "github.com/prashantv/gostub",
+        sum = "h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=",
+        version = "v1.1.0",
+    )
     go_repository(
         name = "com_github_prometheus_client_golang",
         build_file_proto_mode = "disable_global",

diff --git a/Makefile b/Makefile
@@ -358,6 +358,7 @@ build_for_br_integration_test:
 	$(GOBUILD) $(RACE_FLAG) -o bin/gc br/tests/br_z_gc_safepoint/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/oauth br/tests/br_gcs/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/rawkv br/tests/br_rawkv/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/txnkv br/tests/br_txn/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/parquet_gen br/tests/lightning_checkpoint_parquet/*.go \
 	) || (make failpoint-disable && exit 1)
 	@make failpoint-disable

diff --git a/Makefile.common b/Makefile.common
@@ -67,7 +67,7 @@ LDFLAGS += -X "github.com/pingcap/tidb/util/versioninfo.TiDBGitBranch=$(shell gi
 LDFLAGS += -X "github.com/pingcap/tidb/util/versioninfo.TiDBEdition=$(TIDB_EDITION)"
 
 EXTENSION_FLAG =
-ifeq ($(shell if [ -d extension/enterprise/.git ]; then echo "true"; fi),true)
+ifeq ($(shell if [ -f extension/enterprise/.git ]; then echo "true"; fi),true)
 	EXTENSION_FLAG += -X "github.com/pingcap/tidb/util/versioninfo.TiDBEnterpriseExtensionGitHash=$(shell cd extension/enterprise && git rev-parse HEAD)"
 endif
 

diff --git a/OWNERS b/OWNERS
@@ -11,6 +11,7 @@ approvers:
   - asddongmen
   - b41sh
   - bb7133
+  - Benjamin2037
   - breezewish
   - buchuitoudegou
   - c4pt0r
@@ -111,9 +112,9 @@ approvers:
 reviewers:
   - aytrack
   - ben1009
-  - Benjamin2037
   - bobotu
   - BornChanger
+  - CabinfeverB
   - charleszheng44
   - ChenPeng2013
   - dhysum

diff --git a/WORKSPACE b/WORKSPACE
@@ -49,7 +49,7 @@ go_download_sdk(
         "https://mirrors.aliyun.com/golang/{}",
         "https://dl.google.com/go/{}",
     ],
-    version = "1.20.6",
+    version = "1.20.11",
 )
 
 go_register_toolchains(

diff --git a/autoid_service/autoid.go b/autoid_service/autoid.go
@@ -190,6 +190,7 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 	}
 
 	var newBase, newEnd uint64
+	var oldValue int64
 	startTime := time.Now()
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
@@ -198,6 +199,7 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		uCurrentEnd := uint64(currentEnd)
 		newBase = mathutil.Max(uCurrentEnd, requiredBase)
 		newEnd = mathutil.Min(math.MaxUint64-uint64(batch), newBase) + uint64(batch)
@@ -208,6 +210,13 @@ func (alloc *autoIDValue) rebase4Unsigned(ctx context.Context,
 	if err != nil {
 		return err
 	}
+
+	logutil.BgLogger().Info("rebase4Unsigned from",
+		zap.String("category", "autoid service"),
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Uint64("to", newEnd))
 	alloc.base, alloc.end = int64(newBase), int64(newEnd)
 	return nil
 }
@@ -223,22 +232,32 @@ func (alloc *autoIDValue) rebase4Signed(ctx context.Context, store kv.Storage, d
 		return nil
 	}
 
-	var newBase, newEnd int64
+	var oldValue, newBase, newEnd int64
+	startTime := time.Now()
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
 		idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
 		currentEnd, err1 := idAcc.Get()
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		newBase = mathutil.Max(currentEnd, requiredBase)
 		newEnd = mathutil.Min(math.MaxInt64-batch, newBase) + batch
 		_, err1 = idAcc.Inc(newEnd - currentEnd)
 		return err1
 	})
+	metrics.AutoIDHistogram.WithLabelValues(metrics.TableAutoIDRebase, metrics.RetLabel(err)).Observe(time.Since(startTime).Seconds())
 	if err != nil {
 		return err
 	}
+
+	logutil.BgLogger().Info("rebase4Signed from",
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Int64("to", newEnd),
+		zap.String("category", "autoid service"))
 	alloc.base, alloc.end = newBase, newEnd
 	return nil
 }
@@ -284,7 +303,13 @@ func New(selfAddr string, etcdAddr []string, store kv.Storage, tlsConfig *tls.Co
 
 func newWithCli(selfAddr string, cli *clientv3.Client, store kv.Storage) *Service {
 	l := owner.NewOwnerManager(context.Background(), cli, "autoid", selfAddr, autoIDLeaderPath)
-	err := l.CampaignOwner()
+	l.SetBeOwnerHook(func() {
+		logutil.BgLogger().Info("leader change of autoid service, this node become owner",
+			zap.String("addr", selfAddr),
+			zap.String("category", "autoid service"))
+	})
+	// 10 means that autoid service's etcd lease is 10s.
+	err := l.CampaignOwner(10)
 	if err != nil {
 		panic(err)
 	}
@@ -329,7 +354,7 @@ func MockForTest(store kv.Storage) autoid.AutoIDAllocClient {
 
 // Close closes the Service and clean up resource.
 func (s *Service) Close() {
-	if s.leaderShip != nil {
+	if s.leaderShip != nil && s.leaderShip.IsOwner() {
 		for k, v := range s.autoIDMap {
 			if v.base > 0 {
 				err := v.forceRebase(context.Background(), s.store, k.dbID, k.tblID, v.base, v.isUnsigned)
@@ -483,12 +508,14 @@ func (s *Service) allocAutoID(ctx context.Context, req *autoid.AutoIDRequest) (*
 
 func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbID, tblID, requiredBase int64, isUnsigned bool) error {
 	ctx = kv.WithInternalSourceType(ctx, kv.InternalTxnMeta)
+	var oldValue int64
 	err := kv.RunInNewTxn(ctx, store, true, func(ctx context.Context, txn kv.Transaction) error {
 		idAcc := meta.NewMeta(txn).GetAutoIDAccessors(dbID, tblID).IncrementID(model.TableInfoVersion5)
 		currentEnd, err1 := idAcc.Get()
 		if err1 != nil {
 			return err1
 		}
+		oldValue = currentEnd
 		var step int64
 		if !isUnsigned {
 			step = requiredBase - currentEnd
@@ -502,6 +529,13 @@ func (alloc *autoIDValue) forceRebase(ctx context.Context, store kv.Storage, dbI
 	if err != nil {
 		return err
 	}
+	logutil.BgLogger().Info("forceRebase from",
+		zap.Int64("dbID", dbID),
+		zap.Int64("tblID", tblID),
+		zap.Int64("from", oldValue),
+		zap.Int64("to", requiredBase),
+		zap.Bool("isUnsigned", isUnsigned),
+		zap.String("category", "autoid service"))
 	alloc.base, alloc.end = requiredBase, requiredBase
 	return nil
 }

diff --git a/br/cmd/br/main.go b/br/cmd/br/main.go
@@ -49,7 +49,7 @@ func main() {
 		NewBackupCommand(),
 		NewRestoreCommand(),
 		NewStreamCommand(),
-		newOpeartorCommand(),
+		newOperatorCommand(),
 	)
 	// Outputs cmd.Print to stdout.
 	rootCmd.SetOut(os.Stdout)

diff --git a/br/cmd/br/operator.go b/br/cmd/br/operator.go
@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func newOpeartorCommand() *cobra.Command {
+func newOperatorCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "operator <subcommand>",
 		Short: "utilities for operators like tidb-operator.",
@@ -26,24 +26,29 @@ func newOpeartorCommand() *cobra.Command {
 		},
 		Hidden: true,
 	}
-	cmd.AddCommand(newPauseGcCommand())
+	cmd.AddCommand(newPrepareForSnapshotBackupCommand(
+		"pause-gc-and-schedulers",
+		"(Will be replaced with `prepare-for-snapshot-backup`) pause gc, schedulers and importing until the program exits."))
+	cmd.AddCommand(newPrepareForSnapshotBackupCommand(
+		"prepare-for-snapshot-backup",
+		"pause gc, schedulers and importing until the program exits, for snapshot backup."))
 	return cmd
 }
 
-func newPauseGcCommand() *cobra.Command {
+func newPrepareForSnapshotBackupCommand(use string, short string) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "pause-gc-and-schedulers",
-		Short: "pause gc and schedulers to the ts until the program exits.",
+		Use:   use,
+		Short: short,
 		Args:  cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg := operator.PauseGcConfig{}
 			if err := cfg.ParseFromFlags(cmd.Flags()); err != nil {
 				return err
 			}
 			ctx := GetDefaultContext()
-			return operator.PauseGCAndScheduler(ctx, &cfg)
+			return operator.AdaptEnvForSnapshotBackup(ctx, &cfg)
 		},
 	}
-	operator.DefineFlagsForPauseGcConfig(cmd.Flags())
+	operator.DefineFlagsForPrepareSnapBackup(cmd.Flags())
 	return cmd
 }
diff --git a/br/pkg/aws/BUILD.bazel b/br/pkg/aws/BUILD.bazel
@@ -12,6 +12,7 @@ go_library(
         "@com_github_aws_aws_sdk_go//aws",
         "@com_github_aws_aws_sdk_go//aws/awserr",
         "@com_github_aws_aws_sdk_go//aws/session",
+        "@com_github_aws_aws_sdk_go//service/cloudwatch",
         "@com_github_aws_aws_sdk_go//service/ec2",
         "@com_github_aws_aws_sdk_go//service/ec2/ec2iface",
         "@com_github_pingcap_errors//:errors",