Merge branch 'master' into issue_25941

.build.ps1

@@ -8,6 +8,7 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

.github/.licenserc.yaml

@@ -0,0 +1,100 @@
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: PingCAP, Inc.
+  paths-ignore:
+    - 'docs/'
+    - 'br/'
+    - '.gitignore'
+    - '.gitattributes'
+    - '.golangci.yml'
+    - '.licenserc.yaml'
+    - 'LICENSES/'
+    - '**/*.md'
+    - '**/*.json'
+    - '**/*.pem'
+    - '**/*.crt'
+    - '**/*.test'
+    - '**/*.result'
+    - '**/*.example'
+    - '.codecov.yml'
+    - 'circle.yml'
+    - 'errors.toml'
+    - 'Jenkinsfile'
+    - '.editorconfig'
+    - 'hooks/pre-commit'
+    - '**/go.mod'
+    - '**/go.sum'
+    - 'LICENSE'
+    - '.github/'
+    # The license checker think the following 68 files do not have valid header licenses, need to be discussed.
+    # Ignore them for now.
+    - 'cmd/explaintest/config.toml'
+    - 'cmd/importer/config.toml'
+    - 'cmd/portgenerator/portgenerator.go'
+    - 'ddl/ddl.go'
+    - 'ddl/ddl_api.go'
+    - 'ddl/mock.go'
+    - 'ddl/options.go'
+    - 'executor/aggfuncs/func_stddevpop_test.go'
+    - 'executor/aggfuncs/func_stddevsamp_test.go'
+    - 'executor/aggfuncs/func_varpop_test.go'
+    - 'executor/aggfuncs/func_varsamp_test.go'
+    - 'executor/index_lookup_merge_join_test.go'
+    - 'executor/pkg_test.go'
+    - 'expression/aggregation/base_func_test.go'
+    - 'expression/aggregation/util_test.go'
+    - 'expression/builtin.go'
+    - 'expression/builtin_info.go'
+    - 'expression/builtin_math.go'
+    - 'expression/builtin_miscellaneous.go'
+    - 'expression/builtin_miscellaneous_test.go'
+    - 'expression/builtin_string.go'
+    - 'expression/builtin_time.go'
+    - 'metrics/alertmanager/tidb.rules.yml'
+    - 'metrics/topsql.go'
+    - 'planner/core/cache_test.go'
+    - 'planner/core/logical_plans_test.go'
+    - 'planner/core/partition_prune.go'
+    - 'planner/core/partition_pruning_test.go'
+    - 'planner/core/rule_aggregation_push_down.go'
+    - 'planner/core/rule_max_min_eliminate.go'
+    - 'planner/core/rule_partition_processor.go'
+    - 'planner/core/rule_predicate_push_down.go'
+    - 'planner/core/telemetry.go'
+    - 'plugin/conn_ip_example/manifest.toml'
+    - 'server/conn.go'
+    - 'server/conn_stmt.go'
+    - 'server/packetio.go'
+    - 'server/server.go'
+    - 'server/util.go'
+    - 'session/bootstrap.go'
+    - 'session/session.go'
+    - 'session/tidb.go'
+    - 'store/driver/kv_test.go'
+    - 'store/mockstore/unistore/config/config-template.toml'
+    - 'store/mockstore/unistore/server/server.go'
+    - 'table/column.go'
+    - 'table/table.go'
+    - 'table/tables/tables.go'
+    - 'tests/globalkilltest/config.toml'
+    - 'tests/readonlytest/readonly_test.go'
+    - 'tools/check/check-timeout.go'
+    - 'tools/check/revive.toml'
+    - 'types/convert.go'
+    - 'types/etc.go'
+    - 'types/fuzzMarshalJSON.go'
+    - 'types/fuzzNewBitLiteral.go'
+    - 'types/fuzzNewHexLiteral.go'
+    - 'util/collate/collate_bench_test.go'
+    - 'util/format/format.go'
+    - 'util/logutil/slow_query_logger.go'
+    - 'util/mvmap/fnv.go'
+    - 'util/prefix_helper.go'
+    - 'util/resourcegrouptag/resource_group_tag.go'
+    - 'util/set/set_with_memory_usage_test.go'
+    - 'util/testkit/fake.go'
+    - 'util/testleak/leaktest.go'
+    - 'util/timeutil/time_test.go'
+    - 'util/topsql/reporter/mock/server.go'
+  comment: on-failure

.github/workflows/license-checker.yml

@@ -0,0 +1,23 @@
+name: License checker
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check-license:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check License Header
+        uses: apache/skywalking-eyes@main
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          log: info
+          config: .github/.licenserc.yaml

Dockerfile

@@ -8,6 +8,7 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

Makefile

@@ -8,6 +8,7 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

Makefile.common

@@ -8,6 +8,7 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

bindinfo/bind_test.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
@@ -2306,7 +2307,73 @@ func (s *testSuite) TestLocalTemporaryTable(c *C) {
 	tk.MustGetErrCode("create binding for replace into tmp2 select * from t2 where t2.b = 1 and t2.c > 1 using replace into t select /*+ use_index(t2,c) */ * from t2 where t2.b = 1 and t2.c > 1", errno.ErrOptOnTemporaryTable)
 	tk.MustGetErrCode("create binding for update tmp2 set a = 1 where b = 1 and c > 1 using update /*+ use_index(t, c) */ t set a = 1 where b = 1 and c > 1", errno.ErrOptOnTemporaryTable)
 	tk.MustGetErrCode("create binding for delete from tmp2 where b = 1 and c > 1 using delete /*+ use_index(t, c) */ from t where b = 1 and c > 1", errno.ErrOptOnTemporaryTable)
+}
+
+func (s *testSuite) TestIssue25505(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	stmtsummary.StmtSummaryByDigestMap.Clear()
+	s.cleanBindingEnv(tk)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	defer func() {
+		tk.MustExec("set tidb_slow_log_threshold = 300")
+	}()
+	tk.MustExec("set tidb_slow_log_threshold = 0")
+	tk.MustExec("create table t (a int(11) default null,b int(11) default null,key b (b),key ba (b))")
+	tk.MustExec("create table t1 (a int(11) default null,b int(11) default null,key idx_ab (a,b),key idx_a (a),key idx_b (b))")
+	tk.MustExec("create table t2 (a int(11) default null,b int(11) default null,key idx_ab (a,b),key idx_a (a),key idx_b (b))")
+	c.Assert(tk.Se.Auth(&auth.UserIdentity{Username: "root", Hostname: "%"}, nil, nil), IsTrue)
 
+	spmMap := map[string]string{}
+	spmMap["with recursive `cte` ( `a` ) as ( select ? union select `a` + ? from `test` . `t1` where `a` < ? ) select * from `cte`"] =
+		"WITH RECURSIVE `cte` (`a`) AS (SELECT 2 UNION SELECT `a` + 1 FROM `test`.`t1` WHERE `a` < 5) SELECT /*+ use_index(@`sel_3` `test`.`t1` `idx_ab`), hash_agg(@`sel_1`)*/ * FROM `cte`"
+	spmMap["with recursive `cte1` ( `a` , `b` ) as ( select * from `test` . `t` where `b` = ? union select `a` + ? , `b` + ? from `cte1` where `a` < ? ) select * from `test` . `t`"] =
+		"WITH RECURSIVE `cte1` (`a`, `b`) AS (SELECT * FROM `test`.`t` WHERE `b` = 1 UNION SELECT `a` + 1,`b` + 1 FROM `cte1` WHERE `a` < 2) SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `test`.`t`"
+	spmMap["with `cte1` as ( select * from `test` . `t` ) , `cte2` as ( select ? ) select * from `test` . `t`"] =
+		"WITH `cte1` AS (SELECT * FROM `test`.`t`), `cte2` AS (SELECT 4) SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `test`.`t`"
+	spmMap["with `cte` as ( select * from `test` . `t` where `b` = ? ) select * from `test` . `t`"] =
+		"WITH `cte` AS (SELECT * FROM `test`.`t` WHERE `b` = 6) SELECT /*+ use_index(@`sel_1` `test`.`t` )*/ * FROM `test`.`t`"
+	spmMap["with recursive `cte` ( `a` ) as ( select ? union select `a` + ? from `test` . `t1` where `a` > ? ) select * from `cte`"] =
+		"WITH RECURSIVE `cte` (`a`) AS (SELECT 2 UNION SELECT `a` + 1 FROM `test`.`t1` WHERE `a` > 5) SELECT /*+ use_index(@`sel_3` `test`.`t1` `idx_b`), hash_agg(@`sel_1`)*/ * FROM `cte`"
+	spmMap["with `cte` as ( with `cte1` as ( select * from `test` . `t2` where `a` > ? and `b` > ? ) select * from `cte1` ) select * from `cte` join `test` . `t1` on `t1` . `a` = `cte` . `a`"] =
+		"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` > 1 AND `b` > 1) SELECT * FROM `cte1`) SELECT /*+ use_index(@`sel_3` `test`.`t2` `idx_ab`), use_index(@`sel_1` `test`.`t1` `idx_ab`), inl_join(@`sel_1` `test`.`t1`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"
+	spmMap["with `cte` as ( with `cte1` as ( select * from `test` . `t2` where `a` = ? and `b` = ? ) select * from `cte1` ) select * from `cte` join `test` . `t1` on `t1` . `a` = `cte` . `a`"] =
+		"WITH `cte` AS (WITH `cte1` AS (SELECT * FROM `test`.`t2` WHERE `a` = 1 AND `b` = 1) SELECT * FROM `cte1`) SELECT /*+ use_index(@`sel_3` `test`.`t2` `idx_a`), use_index(@`sel_1` `test`.`t1` `idx_a`), inl_join(@`sel_1` `test`.`t1`)*/ * FROM `cte` JOIN `test`.`t1` ON `t1`.`a` = `cte`.`a`"
+
+	tk.MustExec("with cte as (with cte1 as (select /*+use_index(t2 idx_a)*/ * from t2 where a = 1 and b = 1) select * from cte1) select /*+use_index(t1 idx_a)*/ * from cte join t1 on t1.a=cte.a;")
+	tk.MustExec("with cte as (with cte1 as (select /*+use_index(t2 idx_a)*/ * from t2 where a = 1 and b = 1) select * from cte1) select /*+use_index(t1 idx_a)*/ * from cte join t1 on t1.a=cte.a;")
+	tk.MustExec("with cte as (with cte1 as (select /*+use_index(t2 idx_a)*/ * from t2 where a = 1 and b = 1) select * from cte1) select /*+use_index(t1 idx_a)*/ * from cte join t1 on t1.a=cte.a;")
+
+	tk.MustExec("with cte as (with cte1 as (select * from t2 use index(idx_ab) where a > 1 and b > 1) select * from cte1) select /*+use_index(t1 idx_ab)*/ * from cte join t1 on t1.a=cte.a;")
+	tk.MustExec("with cte as (with cte1 as (select * from t2 use index(idx_ab) where a > 1 and b > 1) select * from cte1) select /*+use_index(t1 idx_ab)*/ * from cte join t1 on t1.a=cte.a;")
+	tk.MustExec("with cte as (with cte1 as (select * from t2 use index(idx_ab) where a > 1 and b > 1) select * from cte1) select /*+use_index(t1 idx_ab)*/ * from cte join t1 on t1.a=cte.a;")
+
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT a+1 FROM t1 use index(idx_ab) WHERE a < 5) SELECT * FROM cte;")
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT a+1 FROM t1 use index(idx_ab) WHERE a < 5) SELECT * FROM cte;")
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT a+1 FROM t1 use index(idx_ab) WHERE a < 5) SELECT * FROM cte;")
+
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT /*+use_index(t1 idx_b)*/  a+1 FROM t1  WHERE a > 5) SELECT * FROM cte;")
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT /*+use_index(t1 idx_b)*/  a+1 FROM t1  WHERE a > 5) SELECT * FROM cte;")
+	tk.MustExec("WITH RECURSIVE cte(a) AS (SELECT 2 UNION SELECT /*+use_index(t1 idx_b)*/  a+1 FROM t1  WHERE a > 5) SELECT * FROM cte;")
+
+	tk.MustExec("with cte as (select * from t where b=6) select * from t")
+	tk.MustExec("with cte as (select * from t where b=6) select * from t")
+	tk.MustExec("with cte as (select * from t where b=6) select * from t")
+
+	tk.MustExec("with cte1 as (select * from t), cte2 as (select 4) select * from t")
+	tk.MustExec("with cte1 as (select * from t), cte2 as (select 5) select * from t")
+	tk.MustExec("with cte1 as (select * from t), cte2 as (select 6) select * from t")
+
+	tk.MustExec("with recursive cte1(a,b) as (select * from t where b = 1 union select a+1,b+1 from cte1 where a < 2) select * from t")
+	tk.MustExec("with recursive cte1(a,b) as (select * from t where b = 1 union select a+1,b+1 from cte1 where a < 2) select * from t")
+	tk.MustExec("with recursive cte1(a,b) as (select * from t where b = 1 union select a+1,b+1 from cte1 where a < 2) select * from t")
+	tk.MustExec("admin capture bindings")
+	rows := tk.MustQuery("show global bindings").Rows()
+	c.Assert(len(rows), Equals, 7)
+	for _, row := range rows {
+		str := fmt.Sprintf("%s", row[0])
+		c.Assert(row[1], Equals, spmMap[str])
+	}
 }
 
 func (s *testSuite) TestBindingLastUpdateTime(c *C) {

bindinfo/cache.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

bindinfo/handle.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
@@ -25,6 +26,7 @@ import (
 
 	"github.com/pingcap/parser"
 	"github.com/pingcap/parser/ast"
+	"github.com/pingcap/parser/format"
 	"github.com/pingcap/parser/mysql"
 	"github.com/pingcap/parser/terror"
 	"github.com/pingcap/tidb/metrics"
@@ -752,7 +754,22 @@ func GenerateBindSQL(ctx context.Context, stmtNode ast.StmtNode, planHint string
 		bindSQL = bindSQL[updateIdx:]
 		return strings.Replace(bindSQL, "UPDATE", fmt.Sprintf("UPDATE /*+ %s*/", planHint), 1)
 	case *ast.SelectStmt:
-		selectIdx := strings.Index(bindSQL, "SELECT")
+		var selectIdx int
+		if n.With != nil {
+			var withSb strings.Builder
+			withIdx := strings.Index(bindSQL, "WITH")
+			restoreCtx := format.NewRestoreCtx(format.RestoreStringSingleQuotes|format.RestoreSpacesAroundBinaryOperation|format.RestoreStringWithoutCharset|format.RestoreNameBackQuotes, &withSb)
+			restoreCtx.DefaultDB = defaultDB
+			err := n.With.Restore(restoreCtx)
+			if err != nil {
+				logutil.BgLogger().Debug("[sql-bind] restore SQL failed", zap.Error(err))
+				return ""
+			}
+			withEnd := withIdx + len(withSb.String())
+			tmp := strings.Replace(bindSQL[withEnd:], "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1)
+			return strings.Join([]string{bindSQL[withIdx:withEnd], tmp}, "")
+		}
+		selectIdx = strings.Index(bindSQL, "SELECT")
 		// Remove possible `explain` prefix.
 		bindSQL = bindSQL[selectIdx:]
 		return strings.Replace(bindSQL, "SELECT", fmt.Sprintf("SELECT /*+ %s*/", planHint), 1)

bindinfo/session_handle.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

bindinfo/stat.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/cmd/br/main_test.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/cmd/tidb-lightning-ctl/main.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/cmd/tidb-lightning-ctl/main_test.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/cmd/tidb-lightning/main.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/cmd/tidb-lightning/main_test.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
 

br/compatibility/prepare_backup.sh

@@ -10,6 +10,7 @@
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 

br/pkg/backup/schema_test.go

@@ -132,6 +132,8 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 		ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
 	c.Assert(updateCh.get(), Equals, int64(1))
 	c.Assert(err, IsNil)
+	err = metaWriter.FlushBackupMeta(ctx)
+	c.Assert(err, IsNil)
 
 	schemas := s.GetSchemasFromMeta(c, es)
 	c.Assert(len(schemas), Equals, 1)
@@ -157,6 +159,8 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchema(c *C) {
 		ctx, metaWriter2, s.mock.Storage, nil, math.MaxUint64, 2, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
 	c.Assert(updateCh.get(), Equals, int64(2))
 	c.Assert(err, IsNil)
+	err = metaWriter2.FlushBackupMeta(ctx)
+	c.Assert(err, IsNil)
 
 	schemas = s.GetSchemasFromMeta(c, es2)
 
@@ -201,6 +205,9 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c *
 	ctx := context.Background()
 	err = backupSchemas.BackupSchemas(
 		ctx, metaWriter, s.mock.Storage, nil, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
+	c.Assert(err, IsNil)
+	err = metaWriter.FlushBackupMeta(ctx)
+	c.Assert(err, IsNil)
 
 	schemas := s.GetSchemasFromMeta(c, es)
 	c.Assert(err, IsNil)
@@ -227,6 +234,8 @@ func (s *testBackupSchemaSuite) TestBuildBackupRangeAndSchemaWithBrokenStats(c *
 	err = backupSchemas.BackupSchemas(
 		ctx, metaWriter2, s.mock.Storage, statsHandle, math.MaxUint64, 1, variable.DefChecksumTableConcurrency, skipChecksum, updateCh)
 	c.Assert(err, IsNil)
+	err = metaWriter2.FlushBackupMeta(ctx)
+	c.Assert(err, IsNil)
 
 	schemas2 := s.GetSchemasFromMeta(c, es2)
 	c.Assert(schemas2, HasLen, 1)

br/pkg/cdclog/buffer.go

@@ -8,6 +8,7 @@
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.