From 396be5bb27ad4ff4d3e044844b9c4909b52e6bce Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 29 May 2024 15:01:21 +0800 Subject: [PATCH] This is an automated cherry-pick of #53617 Signed-off-by: ti-chi-bot --- ddl/column.go | 2 + executor/executor_txn_test.go | 47 ++++ pkg/executor/BUILD.bazel | 487 ++++++++++++++++++++++++++++++++++ table/tables/tables.go | 5 + 4 files changed, 541 insertions(+) create mode 100644 pkg/executor/BUILD.bazel diff --git a/ddl/column.go b/ddl/column.go index 7cef23ac56ffc..334960f313941 100644 --- a/ddl/column.go +++ b/ddl/column.go @@ -169,6 +169,7 @@ func onAddColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, err error) case model.StateWriteReorganization: // reorganization -> public // Adjust table column offset. + failpoint.InjectCall("onAddColumnStateWriteReorg") offset, err := LocateOffsetToMove(columnInfo.Offset, pos, tblInfo) if err != nil { return ver, errors.Trace(err) @@ -263,6 +264,7 @@ func onDropColumn(d *ddlCtx, t *meta.Meta, job *model.Job) (ver int64, _ error) } case model.StateWriteOnly: // write only -> delete only + failpoint.InjectCall("onDropColumnStateWriteOnly") colInfo.State = model.StateDeleteOnly tblInfo.MoveColumnInfo(colInfo.Offset, len(tblInfo.Columns)-1) if len(idxInfos) > 0 { diff --git a/executor/executor_txn_test.go b/executor/executor_txn_test.go index d2bd8f862830e..45041d2e852fb 100644 --- a/executor/executor_txn_test.go +++ b/executor/executor_txn_test.go @@ -19,13 +19,22 @@ import ( "fmt" "strconv" "strings" + "sync" "testing" "time" +<<<<<<< HEAD:executor/executor_txn_test.go "github.com/pingcap/tidb/executor" "github.com/pingcap/tidb/sessionctx/binloginfo" "github.com/pingcap/tidb/testkit" "github.com/pingcap/tipb/go-binlog" +======= + "github.com/pingcap/tidb/pkg/errno" + "github.com/pingcap/tidb/pkg/executor" + "github.com/pingcap/tidb/pkg/sessionctx/binloginfo" + "github.com/pingcap/tidb/pkg/testkit" + "github.com/pingcap/tidb/pkg/testkit/testfailpoint" +>>>>>>> 6efce0f061d (executor: sync deletable columns to binlog when remove record (#53617)):pkg/executor/executor_txn_test.go "github.com/stretchr/testify/require" "google.golang.org/grpc" ) @@ -789,6 +798,7 @@ func TestSavepointWithBinlog(t *testing.T) { tk.MustQuery("select * from t").Check(testkit.Rows("1 1")) } +<<<<<<< HEAD:executor/executor_txn_test.go type mockPumpClient struct{} func (m mockPumpClient) WriteBinlog(ctx context.Context, in *binlog.WriteBinlogReq, opts ...grpc.CallOption) (*binlog.WriteBinlogResp, error) { @@ -797,4 +807,41 @@ func (m mockPumpClient) WriteBinlog(ctx context.Context, in *binlog.WriteBinlogR func (m mockPumpClient) PullBinlogs(ctx context.Context, in *binlog.PullBinlogReq, opts ...grpc.CallOption) (binlog.Pump_PullBinlogsClient, error) { return nil, nil +======= +func TestColumnNotMatchError(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.Session().GetSessionVars().BinlogClient = binloginfo.MockPumpsClient(&testkit.MockPumpClient{}) + tk.MustExec("set @@global.tidb_enable_metadata_lock=0") + tk.MustExec("use test") + tk2 := testkit.NewTestKit(t, store) + tk2.MustExec("use test") + tk.MustExec("create table t(id int primary key, a int)") + tk.MustExec("insert into t values(1, 2)") + + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onAddColumnStateWriteReorg", func() { + tk.MustExec("begin;") + }) + var wg sync.WaitGroup + wg.Add(1) + go func() { + tk2.MustExec("alter table t add column wait_notify int") + wg.Done() + }() + wg.Wait() + tk.MustExec("delete from t where id=1") + tk.MustGetErrCode("commit", errno.ErrInfoSchemaChanged) + + testfailpoint.EnableCall(t, "github.com/pingcap/tidb/pkg/ddl/onDropColumnStateWriteOnly", func() { + tk.MustExec("begin;") + }) + wg.Add(1) + go func() { + tk2.MustExec("alter table t drop column wait_notify") + wg.Done() + }() + wg.Wait() + tk.MustExec("delete from t where id=1") + tk.MustGetErrCode("commit", errno.ErrInfoSchemaChanged) +>>>>>>> 6efce0f061d (executor: sync deletable columns to binlog when remove record (#53617)):pkg/executor/executor_txn_test.go } diff --git a/pkg/executor/BUILD.bazel b/pkg/executor/BUILD.bazel new file mode 100644 index 0000000000000..b3b89fd9f2a2d --- /dev/null +++ b/pkg/executor/BUILD.bazel @@ -0,0 +1,487 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "executor", + srcs = [ + "adapter.go", + "admin.go", + "admin_plugins.go", + "analyze.go", + "analyze_col.go", + "analyze_col_v2.go", + "analyze_global_stats.go", + "analyze_idx.go", + "analyze_utils.go", + "analyze_worker.go", + "batch_checker.go", + "batch_point_get.go", + "bind.go", + "brie.go", + "brie_utils.go", + "builder.go", + "change.go", + "checksum.go", + "compact_table.go", + "compiler.go", + "coprocessor.go", + "cte.go", + "cte_table_reader.go", + "ddl.go", + "delete.go", + "distsql.go", + "executor.go", + "explain.go", + "foreign_key.go", + "grant.go", + "import_into.go", + "index_advise.go", + "index_merge_reader.go", + "infoschema_reader.go", + "insert.go", + "insert_common.go", + "inspection_common.go", + "inspection_profile.go", + "inspection_result.go", + "inspection_summary.go", + "load_data.go", + "load_stats.go", + "mem_reader.go", + "memtable_reader.go", + "metrics_reader.go", + "mpp_gather.go", + "opt_rule_blacklist.go", + "parallel_apply.go", + "pipelined_window.go", + "plan_replayer.go", + "point_get.go", + "prepared.go", + "projection.go", + "reload_expr_pushdown_blacklist.go", + "replace.go", + "revoke.go", + "sample.go", + "select_into.go", + "set.go", + "set_config.go", + "show.go", + "show_placement.go", + "show_stats.go", + "shuffle.go", + "simple.go", + "slow_query.go", + "split.go", + "stmtsummary.go", + "table_reader.go", + "trace.go", + "union_scan.go", + "update.go", + "utils.go", + "window.go", + "write.go", + ], + importpath = "github.com/pingcap/tidb/pkg/executor", + visibility = ["//visibility:public"], + deps = [ + "//br/pkg/glue", + "//br/pkg/storage", + "//br/pkg/task", + "//br/pkg/task/show", + "//br/pkg/utils", + "//pkg/bindinfo", + "//pkg/config", + "//pkg/ddl", + "//pkg/ddl/label", + "//pkg/ddl/placement", + "//pkg/ddl/schematracker", + "//pkg/distsql", + "//pkg/distsql/context", + "//pkg/disttask/framework/handle", + "//pkg/disttask/framework/proto", + "//pkg/disttask/framework/storage", + "//pkg/disttask/importinto", + "//pkg/domain", + "//pkg/domain/infosync", + "//pkg/domain/resourcegroup", + "//pkg/errctx", + "//pkg/errno", + "//pkg/executor/aggfuncs", + "//pkg/executor/aggregate", + "//pkg/executor/importer", + "//pkg/executor/internal/applycache", + "//pkg/executor/internal/builder", + "//pkg/executor/internal/calibrateresource", + "//pkg/executor/internal/exec", + "//pkg/executor/internal/mpp", + "//pkg/executor/internal/pdhelper", + "//pkg/executor/internal/querywatch", + "//pkg/executor/internal/testutil", + "//pkg/executor/internal/util", + "//pkg/executor/internal/vecgroupchecker", + "//pkg/executor/join", + "//pkg/executor/lockstats", + "//pkg/executor/metrics", + "//pkg/executor/sortexec", + "//pkg/executor/unionexec", + "//pkg/expression", + "//pkg/expression/aggregation", + "//pkg/expression/context", + "//pkg/infoschema", + "//pkg/infoschema/context", + "//pkg/keyspace", + "//pkg/kv", + "//pkg/lightning/log", + "//pkg/lightning/mydump", + "//pkg/meta", + "//pkg/meta/autoid", + "//pkg/metrics", + "//pkg/parser", + "//pkg/parser/ast", + "//pkg/parser/auth", + "//pkg/parser/charset", + "//pkg/parser/format", + "//pkg/parser/model", + "//pkg/parser/mysql", + "//pkg/parser/terror", + "//pkg/parser/tidb", + "//pkg/parser/types", + "//pkg/planner", + "//pkg/planner/cardinality", + "//pkg/planner/context", + "//pkg/planner/core", + "//pkg/planner/core/base", + "//pkg/planner/util", + "//pkg/planner/util/coreusage", + "//pkg/planner/util/fixcontrol", + "//pkg/plugin", + "//pkg/privilege", + "//pkg/privilege/privileges", + "//pkg/resourcemanager/pool/workerpool", + "//pkg/resourcemanager/util", + "//pkg/session/txninfo", + "//pkg/sessionctx", + "//pkg/sessionctx/binloginfo", + "//pkg/sessionctx/sessionstates", + "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", + "//pkg/sessiontxn", + "//pkg/sessiontxn/staleread", + "//pkg/statistics", + "//pkg/statistics/handle", + "//pkg/statistics/handle/cache", + "//pkg/statistics/handle/storage", + "//pkg/statistics/handle/types", + "//pkg/statistics/handle/util", + "//pkg/store/driver/backoff", + "//pkg/store/driver/txn", + "//pkg/store/helper", + "//pkg/table", + "//pkg/table/tables", + "//pkg/table/temptable", + "//pkg/tablecodec", + "//pkg/tidb-binlog/node", + "//pkg/types", + "//pkg/types/parser_driver", + "//pkg/util", + "//pkg/util/admin", + "//pkg/util/breakpoint", + "//pkg/util/channel", + "//pkg/util/chunk", + "//pkg/util/codec", + "//pkg/util/collate", + "//pkg/util/context", + "//pkg/util/cteutil", + "//pkg/util/dbterror", + "//pkg/util/dbterror/exeerrors", + "//pkg/util/dbterror/plannererrors", + "//pkg/util/deadlockhistory", + "//pkg/util/disk", + "//pkg/util/disttask", + "//pkg/util/etcd", + "//pkg/util/execdetails", + "//pkg/util/filter", + "//pkg/util/format", + "//pkg/util/gcutil", + "//pkg/util/globalconn", + "//pkg/util/hack", + "//pkg/util/hint", + "//pkg/util/intest", + "//pkg/util/keydecoder", + "//pkg/util/logutil", + "//pkg/util/logutil/consistency", + "//pkg/util/mathutil", + "//pkg/util/memory", + "//pkg/util/password-validation", + "//pkg/util/plancodec", + "//pkg/util/printer", + "//pkg/util/ranger", + "//pkg/util/ranger/context", + "//pkg/util/redact", + "//pkg/util/replayer", + "//pkg/util/resourcegrouptag", + "//pkg/util/rowDecoder", + "//pkg/util/rowcodec", + "//pkg/util/sem", + "//pkg/util/servermemorylimit", + "//pkg/util/set", + "//pkg/util/size", + "//pkg/util/sqlescape", + "//pkg/util/sqlexec", + "//pkg/util/sqlkiller", + "//pkg/util/stmtsummary", + "//pkg/util/stmtsummary/v2:stmtsummary", + "//pkg/util/stringutil", + "//pkg/util/syncutil", + "//pkg/util/table-filter", + "//pkg/util/tiflash", + "//pkg/util/timeutil", + "//pkg/util/tls", + "//pkg/util/topsql", + "//pkg/util/topsql/state", + "//pkg/util/tracing", + "@com_github_burntsushi_toml//:toml", + "@com_github_docker_go_units//:go-units", + "@com_github_gogo_protobuf//proto", + "@com_github_google_uuid//:uuid", + "@com_github_ngaut_pools//:pools", + "@com_github_opentracing_basictracer_go//:basictracer-go", + "@com_github_opentracing_opentracing_go//:opentracing-go", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@com_github_pingcap_kvproto//pkg/brpb", + "@com_github_pingcap_kvproto//pkg/coprocessor", + "@com_github_pingcap_kvproto//pkg/deadlock", + "@com_github_pingcap_kvproto//pkg/diagnosticspb", + "@com_github_pingcap_kvproto//pkg/encryptionpb", + "@com_github_pingcap_kvproto//pkg/kvrpcpb", + "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_kvproto//pkg/resource_manager", + "@com_github_pingcap_kvproto//pkg/tikvpb", + "@com_github_pingcap_log//:log", + "@com_github_pingcap_sysutil//:sysutil", + "@com_github_pingcap_tipb//go-tipb", + "@com_github_prometheus_client_golang//api", + "@com_github_prometheus_client_golang//api/prometheus/v1:prometheus", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_common//model", + "@com_github_tiancaiamao_gp//:gp", + "@com_github_tikv_client_go_v2//error", + "@com_github_tikv_client_go_v2//kv", + "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//txnkv", + "@com_github_tikv_client_go_v2//txnkv/txnlock", + "@com_github_tikv_client_go_v2//txnkv/txnsnapshot", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//:client", + "@com_github_tikv_pd_client//http", + "@com_github_twmb_murmur3//:murmur3", + "@com_sourcegraph_sourcegraph_appdash//:appdash", + "@com_sourcegraph_sourcegraph_appdash//opentracing", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//codes", + "@org_golang_google_grpc//credentials", + "@org_golang_google_grpc//credentials/insecure", + "@org_golang_google_grpc//status", + "@org_golang_x_sync//errgroup", + "@org_uber_go_atomic//:atomic", + "@org_uber_go_zap//:zap", + "@org_uber_go_zap//zapcore", + ], +) + +go_test( + name = "executor_test", + timeout = "moderate", + srcs = [ + "adapter_test.go", + "analyze_test.go", + "analyze_utils_test.go", + "batch_point_get_test.go", + "benchmark_test.go", + "brie_test.go", + "brie_utils_test.go", + "checksum_test.go", + "chunk_size_control_test.go", + "cluster_table_test.go", + "compact_table_test.go", + "copr_cache_test.go", + "cte_test.go", + "delete_test.go", + "distsql_test.go", + "executor_failpoint_test.go", + "executor_pkg_test.go", + "executor_required_rows_test.go", + "executor_test.go", + "executor_txn_test.go", + "explain_test.go", + "explain_unit_test.go", + "explainfor_test.go", + "grant_test.go", + "historical_stats_test.go", + "hot_regions_history_table_test.go", + "import_into_test.go", + "index_advise_test.go", + "infoschema_cluster_table_test.go", + "infoschema_reader_internal_test.go", + "infoschema_reader_test.go", + "insert_test.go", + "inspection_result_test.go", + "inspection_summary_test.go", + "join_pkg_test.go", + "main_test.go", + "memtable_reader_test.go", + "metrics_reader_test.go", + "parallel_apply_test.go", + "partition_table_test.go", + "pkg_test.go", + "point_get_test.go", + "prepared_test.go", + "recover_test.go", + "resource_tag_test.go", + "revoke_test.go", + "sample_test.go", + "select_into_test.go", + "set_test.go", + "show_placement_labels_test.go", + "show_placement_test.go", + "show_stats_test.go", + "show_test.go", + "shuffle_test.go", + "slow_query_sql_test.go", + "slow_query_test.go", + "split_test.go", + "stale_txn_test.go", + "stmtsummary_test.go", + "table_readers_required_rows_test.go", + "temporary_table_test.go", + "tikv_regions_peers_table_test.go", + "trace_test.go", + "union_scan_test.go", + "update_test.go", + "utils_test.go", + "window_test.go", + "write_concurrent_test.go", + ], + data = glob(["testdata/**"]), + embed = [":executor"], + flaky = True, + shard_count = 50, + deps = [ + "//pkg/config", + "//pkg/ddl", + "//pkg/ddl/placement", + "//pkg/ddl/util", + "//pkg/distsql", + "//pkg/distsql/context", + "//pkg/domain", + "//pkg/domain/infosync", + "//pkg/errctx", + "//pkg/errno", + "//pkg/executor/aggfuncs", + "//pkg/executor/aggregate", + "//pkg/executor/importer", + "//pkg/executor/internal/builder", + "//pkg/executor/internal/exec", + "//pkg/executor/internal/testutil", + "//pkg/executor/join", + "//pkg/executor/sortexec", + "//pkg/expression", + "//pkg/expression/aggregation", + "//pkg/infoschema", + "//pkg/kv", + "//pkg/meta", + "//pkg/meta/autoid", + "//pkg/metrics", + "//pkg/parser", + "//pkg/parser/ast", + "//pkg/parser/auth", + "//pkg/parser/model", + "//pkg/parser/mysql", + "//pkg/parser/terror", + "//pkg/planner", + "//pkg/planner/core", + "//pkg/planner/core/base", + "//pkg/planner/property", + "//pkg/planner/util", + "//pkg/server", + "//pkg/session", + "//pkg/session/types", + "//pkg/sessionctx", + "//pkg/sessionctx/binloginfo", + "//pkg/sessionctx/stmtctx", + "//pkg/sessionctx/variable", + "//pkg/sessiontxn", + "//pkg/sessiontxn/staleread", + "//pkg/statistics", + "//pkg/statistics/handle/storage", + "//pkg/statistics/handle/util", + "//pkg/store/copr", + "//pkg/store/driver/error", + "//pkg/store/helper", + "//pkg/store/mockstore", + "//pkg/store/mockstore/unistore", + "//pkg/table/tables", + "//pkg/tablecodec", + "//pkg/testkit", + "//pkg/testkit/external", + "//pkg/testkit/testdata", + "//pkg/testkit/testfailpoint", + "//pkg/testkit/testmain", + "//pkg/testkit/testsetup", + "//pkg/types", + "//pkg/util", + "//pkg/util/benchdaily", + "//pkg/util/chunk", + "//pkg/util/codec", + "//pkg/util/collate", + "//pkg/util/dbterror", + "//pkg/util/dbterror/exeerrors", + "//pkg/util/dbterror/plannererrors", + "//pkg/util/deadlockhistory", + "//pkg/util/disk", + "//pkg/util/execdetails", + "//pkg/util/gcutil", + "//pkg/util/logutil", + "//pkg/util/memory", + "//pkg/util/mock", + "//pkg/util/paging", + "//pkg/util/ranger", + "//pkg/util/sem", + "//pkg/util/set", + "//pkg/util/sqlkiller", + "//pkg/util/stmtsummary/v2:stmtsummary", + "//pkg/util/stringutil", + "//pkg/util/syncutil", + "//pkg/util/tableutil", + "//pkg/util/topsql/state", + "@com_github_docker_go_units//:go-units", + "@com_github_gorilla_mux//:mux", + "@com_github_hashicorp_go_version//:go-version", + "@com_github_pingcap_errors//:errors", + "@com_github_pingcap_failpoint//:failpoint", + "@com_github_pingcap_fn//:fn", + "@com_github_pingcap_kvproto//pkg/brpb", + "@com_github_pingcap_kvproto//pkg/diagnosticspb", + "@com_github_pingcap_kvproto//pkg/encryptionpb", + "@com_github_pingcap_kvproto//pkg/kvrpcpb", + "@com_github_pingcap_kvproto//pkg/metapb", + "@com_github_pingcap_log//:log", + "@com_github_pingcap_sysutil//:sysutil", + "@com_github_pingcap_tipb//go-tipb", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_model//go", + "@com_github_prometheus_common//model", + "@com_github_stretchr_testify//require", + "@com_github_tikv_client_go_v2//oracle", + "@com_github_tikv_client_go_v2//testutils", + "@com_github_tikv_client_go_v2//tikv", + "@com_github_tikv_client_go_v2//tikvrpc", + "@com_github_tikv_client_go_v2//util", + "@com_github_tikv_pd_client//http", + "@org_golang_google_grpc//:grpc", + "@org_uber_go_atomic//:atomic", + "@org_uber_go_goleak//:goleak", + "@org_uber_go_zap//zapcore", + ], +) diff --git a/table/tables/tables.go b/table/tables/tables.go index 082715d571b1c..078ecb73a283e 100644 --- a/table/tables/tables.go +++ b/table/tables/tables.go @@ -1174,8 +1174,13 @@ func (t *TableCommon) RemoveRecord(ctx sessionctx.Context, h kv.Handle, r []type } memBuffer.Release(sh) +<<<<<<< HEAD:table/tables/tables.go if shouldWriteBinlog(ctx, t.meta) { cols := t.Cols() +======= + if shouldWriteBinlog(ctx.GetSessionVars(), t.meta) { + cols := t.DeletableCols() +>>>>>>> 6efce0f061d (executor: sync deletable columns to binlog when remove record (#53617)):pkg/table/tables/tables.go colIDs := make([]int64, 0, len(cols)+1) for _, col := range cols { colIDs = append(colIDs, col.ID)