diff --git a/ddl/ddl_worker.go b/ddl/ddl_worker.go index 56410a96ded41..baf374a15841f 100644 --- a/ddl/ddl_worker.go +++ b/ddl/ddl_worker.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/parser/model" "github.com/pingcap/parser/terror" + pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client" "github.com/pingcap/tidb/kv" "github.com/pingcap/tidb/meta" "github.com/pingcap/tidb/metrics" @@ -396,9 +397,7 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error { if err = w.handleUpdateJobError(t, job, err); err != nil { return errors.Trace(err) } - if job.IsDone() || job.IsRollbackDone() { - binloginfo.SetDDLBinlog(d.binlogCli, txn, job.ID, job.Query) - } + writeBinlog(d.binlogCli, txn, job) return nil }) @@ -431,6 +430,16 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error { } } +func writeBinlog(binlogCli *pumpcli.PumpsClient, txn kv.Transaction, job *model.Job) { + if job.IsDone() || job.IsRollbackDone() || + // When this column is in the "delete only" and "delete reorg" states, the binlog of "drop column" has not been written yet, + // but the column has been removed from the binlog of the write operation. + // So we add this binlog to enable downstream components to handle DML correctly in this schema state. + (job.Type == model.ActionDropColumn && job.SchemaState == model.StateDeleteOnly) { + binloginfo.SetDDLBinlog(binlogCli, txn, job.ID, int32(job.SchemaState), job.Query) + } +} + // waitDependencyJobFinished waits for the dependency-job to be finished. // If the dependency job isn't finished yet, we'd better wait a moment. func (w *worker) waitDependencyJobFinished(job *model.Job, cnt *int) { diff --git a/go.mod b/go.mod index 3fb18a02f1555..77c5869dec4fb 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/pingcap/parser v0.0.0-20191118062231-d32e257db9f2 github.com/pingcap/pd v2.1.12+incompatible github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible - github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 + github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 github.com/prometheus/client_golang v0.8.0 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 github.com/prometheus/common v0.0.0-20180426121432-d811d2e9bf89 // indirect diff --git a/go.sum b/go.sum index 8fccd0c3df7d4..ed1c336c3bcd1 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/pingcap/pd v2.1.12+incompatible h1:6N3LBxx2aSZqT+IWEG730EDNDttP7dXO8J github.com/pingcap/pd v2.1.12+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k= github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 h1:gI5bOzLMxjUq6ui+md/JnT4pYpkzrABJ/PeYORWiYYs= -github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= +github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 h1:HPgqtaqIFIZXTvQNiZoJ9Y79AXz3pmDpYFL28KraTKE= +github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/sessionctx/binloginfo/binloginfo.go b/sessionctx/binloginfo/binloginfo.go index dec30cdf1d00f..6df2f20382973 100644 --- a/sessionctx/binloginfo/binloginfo.go +++ b/sessionctx/binloginfo/binloginfo.go @@ -192,7 +192,7 @@ func (info *BinlogInfo) WriteBinlog(clusterID uint64) error { } // SetDDLBinlog sets DDL binlog in the kv.Transaction. -func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) { +func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlSchemaState int32, ddlQuery string) { if client == nil { return } @@ -200,9 +200,10 @@ func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery = AddSpecialComment(ddlQuery) info := &BinlogInfo{ Data: &binlog.Binlog{ - Tp: binlog.BinlogType_Prewrite, - DdlJobId: jobID, - DdlQuery: []byte(ddlQuery), + Tp: binlog.BinlogType_Prewrite, + DdlJobId: jobID, + DdlSchemaState: ddlSchemaState, + DdlQuery: []byte(ddlQuery), }, Client: client, } diff --git a/statistics/cmsketch_test.go b/statistics/cmsketch_test.go index d9fcfce253b9e..51f835996dd53 100644 --- a/statistics/cmsketch_test.go +++ b/statistics/cmsketch_test.go @@ -122,7 +122,7 @@ func (s *testStatisticsSuite) TestCMSketchCoding(c *C) { } bytes, err := encodeCMSketch(lSketch) c.Assert(err, IsNil) - c.Assert(len(bytes), Equals, 61455) + c.Assert(len(bytes), Equals, 61457) rSketch, err := decodeCMSketch(bytes) c.Assert(err, IsNil) c.Assert(lSketch.Equal(rSketch), IsTrue)