Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ddl: add a binlog when dropping column (#13536) (#13613) #13974

Merged
merged 4 commits into from
Dec 20, 2019
Merged

ddl: add a binlog when dropping column (#13536) (#13613) #13974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8ddd007
ddl: add a binlog when dropping column (#13536) (#13613)
zimulala Nov 20, 2019
feb0d27
*: update go.mod
zimulala Dec 9, 2019
35a8cef
statistics: update
zimulala Dec 20, 2019
862df1a
Merge branch 'release-2.1' into zimuxia/drop-col
zimulala Dec 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions ddl/ddl_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/parser/model"
"github.com/pingcap/parser/terror"
pumpcli "github.com/pingcap/tidb-tools/tidb-binlog/pump_client"
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/meta"
"github.com/pingcap/tidb/metrics"
Expand Down Expand Up @@ -396,9 +397,7 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error {
if err = w.handleUpdateJobError(t, job, err); err != nil {
return errors.Trace(err)
}
if job.IsDone() || job.IsRollbackDone() {
binloginfo.SetDDLBinlog(d.binlogCli, txn, job.ID, job.Query)
}
writeBinlog(d.binlogCli, txn, job)
return nil
})

Expand Down Expand Up @@ -431,6 +430,16 @@ func (w *worker) handleDDLJobQueue(d *ddlCtx) error {
}
}

func writeBinlog(binlogCli *pumpcli.PumpsClient, txn kv.Transaction, job *model.Job) {
if job.IsDone() || job.IsRollbackDone() ||
// When this column is in the "delete only" and "delete reorg" states, the binlog of "drop column" has not been written yet,
// but the column has been removed from the binlog of the write operation.
// So we add this binlog to enable downstream components to handle DML correctly in this schema state.
(job.Type == model.ActionDropColumn && job.SchemaState == model.StateDeleteOnly) {
binloginfo.SetDDLBinlog(binlogCli, txn, job.ID, int32(job.SchemaState), job.Query)
}
}

// waitDependencyJobFinished waits for the dependency-job to be finished.
// If the dependency job isn't finished yet, we'd better wait a moment.
func (w *worker) waitDependencyJobFinished(job *model.Job, cnt *int) {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ require (
github.com/pingcap/parser v0.0.0-20191118062231-d32e257db9f2
github.com/pingcap/pd v2.1.12+incompatible
github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible
github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93
github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6
github.com/prometheus/client_golang v0.8.0
github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5
github.com/prometheus/common v0.0.0-20180426121432-d811d2e9bf89 // indirect
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ github.com/pingcap/pd v2.1.12+incompatible h1:6N3LBxx2aSZqT+IWEG730EDNDttP7dXO8J
github.com/pingcap/pd v2.1.12+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible h1:e9Gi/LP9181HT3gBfSOeSBA+5JfemuE4aEAhqNgoE4k=
github.com/pingcap/tidb-tools v2.1.3-0.20190116051332-34c808eef588+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM=
github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93 h1:gI5bOzLMxjUq6ui+md/JnT4pYpkzrABJ/PeYORWiYYs=
github.com/pingcap/tipb v0.0.0-20180910045846-371b48b15d93/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6 h1:HPgqtaqIFIZXTvQNiZoJ9Y79AXz3pmDpYFL28KraTKE=
github.com/pingcap/tipb v0.0.0-20191120045257-1b9900292ab6/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand Down
9 changes: 5 additions & 4 deletions sessionctx/binloginfo/binloginfo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,17 +192,18 @@ func (info *BinlogInfo) WriteBinlog(clusterID uint64) error {
}

// SetDDLBinlog sets DDL binlog in the kv.Transaction.
func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlQuery string) {
func SetDDLBinlog(client *pumpcli.PumpsClient, txn kv.Transaction, jobID int64, ddlSchemaState int32, ddlQuery string) {
if client == nil {
return
}

ddlQuery = AddSpecialComment(ddlQuery)
info := &BinlogInfo{
Data: &binlog.Binlog{
Tp: binlog.BinlogType_Prewrite,
DdlJobId: jobID,
DdlQuery: []byte(ddlQuery),
Tp: binlog.BinlogType_Prewrite,
DdlJobId: jobID,
DdlSchemaState: ddlSchemaState,
DdlQuery: []byte(ddlQuery),
},
Client: client,
}
Expand Down
2 changes: 1 addition & 1 deletion statistics/cmsketch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func (s *testStatisticsSuite) TestCMSketchCoding(c *C) {
}
bytes, err := encodeCMSketch(lSketch)
c.Assert(err, IsNil)
c.Assert(len(bytes), Equals, 61455)
c.Assert(len(bytes), Equals, 61457)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update it because tipb is newer.
PTAL @lamxTyler

rSketch, err := decodeCMSketch(bytes)
c.Assert(err, IsNil)
c.Assert(lSketch.Equal(rSketch), IsTrue)
Expand Down