Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DropSources: change table rename logic #7230

Merged
merged 2 commits into from
Jan 5, 2021
Merged

DropSources: change table rename logic #7230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
88997a7
Change table rename template during DropSources for a MoveTables work…
rohit-nayak-ps Dec 30, 2020
01ed81b
gofmted
rohit-nayak-ps Jan 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions go/test/endtoend/vreplication/vreplication_test_env.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,8 @@ var dryRunResultsSwitchWritesM2m3 = []string{
var dryRunResultsDropSourcesDropCustomerShard = []string{
"Lock keyspace product",
"Lock keyspace customer",
"Dropping following tables from the database and from the vschema for keyspace product:",
" Keyspace product Shard 0 DbName vt_product Tablet 100 Table customer RemovalType DROP TABLE",
"Dropping these tables from the database and removing them from the vschema for keyspace product:",
" Keyspace product Shard 0 DbName vt_product Tablet 100 Table customer",
"Blacklisted tables customer will be removed from:",
" Keyspace product Shard 0 Tablet 100",
"Delete reverse vreplication streams on source:",
Expand All @@ -114,8 +114,8 @@ var dryRunResultsDropSourcesDropCustomerShard = []string{
var dryRunResultsDropSourcesRenameCustomerShard = []string{
"Lock keyspace product",
"Lock keyspace customer",
"Dropping following tables from the database and from the vschema for keyspace product:",
" Keyspace product Shard 0 DbName vt_product Tablet 100 Table customer RemovalType RENAME TABLE",
"Renaming these tables from the database and removing them from the vschema for keyspace product:",
" Keyspace product Shard 0 DbName vt_product Tablet 100 Table customer",
"Blacklisted tables customer will be removed from:",
" Keyspace product Shard 0 Tablet 100",
"Delete reverse vreplication streams on source:",
Expand Down
15 changes: 9 additions & 6 deletions go/vt/wrangler/switcher_dry_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,14 +234,17 @@ func (dr *switcherDryRun) removeSourceTables(ctx context.Context, removalType Ta
logs := make([]string, 0)
for _, source := range dr.ts.sources {
for _, tableName := range dr.ts.tables {
logs = append(logs, fmt.Sprintf("\tKeyspace %s Shard %s DbName %s Tablet %d Table %s RemovalType %s",
source.master.Keyspace, source.master.Shard, source.master.DbName(), source.master.Alias.Uid, tableName,
removalType))
logs = append(logs, fmt.Sprintf("\tKeyspace %s Shard %s DbName %s Tablet %d Table %s",
source.master.Keyspace, source.master.Shard, source.master.DbName(), source.master.Alias.Uid, tableName))
}
}
action := "Dropping"
if removalType == RenameTable {
action = "Renaming"
}
if len(logs) > 0 {
dr.drLog.Log(fmt.Sprintf("Dropping following tables from the database and from the vschema for keyspace %s:",
dr.ts.sourceKeyspace))
dr.drLog.Log(fmt.Sprintf("%s these tables from the database and removing them from the vschema for keyspace %s:",
action, dr.ts.sourceKeyspace))
dr.drLog.LogSlice(logs)
}
return nil
Expand Down Expand Up @@ -335,7 +338,7 @@ func (dr *switcherDryRun) removeTargetTables(ctx context.Context) error {
}
}
if len(logs) > 0 {
dr.drLog.Log(fmt.Sprintf("Dropping following tables from the database and from the vschema for keyspace %s:",
dr.drLog.Log(fmt.Sprintf("Dropping these tables from the database and removing from the vschema for keyspace %s:",
dr.ts.targetKeyspace))
dr.drLog.LogSlice(logs)
}
Expand Down
13 changes: 10 additions & 3 deletions go/vt/wrangler/traffic_switcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"vitess.io/vitess/go/vt/log"

"context"

"github.com/golang/protobuf/proto"
"vitess.io/vitess/go/sqltypes"
"vitess.io/vitess/go/vt/binlog/binlogplayer"
Expand All @@ -49,6 +50,8 @@ import (
const (
frozenStr = "FROZEN"
errorNoStreams = "no streams found in keyspace %s for: %s"
// use pt-osc's naming convention, this format also ensures vstreamer ignores such tables
renameTableTemplate = "_%.59s_old" // limit table name to 64 characters
)

// TrafficSwitchDirection specifies the switching direction.
Expand Down Expand Up @@ -1478,14 +1481,18 @@ func doValidateWorkflowHasCompleted(ctx context.Context, ts *trafficSwitcher) er

}

func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType TableRemovalType) (err error) {
err = ts.forAllSources(func(source *tsSource) error {
func getRenameFileName(tableName string) string {
return fmt.Sprintf(renameTableTemplate, tableName)
}

func (ts *trafficSwitcher) removeSourceTables(ctx context.Context, removalType TableRemovalType) error {
err := ts.forAllSources(func(source *tsSource) error {
for _, tableName := range ts.tables {
query := fmt.Sprintf("drop table %s.%s", source.master.DbName(), tableName)
if removalType == DropTable {
ts.wr.Logger().Infof("Dropping table %s.%s\n", source.master.DbName(), tableName)
} else {
renameName := fmt.Sprintf("_%.63s", tableName)
renameName := getRenameFileName(tableName)
ts.wr.Logger().Infof("Renaming table %s.%s to %s.%s\n", source.master.DbName(), tableName, source.master.DbName(), renameName)
query = fmt.Sprintf("rename table %s.%s TO %s.%s", source.master.DbName(), tableName, source.master.DbName(), renameName)
}
Expand Down
19 changes: 10 additions & 9 deletions go/vt/wrangler/traffic_switcher_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/stretchr/testify/require"

"context"

"vitess.io/vitess/go/sqltypes"
binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata"
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
Expand Down Expand Up @@ -857,9 +858,9 @@ func TestTableMigrateOneToMany(t *testing.T) {
wantdryRunDropSources := []string{
"Lock keyspace ks1",
"Lock keyspace ks2",
"Dropping following tables from the database and from the vschema for keyspace ks1:",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1 RemovalType DROP TABLE",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2 RemovalType DROP TABLE",
"Dropping these tables from the database and removing them from the vschema for keyspace ks1:",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2",
"Blacklisted tables t1,t2 will be removed from:",
" Keyspace ks1 Shard 0 Tablet 10",
"Delete reverse vreplication streams on source:",
Expand All @@ -884,9 +885,9 @@ func TestTableMigrateOneToMany(t *testing.T) {
wantdryRunRenameSources := []string{
"Lock keyspace ks1",
"Lock keyspace ks2",
"Dropping following tables from the database and from the vschema for keyspace ks1:",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1 RemovalType RENAME TABLE",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2 RemovalType RENAME TABLE",
"Renaming these tables from the database and removing them from the vschema for keyspace ks1:", " " +
"Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t1",
" Keyspace ks1 Shard 0 DbName vt_ks1 Tablet 10 Table t2",
"Blacklisted tables t1,t2 will be removed from:",
" Keyspace ks1 Shard 0 Tablet 10",
"Delete reverse vreplication streams on source:",
Expand All @@ -907,14 +908,14 @@ func TestTableMigrateOneToMany(t *testing.T) {
tme.dbTargetClients[0].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil)
tme.dbTargetClients[1].addQuery("select 1 from _vt.vreplication where db_name='vt_ks2' and workflow='test' and message!='FROZEN'", &sqltypes.Result{}, nil)
tme.dbSourceClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks1' and workflow = 'test_reverse'", &sqltypes.Result{}, nil)
tme.tmeDB.AddQuery("drop table vt_ks1.t1", &sqltypes.Result{})
tme.tmeDB.AddQuery("drop table vt_ks1.t2", &sqltypes.Result{})
tme.tmeDB.AddQuery(fmt.Sprintf("rename table vt_ks1.t1 TO vt_ks1.%s", getRenameFileName("t1")), &sqltypes.Result{})
tme.tmeDB.AddQuery(fmt.Sprintf("rename table vt_ks1.t2 TO vt_ks1.%s", getRenameFileName("t2")), &sqltypes.Result{})
tme.dbTargetClients[0].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", &sqltypes.Result{}, nil) //
tme.dbTargetClients[1].addQuery("select id from _vt.vreplication where db_name = 'vt_ks2' and workflow = 'test'", &sqltypes.Result{}, nil)
}
dropSources()

_, err = tme.wr.DropSources(ctx, tme.targetKeyspace, "test", DropTable, false, false, false)
_, err = tme.wr.DropSources(ctx, tme.targetKeyspace, "test", RenameTable, false, false, false)
require.NoError(t, err)
checkBlacklist(t, tme.ts, fmt.Sprintf("%s:%s", "ks1", "0"), nil)

Expand Down