From 8f52192267420e2cce49ccc82c75fb11c3cb9839 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 2 Nov 2020 10:02:13 -0500 Subject: [PATCH 1/5] Add option to GetSchema to only send the row count and data length over the wire Signed-off-by: Andrew Mason --- go/vt/vtctl/vtctl.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 9137fbe179d..7098664ead4 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -117,6 +117,7 @@ import ( "vitess.io/vitess/go/vt/wrangler" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" + "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" @@ -2296,6 +2297,8 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag excludeTables := subFlags.String("exclude_tables", "", "Specifies a comma-separated list of tables to exclude. Each is either an exact match, or a regular expression of the form /regexp/") includeViews := subFlags.Bool("include-views", false, "Includes views in the output") tableNamesOnly := subFlags.Bool("table_names_only", false, "Only displays table names that match") + tableSizesOnly := subFlags.Bool("table_sizes_only", false, "Only displays size information for tables") + if err := subFlags.Parse(args); err != nil { return err } @@ -2325,6 +2328,21 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag } return nil } + + if *tableSizesOnly { + sizeTds := make([]*tabletmanagerdata.TableDefinition, len(sd.TableDefinitions)) + for i, td := range sd.TableDefinitions { + sizeTds[i] = &tabletmanagerdata.TableDefinition{ + Name: td.Name, + Type: td.Type, + RowCount: td.RowCount, + DataLength: td.DataLength, + } + } + + sd.TableDefinitions = sizeTds + } + return printJSON(wr.Logger(), sd) } From b1966bdc2e44e31a4f28acd2db2fa7bb4e629159 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 2 Nov 2020 17:44:58 -0500 Subject: [PATCH 2/5] Update help text to clarify behavior Signed-off-by: Andrew Mason --- go/vt/vtctl/vtctl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 7098664ead4..5f7f5f94f14 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -2297,7 +2297,7 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag excludeTables := subFlags.String("exclude_tables", "", "Specifies a comma-separated list of tables to exclude. Each is either an exact match, or a regular expression of the form /regexp/") includeViews := subFlags.Bool("include-views", false, "Includes views in the output") tableNamesOnly := subFlags.Bool("table_names_only", false, "Only displays table names that match") - tableSizesOnly := subFlags.Bool("table_sizes_only", false, "Only displays size information for tables") + tableSizesOnly := subFlags.Bool("table_sizes_only", false, "Only displays size information for tables. Ignored if -table_names_only is passed.") if err := subFlags.Parse(args); err != nil { return err From cfac839515534a04e7739f0d1d10c5489edb385d Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 27 Nov 2020 11:09:00 -0500 Subject: [PATCH 3/5] Add endtoend test for vtctl GetSchema Signed-off-by: Andrew Mason --- go/vt/vtctl/endtoend/get_schema_test.go | 233 ++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 go/vt/vtctl/endtoend/get_schema_test.go diff --git a/go/vt/vtctl/endtoend/get_schema_test.go b/go/vt/vtctl/endtoend/get_schema_test.go new file mode 100644 index 00000000000..73e93201a13 --- /dev/null +++ b/go/vt/vtctl/endtoend/get_schema_test.go @@ -0,0 +1,233 @@ +package endtoend + +import ( + "context" + "fmt" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/json2" + "vitess.io/vitess/go/vt/logutil" + "vitess.io/vitess/go/vt/proto/query" + "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/memorytopo" + "vitess.io/vitess/go/vt/topo/topoproto" + "vitess.io/vitess/go/vt/vtctl" + "vitess.io/vitess/go/vt/vttablet/faketmclient" + "vitess.io/vitess/go/vt/vttablet/tmclient" + "vitess.io/vitess/go/vt/wrangler" +) + +type fakeTabletManagerClient struct { + tmclient.TabletManagerClient + schemas map[string]*tabletmanagerdata.SchemaDefinition +} + +func newTMClient() *fakeTabletManagerClient { + return &fakeTabletManagerClient{ + TabletManagerClient: faketmclient.NewFakeTabletManagerClient(), + schemas: map[string]*tabletmanagerdata.SchemaDefinition{}, + } +} + +func (c *fakeTabletManagerClient) GetSchema(ctx context.Context, tablet *topodata.Tablet, tablets []string, excludeTables []string, includeViews bool) (*tabletmanagerdata.SchemaDefinition, error) { + key := topoproto.TabletAliasString(tablet.Alias) + + schema, ok := c.schemas[key] + if !ok { + return nil, fmt.Errorf("no schemas for %s", key) + } + + return schema, nil +} + +func TestGetSchema(t *testing.T) { + ctx := context.Background() + + topo := memorytopo.NewServer("zone1", "zone2", "zone3") + + tablet := &topodata.Tablet{ + Alias: &topodata.TabletAlias{ + Cell: "zone1", + Uid: uuid.New().ID(), + }, + Hostname: "abcd", + Keyspace: "testkeyspace", + Shard: "-", + Type: topodata.TabletType_MASTER, + } + require.NoError(t, topo.CreateTablet(ctx, tablet)) + + sd := &tabletmanagerdata.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdata.TableDefinition{ + { + Name: "foo", + RowCount: 1000, + DataLength: 1000000, + Schema: `CREATE TABLE foo ( + id INT(11) NOT NULL, + name VARCHAR(255) NOT NULL, + PRIMARY KEY(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`, + Columns: []string{ + "id", + "name", + }, + PrimaryKeyColumns: []string{ + "id", + }, + Fields: []*query.Field{ + { + Name: "id", + Type: query.Type_INT32, + Table: "foo", + OrgTable: "foo", + Database: "vt_testkeyspace", + OrgName: "id", + ColumnLength: 11, + Charset: 63, + Decimals: 0, + }, + { + Name: "name", + Type: query.Type_VARCHAR, + Table: "foo", + OrgTable: "foo", + Database: "vt_testkeyspace", + OrgName: "name", + ColumnLength: 1020, + Charset: 45, + Decimals: 0, + }, + }, + }, + { + Name: "bar", + RowCount: 1, + DataLength: 10, + Schema: `CREATE TABLE bar ( + id INT(11) NOT NULL + foo_id INT(11) NOT NULL + is_active TINYINT(1) NOT NULL DEFAULT 1 +) ENGINE=InnoDB`, + Columns: []string{ + "id", + "foo_id", + "is_active", + }, + PrimaryKeyColumns: []string{ + "id", + }, + Fields: []*query.Field{ + { + Name: "id", + Type: query.Type_INT32, + Table: "bar", + OrgTable: "bar", + Database: "vt_testkeyspace", + OrgName: "id", + ColumnLength: 11, + Charset: 63, + Decimals: 0, + }, + { + Name: "foo_id", + Type: query.Type_INT32, + Table: "bar", + OrgTable: "bar", + Database: "vt_testkeyspace", + OrgName: "foo_id", + ColumnLength: 11, + Charset: 63, + Decimals: 0, + }, + { + Name: "is_active", + Type: query.Type_INT8, + Table: "bar", + OrgTable: "bar", + Database: "vt_testkeyspace", + OrgName: "is_active", + ColumnLength: 1, + Charset: 63, + Decimals: 0, + }, + }, + }, + }, + } + + tmc := newTMClient() + tmc.schemas[topoproto.TabletAliasString(tablet.Alias)] = sd + + logger := logutil.NewMemoryLogger() + + err := vtctl.RunCommand(ctx, wrangler.New(logger, topo, tmc), []string{ + "GetSchema", + topoproto.TabletAliasString(tablet.Alias), + }) + require.NoError(t, err) + + events := logger.Events + assert.Equal(t, 1, len(events), "expected 1 event from GetSchema") + val := events[0].Value + + actual := &tabletmanagerdata.SchemaDefinition{} + err = json2.Unmarshal([]byte(val), actual) + require.NoError(t, err) + + assert.Equal(t, sd, actual) + + // reset for the next invocation, where we verify that passing + // -table_sizes_only does not include the create table statement or columns. + logger.Events = nil + sd = &tabletmanagerdata.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdata.TableDefinition{ + { + Name: "foo", + RowCount: 1000, + DataLength: 1000000, + Columns: []string{}, + PrimaryKeyColumns: []string{}, + Fields: []*query.Field{}, + }, + { + Name: "bar", + RowCount: 1, + DataLength: 10, + Columns: []string{}, + PrimaryKeyColumns: []string{}, + Fields: []*query.Field{}, + }, + }, + } + + err = vtctl.RunCommand(ctx, wrangler.New(logger, topo, tmc), []string{ + "GetSchema", + "-table_sizes_only", + topoproto.TabletAliasString(tablet.Alias), + }) + require.NoError(t, err) + + events = logger.Events + assert.Equal(t, 1, len(events), "expected 1 event from GetSchema") + val = events[0].Value + + actual = &tabletmanagerdata.SchemaDefinition{} + err = json2.Unmarshal([]byte(val), actual) + require.NoError(t, err) + + assert.Equal(t, sd, actual) +} + +func init() { + // enforce we will use the right protocol (gRPC) (note the + // client is unused, but it is initialized, so it needs to exist) + *tmclient.TabletManagerProtocol = "grpc" + tmclient.RegisterTabletManagerClientFactory("grpc", func() tmclient.TabletManagerClient { + return nil + }) +} From d3e505320d458e973d36379460374dd77d1c294f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 27 Nov 2020 19:17:12 -0500 Subject: [PATCH 4/5] Alias the proto imports Signed-off-by: Andrew Mason --- go/vt/vtctl/endtoend/get_schema_test.go | 48 ++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/go/vt/vtctl/endtoend/get_schema_test.go b/go/vt/vtctl/endtoend/get_schema_test.go index 73e93201a13..9591ecc15ee 100644 --- a/go/vt/vtctl/endtoend/get_schema_test.go +++ b/go/vt/vtctl/endtoend/get_schema_test.go @@ -10,9 +10,9 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/json2" "vitess.io/vitess/go/vt/logutil" - "vitess.io/vitess/go/vt/proto/query" - "vitess.io/vitess/go/vt/proto/tabletmanagerdata" - "vitess.io/vitess/go/vt/proto/topodata" + querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl" @@ -23,17 +23,17 @@ import ( type fakeTabletManagerClient struct { tmclient.TabletManagerClient - schemas map[string]*tabletmanagerdata.SchemaDefinition + schemas map[string]*tabletmanagerdatapb.SchemaDefinition } func newTMClient() *fakeTabletManagerClient { return &fakeTabletManagerClient{ TabletManagerClient: faketmclient.NewFakeTabletManagerClient(), - schemas: map[string]*tabletmanagerdata.SchemaDefinition{}, + schemas: map[string]*tabletmanagerdatapb.SchemaDefinition{}, } } -func (c *fakeTabletManagerClient) GetSchema(ctx context.Context, tablet *topodata.Tablet, tablets []string, excludeTables []string, includeViews bool) (*tabletmanagerdata.SchemaDefinition, error) { +func (c *fakeTabletManagerClient) GetSchema(ctx context.Context, tablet *topodatapb.Tablet, tablets []string, excludeTables []string, includeViews bool) (*tabletmanagerdatapb.SchemaDefinition, error) { key := topoproto.TabletAliasString(tablet.Alias) schema, ok := c.schemas[key] @@ -49,20 +49,20 @@ func TestGetSchema(t *testing.T) { topo := memorytopo.NewServer("zone1", "zone2", "zone3") - tablet := &topodata.Tablet{ - Alias: &topodata.TabletAlias{ + tablet := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ Cell: "zone1", Uid: uuid.New().ID(), }, Hostname: "abcd", Keyspace: "testkeyspace", Shard: "-", - Type: topodata.TabletType_MASTER, + Type: topodatapb.TabletType_MASTER, } require.NoError(t, topo.CreateTablet(ctx, tablet)) - sd := &tabletmanagerdata.SchemaDefinition{ - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + sd := &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "foo", RowCount: 1000, @@ -79,10 +79,10 @@ func TestGetSchema(t *testing.T) { PrimaryKeyColumns: []string{ "id", }, - Fields: []*query.Field{ + Fields: []*querypb.Field{ { Name: "id", - Type: query.Type_INT32, + Type: querypb.Type_INT32, Table: "foo", OrgTable: "foo", Database: "vt_testkeyspace", @@ -93,7 +93,7 @@ func TestGetSchema(t *testing.T) { }, { Name: "name", - Type: query.Type_VARCHAR, + Type: querypb.Type_VARCHAR, Table: "foo", OrgTable: "foo", Database: "vt_testkeyspace", @@ -121,10 +121,10 @@ func TestGetSchema(t *testing.T) { PrimaryKeyColumns: []string{ "id", }, - Fields: []*query.Field{ + Fields: []*querypb.Field{ { Name: "id", - Type: query.Type_INT32, + Type: querypb.Type_INT32, Table: "bar", OrgTable: "bar", Database: "vt_testkeyspace", @@ -135,7 +135,7 @@ func TestGetSchema(t *testing.T) { }, { Name: "foo_id", - Type: query.Type_INT32, + Type: querypb.Type_INT32, Table: "bar", OrgTable: "bar", Database: "vt_testkeyspace", @@ -146,7 +146,7 @@ func TestGetSchema(t *testing.T) { }, { Name: "is_active", - Type: query.Type_INT8, + Type: querypb.Type_INT8, Table: "bar", OrgTable: "bar", Database: "vt_testkeyspace", @@ -175,7 +175,7 @@ func TestGetSchema(t *testing.T) { assert.Equal(t, 1, len(events), "expected 1 event from GetSchema") val := events[0].Value - actual := &tabletmanagerdata.SchemaDefinition{} + actual := &tabletmanagerdatapb.SchemaDefinition{} err = json2.Unmarshal([]byte(val), actual) require.NoError(t, err) @@ -184,15 +184,15 @@ func TestGetSchema(t *testing.T) { // reset for the next invocation, where we verify that passing // -table_sizes_only does not include the create table statement or columns. logger.Events = nil - sd = &tabletmanagerdata.SchemaDefinition{ - TableDefinitions: []*tabletmanagerdata.TableDefinition{ + sd = &tabletmanagerdatapb.SchemaDefinition{ + TableDefinitions: []*tabletmanagerdatapb.TableDefinition{ { Name: "foo", RowCount: 1000, DataLength: 1000000, Columns: []string{}, PrimaryKeyColumns: []string{}, - Fields: []*query.Field{}, + Fields: []*querypb.Field{}, }, { Name: "bar", @@ -200,7 +200,7 @@ func TestGetSchema(t *testing.T) { DataLength: 10, Columns: []string{}, PrimaryKeyColumns: []string{}, - Fields: []*query.Field{}, + Fields: []*querypb.Field{}, }, }, } @@ -216,7 +216,7 @@ func TestGetSchema(t *testing.T) { assert.Equal(t, 1, len(events), "expected 1 event from GetSchema") val = events[0].Value - actual = &tabletmanagerdata.SchemaDefinition{} + actual = &tabletmanagerdatapb.SchemaDefinition{} err = json2.Unmarshal([]byte(val), actual) require.NoError(t, err) From 332d37c8a0d11febe7d8f67400017eb667493123 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 27 Nov 2020 21:08:19 -0500 Subject: [PATCH 5/5] Format imports to match vtctl.go Signed-off-by: Andrew Mason --- go/vt/vtctl/endtoend/get_schema_test.go | 8 +++++--- go/vt/vtctl/vtctl.go | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go/vt/vtctl/endtoend/get_schema_test.go b/go/vt/vtctl/endtoend/get_schema_test.go index 9591ecc15ee..ddb0c204048 100644 --- a/go/vt/vtctl/endtoend/get_schema_test.go +++ b/go/vt/vtctl/endtoend/get_schema_test.go @@ -8,17 +8,19 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/json2" "vitess.io/vitess/go/vt/logutil" - querypb "vitess.io/vitess/go/vt/proto/query" - tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/topo/memorytopo" "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vtctl" "vitess.io/vitess/go/vt/vttablet/faketmclient" "vitess.io/vitess/go/vt/vttablet/tmclient" "vitess.io/vitess/go/vt/wrangler" + + querypb "vitess.io/vitess/go/vt/proto/query" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) type fakeTabletManagerClient struct { diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 5f7f5f94f14..1b0ec698034 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -117,7 +117,7 @@ import ( "vitess.io/vitess/go/vt/wrangler" replicationdatapb "vitess.io/vitess/go/vt/proto/replicationdata" - "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" vschemapb "vitess.io/vitess/go/vt/proto/vschema" vtctldatapb "vitess.io/vitess/go/vt/proto/vtctldata" @@ -2330,9 +2330,9 @@ func commandGetSchema(ctx context.Context, wr *wrangler.Wrangler, subFlags *flag } if *tableSizesOnly { - sizeTds := make([]*tabletmanagerdata.TableDefinition, len(sd.TableDefinitions)) + sizeTds := make([]*tabletmanagerdatapb.TableDefinition, len(sd.TableDefinitions)) for i, td := range sd.TableDefinitions { - sizeTds[i] = &tabletmanagerdata.TableDefinition{ + sizeTds[i] = &tabletmanagerdatapb.TableDefinition{ Name: td.Name, Type: td.Type, RowCount: td.RowCount,