From a091f6ab3ff3b0946b254df3b35e638eb6209835 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 4 Feb 2021 12:44:16 +0530 Subject: [PATCH 01/14] added file size and allocated size for each table Signed-off-by: Harshit Gangal --- go/mysql/schema.go | 4 +++- go/vt/vttablet/tabletserver/schema/engine.go | 4 +++- go/vt/vttablet/tabletserver/schema/load_table.go | 2 +- go/vt/vttablet/tabletserver/schema/load_table_test.go | 2 +- go/vt/vttablet/tabletserver/schema/schema.go | 3 +++ 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 3174dad7274..9ebbd2411ac 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -31,7 +31,9 @@ import ( const ( // BaseShowTables is the base query used in further methods. - BaseShowTables = "SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()" + BaseShowTables = "SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size " + + "FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i " + + "WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)" // BaseShowPrimary is the base query for fetching primary key info. BaseShowPrimary = "SELECT table_name, column_name FROM information_schema.key_column_usage WHERE table_schema=database() AND constraint_name='PRIMARY' ORDER BY table_name, ordinal_position" diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index f49229c9ca7..33f653eda6c 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -327,11 +327,13 @@ func (se *Engine) reload(ctx context.Context) error { continue } log.V(2).Infof("Reading schema for table: %s", tableName) - table, err := LoadTable(conn, tableName, row[1].ToString(), row[3].ToString()) + table, err := LoadTable(conn, tableName, row[3].ToString()) if err != nil { rec.RecordError(err) continue } + table.FileSize, _ = evalengine.ToUint64(row[4]) + table.AllocatedSize, _ = evalengine.ToUint64(row[5]) changedTables[tableName] = table if _, ok := se.tables[tableName]; ok { altered = append(altered, tableName) diff --git a/go/vt/vttablet/tabletserver/schema/load_table.go b/go/vt/vttablet/tabletserver/schema/load_table.go index ce3b2f3f881..3d32dbd9075 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table.go +++ b/go/vt/vttablet/tabletserver/schema/load_table.go @@ -28,7 +28,7 @@ import ( ) // LoadTable creates a Table from the schema info in the database. -func LoadTable(conn *connpool.DBConn, tableName string, tableType string, comment string) (*Table, error) { +func LoadTable(conn *connpool.DBConn, tableName string, comment string) (*Table, error) { ta := NewTable(tableName) sqlTableName := sqlparser.String(ta.Name) if err := fetchColumns(ta, conn, sqlTableName); err != nil { diff --git a/go/vt/vttablet/tabletserver/schema/load_table_test.go b/go/vt/vttablet/tabletserver/schema/load_table_test.go index 25c0924b693..8d7e784ff0b 100644 --- a/go/vt/vttablet/tabletserver/schema/load_table_test.go +++ b/go/vt/vttablet/tabletserver/schema/load_table_test.go @@ -173,7 +173,7 @@ func newTestLoadTable(tableType string, comment string, db *fakesqldb.DB) (*Tabl } defer conn.Recycle() - return LoadTable(conn, "test_table", tableType, comment) + return LoadTable(conn, "test_table", comment) } func getTestLoadTableQueries() map[string]*sqltypes.Result { diff --git a/go/vt/vttablet/tabletserver/schema/schema.go b/go/vt/vttablet/tabletserver/schema/schema.go index ebf622bddb6..69aae6387ac 100644 --- a/go/vt/vttablet/tabletserver/schema/schema.go +++ b/go/vt/vttablet/tabletserver/schema/schema.go @@ -52,6 +52,9 @@ type Table struct { // MessageInfo contains info for message tables. MessageInfo *MessageInfo + + FileSize uint64 + AllocatedSize uint64 } // SequenceInfo contains info specific to sequence tabels. From 308f7bdcb8b4d9b4e0f122e76b3aec590d98a5b2 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 4 Feb 2021 13:46:35 +0530 Subject: [PATCH 02/14] always update the table size on schema reload Signed-off-by: Harshit Gangal --- go/mysql/schema.go | 2 +- go/vt/vttablet/tabletserver/schema/engine.go | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 9ebbd2411ac..3f73cd82c02 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -32,7 +32,7 @@ import ( const ( // BaseShowTables is the base query used in further methods. BaseShowTables = "SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size " + - "FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i " + + "FROM information_schema.tables t, information_schema.innodb_tablespaces i " + "WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)" // BaseShowPrimary is the base query for fetching primary key info. diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 33f653eda6c..4a52353faac 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -321,19 +321,25 @@ func (se *Engine) reload(ctx context.Context) error { tableName := row[0].ToString() curTables[tableName] = true createTime, _ := evalengine.ToInt64(row[2]) + fileSize, _ := evalengine.ToUint64(row[4]) + allocatedSize, _ := evalengine.ToUint64(row[5]) + // TODO(sougou); find a better way detect changed tables. This method // seems unreliable. The endtoend test flags all tables as changed. - if _, ok := se.tables[tableName]; ok && createTime < se.lastChange { + if t, ok := se.tables[tableName]; ok && createTime < se.lastChange { + t.FileSize = fileSize + t.AllocatedSize = allocatedSize continue } + log.V(2).Infof("Reading schema for table: %s", tableName) table, err := LoadTable(conn, tableName, row[3].ToString()) if err != nil { rec.RecordError(err) continue } - table.FileSize, _ = evalengine.ToUint64(row[4]) - table.AllocatedSize, _ = evalengine.ToUint64(row[5]) + table.FileSize = fileSize + table.AllocatedSize = allocatedSize changedTables[tableName] = table if _, ok := se.tables[tableName]; ok { altered = append(altered, tableName) From 7ddbf2ddfa4a50f61546e8b35663c0529011de4f Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Thu, 4 Feb 2021 17:12:44 +0100 Subject: [PATCH 03/14] schema query needs to be different betwen mysql5.7 and mysql8.0 Signed-off-by: Andres Taylor --- go/mysql/fakesqldb/server.go | 19 +++- go/mysql/flavor.go | 31 +++--- go/mysql/flavor_filepos.go | 5 + go/mysql/flavor_mariadb_binlog_playback.go | 5 + go/mysql/flavor_mysql.go | 23 ++++ go/mysql/flavor_mysql_test.go | 4 +- go/mysql/schema.go | 98 +++++++++-------- go/vt/vtexplain/vtexplain_vttablet.go | 2 +- .../vttablet/tabletserver/connpool/dbconn.go | 5 + .../tabletserver/query_engine_test.go | 12 +++ .../tabletserver/query_executor_test.go | 35 +++--- go/vt/vttablet/tabletserver/schema/engine.go | 22 ++-- .../tabletserver/schema/engine_test.go | 102 ++++++++++++------ .../vttablet/tabletserver/schema/main_test.go | 5 +- .../schema/schematest/schematest.go | 10 -- .../tabletserver/tabletserver_test.go | 15 +-- 16 files changed, 250 insertions(+), 143 deletions(-) diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go index 71bf08f06c8..82c1e320c06 100644 --- a/go/mysql/fakesqldb/server.go +++ b/go/mysql/fakesqldb/server.go @@ -136,6 +136,7 @@ type ExpectedResult struct { type exprResult struct { expr *regexp.Regexp result *sqltypes.Result + err string } // ExpectedExecuteFetch defines for an expected query the to be faked output. @@ -391,6 +392,9 @@ func (db *DB) HandleQuery(c *mysql.Conn, query string, callback func(*sqltypes.R if ok { userCallback(query) } + if pat.err != "" { + return fmt.Errorf(pat.err) + } return callback(pat.result) } } @@ -504,7 +508,20 @@ func (db *DB) AddQueryPattern(queryPattern string, expectedResult *sqltypes.Resu result := *expectedResult db.mu.Lock() defer db.mu.Unlock() - db.patternData = append(db.patternData, exprResult{expr, &result}) + db.patternData = append(db.patternData, exprResult{expr: expr, result: &result}) +} + +// RejectQueryPattern allows a query pattern to be rejected with an error +func (db *DB) RejectQueryPattern(queryPattern, error string) { + expr := regexp.MustCompile("(?is)^" + queryPattern + "$") + db.mu.Lock() + defer db.mu.Unlock() + db.patternData = append(db.patternData, exprResult{expr: expr, err: error}) +} + +// ClearQueryPattern removes all query patterns set up +func (db *DB) ClearQueryPattern() { + db.patternData = nil } // AddQueryPatternWithCallback is similar to AddQueryPattern: in addition it calls the provided callback function diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index 86824be43ca..e4a3c82261d 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -44,6 +44,8 @@ const ( mariaDBReplicationHackPrefix = "5.5.5-" // mariaDBVersionString is present in mariaDBVersionString = "MariaDB" + // mysql57VersionPrefix is the prefix for 5.7 mysql version, such as 5.7.31-log + mysql80VersionPrefix = "8.0." ) // flavor is the abstract interface for a flavor. @@ -111,6 +113,8 @@ type flavor interface { // timestamp cannot be set by regular clients. enableBinlogPlaybackCommand() string disableBinlogPlaybackCommand() string + + baseShowTablesWithSizes() string } // flavors maps flavor names to their implementation. @@ -131,23 +135,21 @@ var flavors = make(map[string]func() flavor) // as well (not matching what c.ServerVersion is, but matching after we remove // the prefix). func (c *Conn) fillFlavor(params *ConnParams) { - if flavorFunc := flavors[params.Flavor]; flavorFunc != nil { - c.flavor = flavorFunc() - return - } + flavorFunc := flavors[params.Flavor] - if strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix) { + switch { + case flavorFunc != nil: + c.flavor = flavorFunc() + case strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix): c.ServerVersion = c.ServerVersion[len(mariaDBReplicationHackPrefix):] c.flavor = mariadbFlavor{} - return - } - - if strings.Contains(c.ServerVersion, mariaDBVersionString) { + case strings.Contains(c.ServerVersion, mariaDBVersionString): c.flavor = mariadbFlavor{} - return + case strings.HasPrefix(c.ServerVersion, mysql80VersionPrefix): + c.flavor = mysqlFlavor80{} + default: + c.flavor = mysqlFlavor57{} } - - c.flavor = mysqlFlavor{} } // @@ -390,3 +392,8 @@ func (c *Conn) EnableBinlogPlaybackCommand() string { func (c *Conn) DisableBinlogPlaybackCommand() string { return c.flavor.disableBinlogPlaybackCommand() } + +// BaseShowTables returns a query that shows tables and their sizes +func (c *Conn) BaseShowTables() string { + return c.flavor.baseShowTablesWithSizes() +} diff --git a/go/mysql/flavor_filepos.go b/go/mysql/flavor_filepos.go index 134de8ba707..05377dee0be 100644 --- a/go/mysql/flavor_filepos.go +++ b/go/mysql/flavor_filepos.go @@ -271,3 +271,8 @@ func (*filePosFlavor) enableBinlogPlaybackCommand() string { func (*filePosFlavor) disableBinlogPlaybackCommand() string { return "" } + +// baseShowTablesWithSizes is part of the Flavor interface. +func (*filePosFlavor) baseShowTablesWithSizes() string { + return "" +} diff --git a/go/mysql/flavor_mariadb_binlog_playback.go b/go/mysql/flavor_mariadb_binlog_playback.go index c30e39d2787..53cdd2d4173 100644 --- a/go/mysql/flavor_mariadb_binlog_playback.go +++ b/go/mysql/flavor_mariadb_binlog_playback.go @@ -29,3 +29,8 @@ func (mariadbFlavor) enableBinlogPlaybackCommand() string { func (mariadbFlavor) disableBinlogPlaybackCommand() string { return "" } + +// baseShowTablesWithSizes is part of the Flavor interface. +func (mariadbFlavor) baseShowTablesWithSizes() string { + return "" +} diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 590cc3ed2d5..5cf81a8b26f 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -29,6 +29,15 @@ import ( // mysqlFlavor implements the Flavor interface for Mysql. type mysqlFlavor struct{} +type mysqlFlavor57 struct { + mysqlFlavor +} +type mysqlFlavor80 struct { + mysqlFlavor +} + +var _ flavor = (*mysqlFlavor57)(nil) +var _ flavor = (*mysqlFlavor80)(nil) // masterGTIDSet is part of the Flavor interface. func (mysqlFlavor) masterGTIDSet(c *Conn) (GTIDSet, error) { @@ -231,3 +240,17 @@ func (mysqlFlavor) enableBinlogPlaybackCommand() string { func (mysqlFlavor) disableBinlogPlaybackCommand() string { return "" } + +// baseShowTablesWithSizes is part of the Flavor interface. +func (mysqlFlavor57) baseShowTablesWithSizes() string { + return `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size + FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i + WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` +} + +// baseShowTablesWithSizes is part of the Flavor interface. +func (mysqlFlavor80) baseShowTablesWithSizes() string { + return `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size + FROM information_schema.tables t, information_schema.innodb_tablespaces i + WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` +} diff --git a/go/mysql/flavor_mysql_test.go b/go/mysql/flavor_mysql_test.go index 398c3d4e147..8f72242a891 100644 --- a/go/mysql/flavor_mysql_test.go +++ b/go/mysql/flavor_mysql_test.go @@ -39,7 +39,7 @@ func TestMysql56SetMasterCommands(t *testing.T) { MASTER_CONNECT_RETRY = 1234, MASTER_AUTO_POSITION = 1` - conn := &Conn{flavor: mysqlFlavor{}} + conn := &Conn{flavor: mysqlFlavor57{}} got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry) if got != want { t.Errorf("mysqlFlavor.SetMasterCommand(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want) @@ -72,7 +72,7 @@ func TestMysql56SetMasterCommandsSSL(t *testing.T) { MASTER_SSL_KEY = 'ssl-key', MASTER_AUTO_POSITION = 1` - conn := &Conn{flavor: mysqlFlavor{}} + conn := &Conn{flavor: mysqlFlavor57{}} got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry) if got != want { t.Errorf("mysqlFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want) diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 3f73cd82c02..0ea29da129a 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -30,11 +30,6 @@ import ( // data. const ( - // BaseShowTables is the base query used in further methods. - BaseShowTables = "SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size " + - "FROM information_schema.tables t, information_schema.innodb_tablespaces i " + - "WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)" - // BaseShowPrimary is the base query for fetching primary key info. BaseShowPrimary = "SELECT table_name, column_name FROM information_schema.key_column_usage WHERE table_schema=database() AND constraint_name='PRIMARY' ORDER BY table_name, ordinal_position" ) @@ -42,48 +37,55 @@ const ( // BaseShowTablesFields contains the fields returned by a BaseShowTables or a BaseShowTablesForTable command. // They are validated by the // testBaseShowTables test. -var BaseShowTablesFields = []*querypb.Field{ - { - Name: "table_name", - Type: querypb.Type_VARCHAR, - Table: "tables", - OrgTable: "TABLES", - Database: "information_schema", - OrgName: "TABLE_NAME", - ColumnLength: 192, - Charset: CharacterSetUtf8, - Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), - }, - { - Name: "table_type", - Type: querypb.Type_VARCHAR, - Table: "tables", - OrgTable: "TABLES", - Database: "information_schema", - OrgName: "TABLE_TYPE", - ColumnLength: 192, - Charset: CharacterSetUtf8, - Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), - }, - { - Name: "unix_timestamp(create_time)", - Type: querypb.Type_INT64, - ColumnLength: 11, - Charset: CharacterSetBinary, - Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), - }, - { - Name: "table_comment", - Type: querypb.Type_VARCHAR, - Table: "tables", - OrgTable: "TABLES", - Database: "information_schema", - OrgName: "TABLE_COMMENT", - ColumnLength: 6144, - Charset: CharacterSetUtf8, - Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), - }, -} +var BaseShowTablesFields = []*querypb.Field{{ + Name: "t.table_name", + Type: querypb.Type_VARCHAR, + Table: "tables", + OrgTable: "TABLES", + Database: "information_schema", + OrgName: "TABLE_NAME", + ColumnLength: 192, + Charset: CharacterSetUtf8, + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), +}, { + Name: "t.table_type", + Type: querypb.Type_VARCHAR, + Table: "tables", + OrgTable: "TABLES", + Database: "information_schema", + OrgName: "TABLE_TYPE", + ColumnLength: 192, + Charset: CharacterSetUtf8, + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), +}, { + Name: "unix_timestamp(t.create_time)", + Type: querypb.Type_INT64, + ColumnLength: 11, + Charset: CharacterSetBinary, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), +}, { + Name: "t.table_comment", + Type: querypb.Type_VARCHAR, + Table: "tables", + OrgTable: "TABLES", + Database: "information_schema", + OrgName: "TABLE_COMMENT", + ColumnLength: 6144, + Charset: CharacterSetUtf8, + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), +}, { + Name: "i.file_size", + Type: querypb.Type_INT64, + ColumnLength: 11, + Charset: CharacterSetBinary, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), +}, { + Name: "i.allocated_size", + Type: querypb.Type_INT64, + ColumnLength: 11, + Charset: CharacterSetBinary, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), +}} // BaseShowTablesRow returns the fields from a BaseShowTables or // BaseShowTablesForTable command. @@ -97,6 +99,8 @@ func BaseShowTablesRow(tableName string, isView bool, comment string) []sqltypes sqltypes.MakeTrusted(sqltypes.VarChar, []byte(tableType)), sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325875")), // unix_timestamp(create_time) sqltypes.MakeTrusted(sqltypes.VarChar, []byte(comment)), + sqltypes.MakeTrusted(sqltypes.Int64, []byte("100")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("150")), // allocated_size } } diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 34773864c79..d57ebe65132 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -382,7 +382,7 @@ func initTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) error { } showTableRows = append(showTableRows, mysql.BaseShowTablesRow(table, false, options)) } - schemaQueries[mysql.BaseShowTables] = &sqltypes.Result{ + schemaQueries["mysql.BaseShowTables"] = &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: showTableRows, } diff --git a/go/vt/vttablet/tabletserver/connpool/dbconn.go b/go/vt/vttablet/tabletserver/connpool/dbconn.go index fefd3ca5c19..a41e7a5039a 100644 --- a/go/vt/vttablet/tabletserver/connpool/dbconn.go +++ b/go/vt/vttablet/tabletserver/connpool/dbconn.go @@ -374,6 +374,11 @@ func (dbc *DBConn) ID() int64 { return dbc.conn.ID() } +// BaseShowTables returns a query that shows tables and their sizes +func (dbc *DBConn) BaseShowTables() string { + return dbc.conn.BaseShowTables() +} + func (dbc *DBConn) reconnect(ctx context.Context) error { dbc.conn.Close() // Reuse MySQLTimings from dbc.conn. diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 029fc363e31..6189297c67b 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -32,6 +32,9 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql" + + "context" "github.com/stretchr/testify/require" "vitess.io/vitess/go/cache" @@ -120,6 +123,15 @@ func TestGetMessageStreamPlan(t *testing.T) { for query, result := range schematest.Queries() { db.AddQuery(query, result) } + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }}) qe := newTestQueryEngine(10*time.Second, true, newDBConfigs(db)) qe.se.Open() qe.Open() diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index b293533c6ea..a6c5996230e 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -604,7 +604,6 @@ func TestQueryExecutorPlanNextval(t *testing.T) { sqltypes.NewInt64(7), sqltypes.NewInt64(3), }}, - RowsAffected: 2, }) updateQuery = "update seq set next_id = 13 where id = 0" db.AddQuery(updateQuery, &sqltypes.Result{}) @@ -1158,14 +1157,24 @@ func newTestQueryExecutor(ctx context.Context, tsv *TabletServer, sql string, tx func setUpQueryExecutorTest(t *testing.T) *fakesqldb.DB { db := fakesqldb.New(t) - initQueryExecutorTestDB(db, true) + initQueryExecutorTestDB(db) return db } -func initQueryExecutorTestDB(db *fakesqldb.DB, testTableHasMultipleUniqueKeys bool) { - for query, result := range getQueryExecutorSupportedQueries(testTableHasMultipleUniqueKeys) { +const baseShowTablesPattern = `SELECT t\.table_name.*` + +func initQueryExecutorTestDB(db *fakesqldb.DB) { + for query, result := range getQueryExecutorSupportedQueries() { db.AddQuery(query, result) } + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) } func getTestTableFields() []*querypb.Field { @@ -1176,7 +1185,7 @@ func getTestTableFields() []*querypb.Field { } } -func getQueryExecutorSupportedQueries(testTableHasMultipleUniqueKeys bool) map[string]*sqltypes.Result { +func getQueryExecutorSupportedQueries() map[string]*sqltypes.Result { return map[string]*sqltypes.Result{ // queries for twopc fmt.Sprintf(sqlCreateSidecarDB, "_vt"): {}, @@ -1238,24 +1247,13 @@ func getQueryExecutorSupportedQueries(testTableHasMultipleUniqueKeys bool) map[s Fields: []*querypb.Field{{ Type: sqltypes.Uint64, }}, - Rows: [][]sqltypes.Value{}, - RowsAffected: 0, + Rows: [][]sqltypes.Value{}, }, "(select 0 as x from dual where 1 != 1) union (select 1 as y from dual where 1 != 1) limit 10001": { Fields: []*querypb.Field{{ Type: sqltypes.Uint64, }}, - Rows: [][]sqltypes.Value{}, - RowsAffected: 0, - }, - mysql.BaseShowTables: { - Fields: mysql.BaseShowTablesFields, - Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesRow("test_table", false, ""), - mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), - }, - RowsAffected: 3, + Rows: [][]sqltypes.Value{}, }, mysql.BaseShowPrimary: { Fields: mysql.ShowPrimaryFields, @@ -1264,7 +1262,6 @@ func getQueryExecutorSupportedQueries(testTableHasMultipleUniqueKeys bool) map[s mysql.ShowPrimaryRow("seq", "id"), mysql.ShowPrimaryRow("msg", "id"), }, - RowsAffected: 3, }, "select * from test_table where 1 != 1": { Fields: []*querypb.Field{{ diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 4a52353faac..69bfe6ae2e0 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -305,7 +305,7 @@ func (se *Engine) reload(ctx context.Context) error { if se.SkipMetaCheck { return nil } - tableData, err := conn.Exec(ctx, mysql.BaseShowTables, maxTableCount, false) + tableData, err := conn.Exec(ctx, conn.BaseShowTables(), maxTableCount, false) if err != nil { return err } @@ -326,9 +326,10 @@ func (se *Engine) reload(ctx context.Context) error { // TODO(sougou); find a better way detect changed tables. This method // seems unreliable. The endtoend test flags all tables as changed. - if t, ok := se.tables[tableName]; ok && createTime < se.lastChange { - t.FileSize = fileSize - t.AllocatedSize = allocatedSize + tbl, isInTablesMap := se.tables[tableName] + if isInTablesMap && createTime < se.lastChange { + tbl.FileSize = fileSize + tbl.AllocatedSize = allocatedSize continue } @@ -341,7 +342,7 @@ func (se *Engine) reload(ctx context.Context) error { table.FileSize = fileSize table.AllocatedSize = allocatedSize changedTables[tableName] = table - if _, ok := se.tables[tableName]; ok { + if isInTablesMap { altered = append(altered, tableName) } else { created = append(created, tableName) @@ -354,11 +355,10 @@ func (se *Engine) reload(ctx context.Context) error { // Compute and handle dropped tables. var dropped []string for tableName := range se.tables { - if curTables[tableName] { - continue + if !curTables[tableName] { + dropped = append(dropped, tableName) + delete(se.tables, tableName) } - dropped = append(dropped, tableName) - delete(se.tables, tableName) } // Populate PKColumns for changed tables. @@ -520,10 +520,10 @@ func (se *Engine) handleDebugSchema(response http.ResponseWriter, request *http. acl.SendError(response, err) return } - se.handleHTTPSchema(response, request) + se.handleHTTPSchema(response) } -func (se *Engine) handleHTTPSchema(response http.ResponseWriter, request *http.Request) { +func (se *Engine) handleHTTPSchema(response http.ResponseWriter) { // Ensure schema engine is Open. If vttablet came up in a non_serving role, // the schema engine may not have been initialized. err := se.Open() diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index ab5dc20cf03..89382a7d7f4 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -18,7 +18,6 @@ package schema import ( "expvar" - "fmt" "net/http" "net/http/httptest" "sort" @@ -26,6 +25,8 @@ import ( "testing" "time" + "vitess.io/vitess/go/test/utils" + "context" "github.com/stretchr/testify/assert" @@ -42,12 +43,36 @@ import ( querypb "vitess.io/vitess/go/vt/proto/query" ) +const baseShowTablesPattern = `SELECT t\.table_name.*` + +var mustMatch = utils.MustMatchFn( + []interface{}{ // types with unexported fields + sqlparser.TableIdent{}, + }, + []string{".Mutex"}, // ignored fields +) + func TestOpenAndReload(t *testing.T) { db := fakesqldb.New(t) defer db.Close() for query, result := range schematest.Queries() { db.AddQuery(query, result) } + db.AddQueryPattern(baseShowTablesPattern, + &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + SessionStateChanges: "", + StatusFlags: 0, + }) // pre-advance to above the default 1427325875. db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( @@ -55,12 +80,12 @@ func TestOpenAndReload(t *testing.T) { "int64"), "1427325876", )) - se := newEngine(10, 10*time.Second, 10*time.Second, true, db) + se := newEngine(10, 10*time.Second, 10*time.Second, db) se.Open() defer se.Close() want := initialSchema() - assert.Equal(t, want, se.GetSchema()) + mustMatch(t, want, se.GetSchema()) // Advance time some more. db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( @@ -71,17 +96,19 @@ func TestOpenAndReload(t *testing.T) { // Modify test_table_03 // Add test_table_04 // Drop msg - db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + db.ClearQueryPattern() + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesRow("test_table_01", false, ""), mysql.BaseShowTablesRow("test_table_02", false, ""), { - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")), - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), - // Match the timestamp. - sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), - sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("test_table_03")), // table_name + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("BASE TABLE")), // table_type + sqltypes.MakeTrusted(sqltypes.Int64, []byte("1427325877")), // unix_timestamp(t.create_time) // Match the timestamp. + sqltypes.MakeTrusted(sqltypes.VarChar, []byte("")), // table_comment + sqltypes.MakeTrusted(sqltypes.Int64, []byte("128")), // file_size + sqltypes.MakeTrusted(sqltypes.Int64, []byte("256")), // allocated_size }, // test_table_04 will in spite of older timestamp because it doesn't exist yet. mysql.BaseShowTablesRow("test_table_04", false, ""), @@ -149,7 +176,9 @@ func TestOpenAndReload(t *testing.T) { Name: "val", Type: sqltypes.Int32, }}, - PKColumns: []int{0, 1}, + PKColumns: []int{0, 1}, + FileSize: 128, + AllocatedSize: 256, } want["test_table_04"] = &Table{ Name: sqlparser.NewTableIdent("test_table_04"), @@ -157,7 +186,9 @@ func TestOpenAndReload(t *testing.T) { Name: "pk", Type: sqltypes.Int32, }}, - PKColumns: []int{0}, + PKColumns: []int{0}, + FileSize: 100, + AllocatedSize: 150, } delete(want, "msg") assert.Equal(t, want, se.GetSchema()) @@ -177,8 +208,8 @@ func TestOpenAndReload(t *testing.T) { require.NoError(t, err) assert.Equal(t, want, se.GetSchema()) - // delete table test_table_03 - db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + db.ClearQueryPattern() + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesRow("test_table_01", false, ""), @@ -220,7 +251,7 @@ func TestOpenFailedDueToMissMySQLTime(t *testing.T) { {sqltypes.NewVarBinary("1427325875")}, }, }) - se := newEngine(10, 1*time.Second, 1*time.Second, false, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) err := se.Open() want := "could not get MySQL time" if err == nil || !strings.Contains(err.Error(), want) { @@ -240,7 +271,7 @@ func TestOpenFailedDueToIncorrectMysqlRowNum(t *testing.T) { {sqltypes.NULL}, }, }) - se := newEngine(10, 1*time.Second, 1*time.Second, false, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) err := se.Open() want := "unexpected result for MySQL time" if err == nil || !strings.Contains(err.Error(), want) { @@ -260,7 +291,7 @@ func TestOpenFailedDueToInvalidTimeFormat(t *testing.T) { {sqltypes.NewVarBinary("invalid_time")}, }, }) - se := newEngine(10, 1*time.Second, 1*time.Second, false, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) err := se.Open() want := "could not parse time" if err == nil || !strings.Contains(err.Error(), want) { @@ -274,10 +305,11 @@ func TestOpenFailedDueToExecErr(t *testing.T) { for query, result := range schematest.Queries() { db.AddQuery(query, result) } - db.AddRejectedQuery(mysql.BaseShowTables, fmt.Errorf("injected error")) - se := newEngine(10, 1*time.Second, 1*time.Second, false, db) - err := se.Open() + want := "injected error" + db.RejectQueryPattern(baseShowTablesPattern, want) + se := newEngine(10, 1*time.Second, 1*time.Second, db) + err := se.Open() if err == nil || !strings.Contains(err.Error(), want) { t.Errorf("se.Open: %v, want %s", err, want) } @@ -289,7 +321,7 @@ func TestOpenFailedDueToTableErr(t *testing.T) { for query, result := range schematest.Queries() { db.AddQuery(query, result) } - db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{ + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesRow("test_table", false, ""), @@ -306,7 +338,7 @@ func TestOpenFailedDueToTableErr(t *testing.T) { {sqltypes.NewVarBinary("")}, }, }) - se := newEngine(10, 1*time.Second, 1*time.Second, false, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) err := se.Open() want := "Row count exceeded" if err == nil || !strings.Contains(err.Error(), want) { @@ -320,7 +352,7 @@ func TestExportVars(t *testing.T) { for query, result := range schematest.Queries() { db.AddQuery(query, result) } - se := newEngine(10, 1*time.Second, 1*time.Second, true, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) se.Open() defer se.Close() expvar.Do(func(kv expvar.KeyValue) { @@ -334,7 +366,7 @@ func TestStatsURL(t *testing.T) { for query, result := range schematest.Queries() { db.AddQuery(query, result) } - se := newEngine(10, 1*time.Second, 1*time.Second, true, db) + se := newEngine(10, 1*time.Second, 1*time.Second, db) se.Open() defer se.Close() @@ -343,7 +375,7 @@ func TestStatsURL(t *testing.T) { se.handleDebugSchema(response, request) } -func newEngine(queryCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, strict bool, db *fakesqldb.DB) *Engine { +func newEngine(queryCacheSize int, reloadTime time.Duration, idleTimeout time.Duration, db *fakesqldb.DB) *Engine { config := tabletenv.NewDefaultConfig() config.QueryCacheSize = queryCacheSize config.SchemaReloadIntervalSeconds.Set(reloadTime) @@ -372,7 +404,9 @@ func initialSchema() map[string]*Table { Name: "pk", Type: sqltypes.Int32, }}, - PKColumns: []int{0}, + PKColumns: []int{0}, + FileSize: 0x64, + AllocatedSize: 0x96, }, "test_table_02": { Name: sqlparser.NewTableIdent("test_table_02"), @@ -380,7 +414,9 @@ func initialSchema() map[string]*Table { Name: "pk", Type: sqltypes.Int32, }}, - PKColumns: []int{0}, + PKColumns: []int{0}, + FileSize: 0x64, + AllocatedSize: 0x96, }, "test_table_03": { Name: sqlparser.NewTableIdent("test_table_03"), @@ -388,7 +424,9 @@ func initialSchema() map[string]*Table { Name: "pk", Type: sqltypes.Int32, }}, - PKColumns: []int{0}, + PKColumns: []int{0}, + FileSize: 0x64, + AllocatedSize: 0x96, }, "seq": { Name: sqlparser.NewTableIdent("seq"), @@ -406,8 +444,10 @@ func initialSchema() map[string]*Table { Name: "increment", Type: sqltypes.Int64, }}, - PKColumns: []int{0}, - SequenceInfo: &SequenceInfo{}, + PKColumns: []int{0}, + FileSize: 0x64, + AllocatedSize: 0x96, + SequenceInfo: &SequenceInfo{}, }, "msg": { Name: sqlparser.NewTableIdent("msg"), @@ -431,7 +471,9 @@ func initialSchema() map[string]*Table { Name: "message", Type: sqltypes.Int64, }}, - PKColumns: []int{0}, + PKColumns: []int{0}, + FileSize: 0x64, + AllocatedSize: 0x96, MessageInfo: &MessageInfo{ Fields: []*querypb.Field{{ Name: "id", diff --git a/go/vt/vttablet/tabletserver/schema/main_test.go b/go/vt/vttablet/tabletserver/schema/main_test.go index e260cbbc697..19d03dd809f 100644 --- a/go/vt/vttablet/tabletserver/schema/main_test.go +++ b/go/vt/vttablet/tabletserver/schema/main_test.go @@ -34,10 +34,9 @@ func getTestSchemaEngine(t *testing.T) (*Engine, *fakesqldb.DB, func()) { "int64"), "1427325876", )) - db.AddQuery(mysql.BaseShowTables, &sqltypes.Result{}) - + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{}) db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{}) - se := newEngine(10, 10*time.Second, 10*time.Second, true, db) + se := newEngine(10, 10*time.Second, 10*time.Second, db) require.NoError(t, se.Open()) cancel := func() { defer db.Close() diff --git a/go/vt/vttablet/tabletserver/schema/schematest/schematest.go b/go/vt/vttablet/tabletserver/schema/schematest/schematest.go index 64395121714..236ed3d024f 100644 --- a/go/vt/vttablet/tabletserver/schema/schematest/schematest.go +++ b/go/vt/vttablet/tabletserver/schema/schematest/schematest.go @@ -61,16 +61,6 @@ func Queries() map[string]*sqltypes.Result { {sqltypes.NewVarBinary("0")}, }, }, - mysql.BaseShowTables: { - Fields: mysql.BaseShowTablesFields, - Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesRow("test_table_01", false, ""), - mysql.BaseShowTablesRow("test_table_02", false, ""), - mysql.BaseShowTablesRow("test_table_03", false, ""), - mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), - }, - }, mysql.BaseShowPrimary: { Fields: mysql.ShowPrimaryFields, Rows: [][]sqltypes.Value{ diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index 96665297eec..f870de335e0 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -2458,6 +2458,14 @@ func setupFakeDB(t *testing.T) *fakesqldb.DB { for query, result := range getSupportedQueries() { db.AddQuery(query, result) } + db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table", false, ""), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) + return db } @@ -2520,13 +2528,6 @@ func getSupportedQueries() map[string]*sqltypes.Result { {sqltypes.NewVarBinary("0")}, }, }, - mysql.BaseShowTables: { - Fields: mysql.BaseShowTablesFields, - Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesRow("test_table", false, ""), - mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), - }, - }, mysql.BaseShowPrimary: { Fields: mysql.ShowPrimaryFields, Rows: [][]sqltypes.Value{ From ac19775d50886439af02fb463af22cdb7e34ad2c Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 5 Feb 2021 12:29:26 +0530 Subject: [PATCH 04/14] added table size guage metrics Signed-off-by: Harshit Gangal --- go/vt/vttablet/tabletserver/schema/engine.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 69bfe6ae2e0..631b8b8f82a 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "vitess.io/vitess/go/stats" "vitess.io/vitess/go/vt/dbconnpool" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -76,6 +77,9 @@ type Engine struct { // dbCreationFailed is for preventing log spam. dbCreationFailed bool + + tableFileSizeGauge *stats.GaugesWithSingleLabel + tableAllocatedSizeGauge *stats.GaugesWithSingleLabel } // NewEngine creates a new Engine. @@ -93,6 +97,8 @@ func NewEngine(env tabletenv.Env) *Engine { reloadTime: reloadTime, } _ = env.Exporter().NewGaugeDurationFunc("SchemaReloadTime", "vttablet keeps table schemas in its own memory and periodically refreshes it from MySQL. This config controls the reload time.", se.ticks.Interval) + se.tableFileSizeGauge = env.Exporter().NewGaugesWithSingleLabel("TableFileSize", "tracks table file size", "Table") + se.tableAllocatedSizeGauge = env.Exporter().NewGaugesWithSingleLabel("TableAllocatedSize", "tracks table allocated size", "Table") env.Exporter().HandleFunc("/debug/schema", se.handleDebugSchema) env.Exporter().HandleFunc("/schemaz", func(w http.ResponseWriter, r *http.Request) { @@ -324,6 +330,10 @@ func (se *Engine) reload(ctx context.Context) error { fileSize, _ := evalengine.ToUint64(row[4]) allocatedSize, _ := evalengine.ToUint64(row[5]) + // publish the size metrics + se.tableFileSizeGauge.Set(tableName, int64(fileSize)) + se.tableAllocatedSizeGauge.Set(tableName, int64(allocatedSize)) + // TODO(sougou); find a better way detect changed tables. This method // seems unreliable. The endtoend test flags all tables as changed. tbl, isInTablesMap := se.tables[tableName] From 5fb26717361b2ec6dcc1992f5f598f4c83e9df11 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 5 Feb 2021 14:57:58 +0530 Subject: [PATCH 05/14] added schema with size query to mariadb flavor Signed-off-by: Harshit Gangal --- go/mysql/flavor_mariadb_binlog_playback.go | 2 +- go/mysql/flavor_mysql.go | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/go/mysql/flavor_mariadb_binlog_playback.go b/go/mysql/flavor_mariadb_binlog_playback.go index 53cdd2d4173..123aa2afa4c 100644 --- a/go/mysql/flavor_mariadb_binlog_playback.go +++ b/go/mysql/flavor_mariadb_binlog_playback.go @@ -32,5 +32,5 @@ func (mariadbFlavor) disableBinlogPlaybackCommand() string { // baseShowTablesWithSizes is part of the Flavor interface. func (mariadbFlavor) baseShowTablesWithSizes() string { - return "" + return TablesWithSize57 } diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 5cf81a8b26f..f77c0f652e3 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -241,16 +241,22 @@ func (mysqlFlavor) disableBinlogPlaybackCommand() string { return "" } -// baseShowTablesWithSizes is part of the Flavor interface. -func (mysqlFlavor57) baseShowTablesWithSizes() string { - return `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size +// TablesWithSize57 is a query to select table along with size for mysql 5.7 +const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` + +// TablesWithSize80 is a query to select table along with size for mysql 8.0 +const TablesWithSize80 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size + FROM information_schema.tables t, information_schema.innodb_tablespaces i + WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` + +// baseShowTablesWithSizes is part of the Flavor interface. +func (mysqlFlavor57) baseShowTablesWithSizes() string { + return TablesWithSize57 } // baseShowTablesWithSizes is part of the Flavor interface. func (mysqlFlavor80) baseShowTablesWithSizes() string { - return `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size - FROM information_schema.tables t, information_schema.innodb_tablespaces i - WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` + return TablesWithSize80 } From 896a2ae6fc271c760c620397d3d818cf416104dd Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 5 Feb 2021 14:58:31 +0530 Subject: [PATCH 06/14] fix vtexpain test Signed-off-by: Harshit Gangal --- go/vt/vtexplain/vtexplain_vttablet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index d57ebe65132..c7f5f3d47fc 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -382,7 +382,7 @@ func initTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) error { } showTableRows = append(showTableRows, mysql.BaseShowTablesRow(table, false, options)) } - schemaQueries["mysql.BaseShowTables"] = &sqltypes.Result{ + schemaQueries[mysql.TablesWithSize57] = &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: showTableRows, } From 0ada42b3791e11ea152378c8bf1f0231e7f5e3cf Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 5 Feb 2021 15:34:05 +0530 Subject: [PATCH 07/14] add query for filepos flavor Signed-off-by: Harshit Gangal --- go/mysql/flavor_filepos.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/flavor_filepos.go b/go/mysql/flavor_filepos.go index 05377dee0be..ba615d05b00 100644 --- a/go/mysql/flavor_filepos.go +++ b/go/mysql/flavor_filepos.go @@ -274,5 +274,5 @@ func (*filePosFlavor) disableBinlogPlaybackCommand() string { // baseShowTablesWithSizes is part of the Flavor interface. func (*filePosFlavor) baseShowTablesWithSizes() string { - return "" + return TablesWithSize57 } From ac9723627f3078a0baf5a57f2faf60f08826899e Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 5 Feb 2021 19:57:45 +0530 Subject: [PATCH 08/14] added mysql56 flavor Signed-off-by: Harshit Gangal --- go/mysql/flavor.go | 6 +++++- go/mysql/flavor_mysql.go | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index e4a3c82261d..826c654120e 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -45,6 +45,8 @@ const ( // mariaDBVersionString is present in mariaDBVersionString = "MariaDB" // mysql57VersionPrefix is the prefix for 5.7 mysql version, such as 5.7.31-log + mysql57VersionPrefix = "5.7." + // mysql80VersionPrefix is the prefix for 8.0 mysql version, such as 8.0.19 mysql80VersionPrefix = "8.0." ) @@ -145,10 +147,12 @@ func (c *Conn) fillFlavor(params *ConnParams) { c.flavor = mariadbFlavor{} case strings.Contains(c.ServerVersion, mariaDBVersionString): c.flavor = mariadbFlavor{} + case strings.HasPrefix(c.ServerVersion, mysql57VersionPrefix): + c.flavor = mysqlFlavor57{} case strings.HasPrefix(c.ServerVersion, mysql80VersionPrefix): c.flavor = mysqlFlavor80{} default: - c.flavor = mysqlFlavor57{} + c.flavor = mysqlFlavor56{} } } diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index f77c0f652e3..88a7d102e85 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -29,6 +29,9 @@ import ( // mysqlFlavor implements the Flavor interface for Mysql. type mysqlFlavor struct{} +type mysqlFlavor56 struct { + mysqlFlavor +} type mysqlFlavor57 struct { mysqlFlavor } @@ -36,6 +39,7 @@ type mysqlFlavor80 struct { mysqlFlavor } +var _ flavor = (*mysqlFlavor56)(nil) var _ flavor = (*mysqlFlavor57)(nil) var _ flavor = (*mysqlFlavor80)(nil) @@ -241,6 +245,10 @@ func (mysqlFlavor) disableBinlogPlaybackCommand() string { return "" } +// TablesWithSize56 is a query to select table along with size for mysql 5.6 +const TablesWithSize56 = `SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) + FROM information_schema.tables WHERE table_schema = database()` + // TablesWithSize57 is a query to select table along with size for mysql 5.7 const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i @@ -251,6 +259,11 @@ const TablesWithSize80 = `SELECT t.table_name, t.table_type, unix_timestamp(t.cr FROM information_schema.tables t, information_schema.innodb_tablespaces i WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` +// baseShowTablesWithSizes is part of the Flavor interface. +func (mysqlFlavor56) baseShowTablesWithSizes() string { + return TablesWithSize56 +} + // baseShowTablesWithSizes is part of the Flavor interface. func (mysqlFlavor57) baseShowTablesWithSizes() string { return TablesWithSize57 From f2791bcff85f91ebbf6d1614128b36ddc5260658 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 6 Feb 2021 00:31:11 +0530 Subject: [PATCH 09/14] added mariadb flavor Signed-off-by: Harshit Gangal --- go/mysql/flavor.go | 15 +++++++++++---- go/mysql/flavor_filepos.go | 2 +- go/mysql/flavor_mariadb.go | 9 +++++++++ go/mysql/flavor_mariadb_binlog_playback.go | 7 ++++++- go/mysql/flavor_mariadb_test.go | 4 ++-- 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/go/mysql/flavor.go b/go/mysql/flavor.go index 826c654120e..b178a5f0d5f 100644 --- a/go/mysql/flavor.go +++ b/go/mysql/flavor.go @@ -144,9 +144,13 @@ func (c *Conn) fillFlavor(params *ConnParams) { c.flavor = flavorFunc() case strings.HasPrefix(c.ServerVersion, mariaDBReplicationHackPrefix): c.ServerVersion = c.ServerVersion[len(mariaDBReplicationHackPrefix):] - c.flavor = mariadbFlavor{} + c.flavor = mariadbFlavor101{} case strings.Contains(c.ServerVersion, mariaDBVersionString): - c.flavor = mariadbFlavor{} + mariadbVersion, err := strconv.ParseFloat(c.ServerVersion[:4], 64) + if err != nil || mariadbVersion < 10.2 { + c.flavor = mariadbFlavor101{} + } + c.flavor = mariadbFlavor102{} case strings.HasPrefix(c.ServerVersion, mysql57VersionPrefix): c.flavor = mysqlFlavor57{} case strings.HasPrefix(c.ServerVersion, mysql80VersionPrefix): @@ -165,8 +169,11 @@ func (c *Conn) fillFlavor(params *ConnParams) { // is identified as MariaDB. Most applications should not care, but // this is useful in tests. func (c *Conn) IsMariaDB() bool { - _, ok := c.flavor.(mariadbFlavor) - return ok + switch c.flavor.(type) { + case mariadbFlavor101, mariadbFlavor102: + return true + } + return false } // MasterPosition returns the current master replication position. diff --git a/go/mysql/flavor_filepos.go b/go/mysql/flavor_filepos.go index ba615d05b00..33e67b76624 100644 --- a/go/mysql/flavor_filepos.go +++ b/go/mysql/flavor_filepos.go @@ -274,5 +274,5 @@ func (*filePosFlavor) disableBinlogPlaybackCommand() string { // baseShowTablesWithSizes is part of the Flavor interface. func (*filePosFlavor) baseShowTablesWithSizes() string { - return TablesWithSize57 + return TablesWithSize56 } diff --git a/go/mysql/flavor_mariadb.go b/go/mysql/flavor_mariadb.go index 6d7db404442..422344d4f5a 100644 --- a/go/mysql/flavor_mariadb.go +++ b/go/mysql/flavor_mariadb.go @@ -30,6 +30,15 @@ import ( // mariadbFlavor implements the Flavor interface for MariaDB. type mariadbFlavor struct{} +type mariadbFlavor101 struct { + mariadbFlavor +} +type mariadbFlavor102 struct { + mariadbFlavor +} + +var _ flavor = (*mariadbFlavor101)(nil) +var _ flavor = (*mariadbFlavor102)(nil) // masterGTIDSet is part of the Flavor interface. func (mariadbFlavor) masterGTIDSet(c *Conn) (GTIDSet, error) { diff --git a/go/mysql/flavor_mariadb_binlog_playback.go b/go/mysql/flavor_mariadb_binlog_playback.go index 123aa2afa4c..e862e744d04 100644 --- a/go/mysql/flavor_mariadb_binlog_playback.go +++ b/go/mysql/flavor_mariadb_binlog_playback.go @@ -31,6 +31,11 @@ func (mariadbFlavor) disableBinlogPlaybackCommand() string { } // baseShowTablesWithSizes is part of the Flavor interface. -func (mariadbFlavor) baseShowTablesWithSizes() string { +func (mariadbFlavor101) baseShowTablesWithSizes() string { + return TablesWithSize56 +} + +// baseShowTablesWithSizes is part of the Flavor interface. +func (mariadbFlavor102) baseShowTablesWithSizes() string { return TablesWithSize57 } diff --git a/go/mysql/flavor_mariadb_test.go b/go/mysql/flavor_mariadb_test.go index 120cde37583..82a5b1312b4 100644 --- a/go/mysql/flavor_mariadb_test.go +++ b/go/mysql/flavor_mariadb_test.go @@ -40,7 +40,7 @@ func TestMariadbSetMasterCommands(t *testing.T) { MASTER_CONNECT_RETRY = 1234, MASTER_USE_GTID = current_pos` - conn := &Conn{flavor: mariadbFlavor{}} + conn := &Conn{flavor: mariadbFlavor101{}} got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry) if got != want { t.Errorf("mariadbFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want) @@ -73,7 +73,7 @@ func TestMariadbSetMasterCommandsSSL(t *testing.T) { MASTER_SSL_KEY = 'ssl-key', MASTER_USE_GTID = current_pos` - conn := &Conn{flavor: mariadbFlavor{}} + conn := &Conn{flavor: mariadbFlavor101{}} got := conn.SetMasterCommand(params, masterHost, masterPort, masterConnectRetry) if got != want { t.Errorf("mariadbFlavor.SetMasterCommands(%#v, %#v, %#v, %#v) = %#v, want %#v", params, masterHost, masterPort, masterConnectRetry, got, want) From 46a09f7eede401bed0478ccfced24986a7653646 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Sat, 6 Feb 2021 00:50:23 +0530 Subject: [PATCH 10/14] fix mysql56 flavour query Signed-off-by: Harshit Gangal --- go/mysql/flavor_mysql.go | 2 +- go/vt/vttablet/tabletserver/query_engine_test.go | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 88a7d102e85..4334cbfb699 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -247,7 +247,7 @@ func (mysqlFlavor) disableBinlogPlaybackCommand() string { // TablesWithSize56 is a query to select table along with size for mysql 5.6 const TablesWithSize56 = `SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) - FROM information_schema.tables WHERE table_schema = database()` + FROM information_schema.tables WHERE table_schema = database() group by table_name` // TablesWithSize57 is a query to select table along with size for mysql 5.7 const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 6189297c67b..58d78ed30b3 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -32,16 +32,13 @@ import ( "testing" "time" - "vitess.io/vitess/go/mysql" - - "context" "github.com/stretchr/testify/require" "vitess.io/vitess/go/cache" - "vitess.io/vitess/go/streamlog" - + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/streamlog" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/tableacl" "vitess.io/vitess/go/vt/vttablet/tabletserver/planbuilder" From f4ebe622a5a2b8033187fb89ac3840afe351bbf2 Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 9 Feb 2021 08:18:40 +0100 Subject: [PATCH 11/14] made the mysql 5.7 tables with sizes query less fragile Signed-off-by: Andres Taylor --- go/mysql/flavor_mysql.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 4334cbfb699..1e5ac53d8aa 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -249,10 +249,19 @@ func (mysqlFlavor) disableBinlogPlaybackCommand() string { const TablesWithSize56 = `SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) FROM information_schema.tables WHERE table_schema = database() group by table_name` -// TablesWithSize57 is a query to select table along with size for mysql 5.7 -const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size - FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i - WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)` +// TablesWithSize57 is a query to select table along with size for mysql 5.7. +// It's a little weird, because the JOIN predicate only works if the table and databases do not contain weird characters. +// As a fallback, we use the mysql 5.6 query, which is not always up to date, but works for all table/db names. +const TablesWithSize57 = ` + SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size + FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i + WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name) +UNION ALL + SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) + FROM information_schema.tables + WHERE table_schema = database() AND NOT EXISTS(SELECT * FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)) + group by table_name +` // TablesWithSize80 is a query to select table along with size for mysql 8.0 const TablesWithSize80 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size From 7b298fb8f55d196f612b881aa59a821248a9580a Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 9 Feb 2021 08:32:21 +0100 Subject: [PATCH 12/14] change the size query so that it handles databases with both easy and difficult table names Signed-off-by: Andres Taylor --- go/mysql/flavor_mysql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 1e5ac53d8aa..3f891ea72be 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -258,8 +258,8 @@ const TablesWithSize57 = ` WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name) UNION ALL SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) - FROM information_schema.tables - WHERE table_schema = database() AND NOT EXISTS(SELECT * FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name)) + FROM information_schema.tables t + WHERE table_schema = database() AND NOT EXISTS(SELECT * FROM information_schema.innodb_sys_tablespaces i WHERE i.name = concat(t.table_schema,'/',t.table_name)) group by table_name ` From f615d794dc0ad6ff3912b8ef85d93a7f4c71e86c Mon Sep 17 00:00:00 2001 From: Andres Taylor Date: Tue, 9 Feb 2021 08:35:20 +0100 Subject: [PATCH 13/14] make sure we group by all non aggregating columns Signed-off-by: Andres Taylor --- go/mysql/flavor_mysql.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 3f891ea72be..065b4e0d727 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -260,7 +260,7 @@ UNION ALL SELECT table_name, table_type, unix_timestamp(create_time), table_comment, SUM( data_length + index_length), SUM( data_length + index_length) FROM information_schema.tables t WHERE table_schema = database() AND NOT EXISTS(SELECT * FROM information_schema.innodb_sys_tablespaces i WHERE i.name = concat(t.table_schema,'/',t.table_name)) - group by table_name + group by table_name, table_type, unix_timestamp(create_time), table_comment ` // TablesWithSize80 is a query to select table along with size for mysql 8.0 From 91b84cbbd7a911d7cbc081a0a0c0ac260f508b2e Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Tue, 9 Feb 2021 13:52:40 +0530 Subject: [PATCH 14/14] fix schema unit test Signed-off-by: Harshit Gangal --- go/mysql/flavor_mysql.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 065b4e0d727..9d4dd5e08da 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -252,8 +252,7 @@ const TablesWithSize56 = `SELECT table_name, table_type, unix_timestamp(create_t // TablesWithSize57 is a query to select table along with size for mysql 5.7. // It's a little weird, because the JOIN predicate only works if the table and databases do not contain weird characters. // As a fallback, we use the mysql 5.6 query, which is not always up to date, but works for all table/db names. -const TablesWithSize57 = ` - SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size +const TablesWithSize57 = `SELECT t.table_name, t.table_type, unix_timestamp(t.create_time), t.table_comment, i.file_size, i.allocated_size FROM information_schema.tables t, information_schema.innodb_sys_tablespaces i WHERE t.table_schema = database() and i.name = concat(t.table_schema,'/',t.table_name) UNION ALL