diff --git a/go/cmd/vttestserver/main.go b/go/cmd/vttestserver/main.go index 5758bd943a9..1b16e90058d 100644 --- a/go/cmd/vttestserver/main.go +++ b/go/cmd/vttestserver/main.go @@ -136,7 +136,6 @@ func init() { "Rdonly tablets per shard") flag.StringVar(&config.Charset, "charset", "utf8mb4", "MySQL charset") - flag.StringVar(&config.Collation, "collation", "", "MySQL collation") flag.StringVar(&config.PlannerVersion, "planner_version", "v3", "Sets the default planner to use when the session has not changed it. Valid values are: V3, Gen4, Gen4Greedy and Gen4Fallback. Gen4Fallback tries the new gen4 planner and falls back to the V3 planner if the gen4 fails. All Gen4 versions should be considered experimental!") diff --git a/go/mysql/client.go b/go/mysql/client.go index 242f798fcf5..8e18e2cb6dc 100644 --- a/go/mysql/client.go +++ b/go/mysql/client.go @@ -167,14 +167,6 @@ func Connect(ctx context.Context, params *ConnParams) (*Conn, error) { } } - // Once we are connected to the server, we set the collation for this connection. - // This step usually occurs during the handshake, however, the handshake protocol - // grants us 8 bits for the collation ID, which is lower than the range of supported - // collations. For this reason, we manually set the collation for the connection. - if err := setCollationForConnection(c, params); err != nil { - return nil, err - } - return c, nil } @@ -202,58 +194,6 @@ func (c *Conn) Ping() error { return vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected packet type: %d", data[0]) } -// setCollationForConnection sets the connection's collation to the given collation. -// -// The charset should always be set as it has a default value ("utf8mb4"), -// however, one can always override its default to an empty string, which -// is not a problem as long as the user has specified the collation. -// If the collation flag was not specified when starting the tablet, we -// attempt to find the default collation for the current charset. -// If either the collation and charset are missing, or the resolution of -// the default collation using the given charset fails, we error out. -// -// This method is also responsible for creating and storing the collation -// environment that will be used by this connection. The collation environment -// allows us to make informed decisions around charset's default collation -// depending on the MySQL/MariaDB version we are using. -func setCollationForConnection(c *Conn, params *ConnParams) error { - // Once we have done the initial handshake with MySQL, we receive the server version - // string. This string is critical as it enables the instantiation of a new collation - // environment variable. - // Certain MySQL or MariaDB versions might have different default collations for some - // charsets, so it is important to use a database-version-aware collation system/API. - env := collations.NewEnvironment(c.ServerVersion) - coll, err := env.ResolveCollation(params.Charset, params.Collation) - if err != nil { - return err - } - - // We send a query to MySQL to set the connection's collation. - // See: https://dev.mysql.com/doc/refman/8.0/en/charset-connection.html - querySetCollation := fmt.Sprintf("SET collation_connection = %s;", coll.Name()) - if _, err := c.ExecuteFetch(querySetCollation, 1, false); err != nil { - return err - } - - c.Collation = coll.ID() - return nil -} - -// getHandshakeCharacterSet returns the collation ID of DefaultCollation in an -// 8 bits integer which will be used to feed the handshake protocol's packet. -func getHandshakeCharacterSet() (uint8, error) { - coll := collations.Local().LookupByName(DefaultCollation) - if coll == nil { - // theoretically, this should never happen from an end user perspective - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "cannot resolve collation ID for collation: '%s'", DefaultCollation) - } - if coll.ID() > 255 { - // same here, this should never happen - return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "collation ID for '%s' will overflow, value: %d", DefaultCollation, coll.ID()) - } - return uint8(coll.ID()), nil -} - // clientHandshake handles the client side of the handshake. // Note the connection can be closed while this is running. // Returns a SQLError. @@ -282,24 +222,7 @@ func (c *Conn) clientHandshake(params *ConnParams) error { c.Capabilities = capabilities & (CapabilityClientDeprecateEOF) } - // The MySQL handshake package uses the "character set" field to define - // which character set must be used. But, the value we give to this field - // correspond in fact to the collation ID. MySQL will then deduce what the - // character set for this collation ID is, and use it. - // Problem is, this field is 8-bits long meaning that the ID can range from - // 0 to 255, which is smaller than the range of IDs we support. - // If, for instance, we used the collation "utf8mb4_0900_as_ci" that has an - // ID equal to 305, the value would overflow when transformed into an 8 bits - // integer. - // To alleviate this issue, we use a default and safe collation for the handshake - // and once the connection is established, we will manually set the collation. - // The code below gets that default character set for the Handshake packet. - // - // Note: this character set might be different from the one we will use - // for the connection. - // - // See: https://dev.mysql.com/doc/internals/en/connection-phase-packets.html - characterSet, err := getHandshakeCharacterSet() + charset, err := collations.Local().ParseConnectionCharset(params.Charset) if err != nil { return err } @@ -340,7 +263,7 @@ func (c *Conn) clientHandshake(params *ConnParams) error { } // Send the SSLRequest packet. - if err := c.writeSSLRequest(capabilities, characterSet, params); err != nil { + if err := c.writeSSLRequest(capabilities, charset, params); err != nil { return err } @@ -371,7 +294,7 @@ func (c *Conn) clientHandshake(params *ConnParams) error { // Build and send our handshake response 41. // Note this one will never have SSL flag on. - if err := c.writeHandshakeResponse41(capabilities, scrambledPassword, characterSet, params); err != nil { + if err := c.writeHandshakeResponse41(capabilities, scrambledPassword, charset, params); err != nil { return err } @@ -473,7 +396,7 @@ func (c *Conn) parseInitialHandshakePacket(data []byte) (uint32, []byte, error) if !ok { return 0, nil, NewSQLError(CRMalformedPacket, SSUnknownSQLState, "parseInitialHandshakePacket: packet has no character set") } - c.CharacterSet = characterSet + c.CharacterSet = collations.ID(characterSet) // Status flags. Ignored. _, pos, ok = readUint16(data, pos) diff --git a/go/mysql/collations/8bit.go b/go/mysql/collations/8bit.go index 07123af1147..3f14ae8be33 100644 --- a/go/mysql/collations/8bit.go +++ b/go/mysql/collations/8bit.go @@ -224,8 +224,6 @@ func weightStringPadingSimple(padChar byte, dst []byte, numCodepoints int, padTo return dst } -const CollationBinaryID ID = 63 - type Collation_binary struct{} func (c *Collation_binary) Init() {} diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index ff36c9a4333..57264054d28 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -123,32 +123,6 @@ func fetchCacheEnvironment(version collver) *Environment { return env } -// ResolveCollation returns the default collation that will be used for the given charset and collation. -// Both charset and collation can be empty strings, in which case utf8mb4 will be used as a charset and its -// default collation will be returned. -func (env *Environment) ResolveCollation(charset, collation string) (Collation, error) { - // if there is no collation or charset, we default to utf8mb4 - if collation == "" && charset == "" { - charset = "utf8mb4" - } - - var coll Collation - if collation == "" { - // If there is no collation we will just use the charset's default collation - // otherwise we directly use the given collation. - coll = env.DefaultCollationForCharset(charset) - } else { - // Here we call the collations API to ensure the collation/charset exist - // and is supported by Vitess. - coll = env.LookupByName(collation) - } - if coll == nil { - // The given collation is most likely unknown or unsupported, we need to fail. - return nil, fmt.Errorf("cannot resolve collation: '%s'", collation) - } - return coll, nil -} - // NewEnvironment creates a collation Environment for the given MySQL version string. // The version string must be in the format that is sent by the server as the version packet // when opening a new MySQL connection @@ -246,3 +220,53 @@ func Local() *Environment { }) return defaultEnv } + +// A few interesting character set values. +// See http://dev.mysql.com/doc/internals/en/character-set.html#packet-Protocol::CharacterSet +const ( + CollationUtf8ID = 33 + CollationUtf8mb4ID = 255 + CollationBinaryID = 63 +) + +// DefaultConnectionCharset is the default charset that Vitess will use when negotiating a +// charset in a MySQL connection handshake. Note that in this context, a 'charset' is equivalent +// to a Collation ID, with the exception that it can only fit in 1 byte. +// For MySQL 8.0+ environments, the default charset is `utf8mb4_0900_ai_ci`. +// For older MySQL environments, the default charset is `utf8mb4_general_ci`. +func (env *Environment) DefaultConnectionCharset() uint8 { + switch env.version { + case collverMySQL80: + return CollationUtf8mb4ID + default: + return 45 + } +} + +// ParseConnectionCharset parses the given charset name and returns its numerical +// identifier to be used in a MySQL connection handshake. The charset name can be: +// - the name of a character set, in which case the default collation ID for the +// character set is returned. +// - the name of a collation, in which case the ID for the collation is returned, +// UNLESS the collation itself has an ID greater than 255; such collations are not +// supported because they cannot be negotiated in a single byte in our connection +// handshake. +// - empty, in which case the default connection charset for this MySQL version +// is returned. +func (env *Environment) ParseConnectionCharset(csname string) (uint8, error) { + if csname == "" { + return env.DefaultConnectionCharset(), nil + } + + var collid ID = 0 + csname = strings.ToLower(csname) + if defaults, ok := env.byCharset[csname]; ok { + collid = defaults.Default.ID() + } else if coll, ok := env.byName[csname]; ok { + collid = coll.ID() + } + if collid == 0 || collid > 255 { + return 0, fmt.Errorf("unsupported connection charset: %q", csname) + } + return uint8(collid), nil +} diff --git a/go/mysql/conn.go b/go/mysql/conn.go index 971a78a7edb..15db83e0702 100644 --- a/go/mysql/conn.go +++ b/go/mysql/conn.go @@ -182,17 +182,14 @@ type Conn struct { // by Handler methods. StatusFlags uint16 - // CharacterSet is the character set used by the other side of the - // connection. - // It is set during the initial handshake. - // See the values in constants.go. - CharacterSet uint8 - - // Collation defines the collation for this connection, it has the same - // value as the collation_connection variable of MySQL. - // Its value is set after we send the initial "SET collation_connection" - // query to MySQL after the handshake is done. - Collation collations.ID + // CharacterSet is the charset for this connection, as negotiated + // in our handshake with the server. Note that although the MySQL protocol lists this + // as a "character set", the returned byte value is actually a Collation ID, + // and hence it's casted as such here. + // If the user has specified a custom Collation in the ConnParams for this + // connection, once the CharacterSet has been negotiated, we will override + // it via SQL and update this field accordingly. + CharacterSet collations.ID // Packet encoding variables. sequence uint8 diff --git a/go/mysql/conn_params.go b/go/mysql/conn_params.go index b0c9ea83d03..7e4decca227 100644 --- a/go/mysql/conn_params.go +++ b/go/mysql/conn_params.go @@ -29,7 +29,6 @@ type ConnParams struct { DbName string `json:"dbname"` UnixSocket string `json:"unix_socket"` Charset string `json:"charset"` - Collation string `json:"collation"` Flags uint64 `json:"flags"` Flavor string `json:"flavor,omitempty"` diff --git a/go/mysql/constants.go b/go/mysql/constants.go index b9ca84ab8ac..cc663c309e0 100644 --- a/go/mysql/constants.go +++ b/go/mysql/constants.go @@ -591,21 +591,6 @@ const ( SSQueryInterrupted = "70100" ) -// A few interesting character set values. -// See http://dev.mysql.com/doc/internals/en/character-set.html#packet-Protocol::CharacterSet -const ( - DefaultCollation = "utf8mb4_general_ci" - - // CharacterSetUtf8 is for UTF8. - CharacterSetUtf8 = 33 - - // CharacterSetUtf8mb4 is for 4-bytes UTF8. - CharacterSetUtf8mb4 = 45 - - // CharacterSetBinary is for binary. Use by integer fields for instance. - CharacterSetBinary = 63 -) - // CharacterSetEncoding maps a charset name to a golang encoder. // golang does not support encoders for all MySQL charsets. // A charset not in this map is unsupported. diff --git a/go/mysql/endtoend/query_test.go b/go/mysql/endtoend/query_test.go index 4a0f2cdccbe..adcab4dbeb4 100644 --- a/go/mysql/endtoend/query_test.go +++ b/go/mysql/endtoend/query_test.go @@ -27,11 +27,20 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" ) +func columnSize(cs collations.ID, size uint32) uint32 { + // utf8_general_ci results in smaller max column sizes because MySQL 5.7 is silly + if collations.Local().LookupByID(cs).Charset().Name() == "utf8" { + return size * 3 / 4 + } + return size +} + // Test the SQL query part of the API. func TestQueries(t *testing.T) { ctx := context.Background() @@ -79,7 +88,7 @@ func TestQueries(t *testing.T) { Database: "vttest", OrgName: "id", ColumnLength: 11, - Charset: mysql.CharacterSetBinary, + Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_PRI_KEY_FLAG | querypb.MySqlFlag_PART_KEY_FLAG | @@ -92,8 +101,8 @@ func TestQueries(t *testing.T) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: 512, - Charset: mysql.CharacterSetUtf8mb4, + ColumnLength: columnSize(conn.CharacterSet, 512), + Charset: uint32(conn.CharacterSet), }, }, Rows: [][]sqltypes.Value{ @@ -188,7 +197,7 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { Database: "vttest", OrgName: "id", ColumnLength: 11, - Charset: mysql.CharacterSetBinary, + Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_PRI_KEY_FLAG | querypb.MySqlFlag_PART_KEY_FLAG | @@ -201,8 +210,8 @@ func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) { OrgTable: "a", Database: "vttest", OrgName: "name", - ColumnLength: 512, - Charset: mysql.CharacterSetUtf8mb4, + ColumnLength: columnSize(conn.CharacterSet, 512), + Charset: uint32(conn.CharacterSet), }, } fields, err := conn.Fields() diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go index 8e3d1e77275..06cbf42ff18 100644 --- a/go/mysql/fakesqldb/server.go +++ b/go/mysql/fakesqldb/server.go @@ -272,8 +272,6 @@ func (db *DB) ConnParams() dbconfigs.Connector { UnixSocket: db.socketFile, Uname: "user1", Pass: "password1", - Charset: "utf8mb4", - Collation: "utf8mb4_general_ci", }) } @@ -283,7 +281,6 @@ func (db *DB) ConnParamsWithUname(uname string) dbconfigs.Connector { UnixSocket: db.socketFile, Uname: uname, Pass: "password1", - Charset: "utf8", }) } diff --git a/go/mysql/schema.go b/go/mysql/schema.go index 2a09b993db8..3ae9f19be42 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -17,6 +17,7 @@ limitations under the License. package mysql import ( + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" @@ -131,7 +132,7 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_NAME", ColumnLength: 192, - Charset: CharacterSetUtf8, + Charset: collations.CollationUtf8ID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "t.table_type", @@ -141,13 +142,13 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_TYPE", ColumnLength: 192, - Charset: CharacterSetUtf8, + Charset: collations.CollationUtf8ID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "unix_timestamp(t.create_time)", Type: querypb.Type_INT64, ColumnLength: 11, - Charset: CharacterSetBinary, + Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }, { Name: "t.table_comment", @@ -157,19 +158,19 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_COMMENT", ColumnLength: 6144, - Charset: CharacterSetUtf8, + Charset: collations.CollationUtf8ID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "i.file_size", Type: querypb.Type_INT64, ColumnLength: 11, - Charset: CharacterSetBinary, + Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }, { Name: "i.allocated_size", Type: querypb.Type_INT64, ColumnLength: 11, - Charset: CharacterSetBinary, + Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }} diff --git a/go/mysql/server.go b/go/mysql/server.go index 75251bf6ffc..fee8573b664 100644 --- a/go/mysql/server.go +++ b/go/mysql/server.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/sqlescape" @@ -591,7 +592,7 @@ func (c *Conn) writeHandshakeV10(serverVersion string, authServer AuthServer, en pos = writeUint16(data, pos, uint16(capabilities)) // Character set. - pos = writeByte(data, pos, CharacterSetUtf8) + pos = writeByte(data, pos, collations.Local().DefaultConnectionCharset()) // Status flag. pos = writeUint16(data, pos, c.StatusFlags) @@ -670,7 +671,7 @@ func (l *Listener) parseClientHandshakePacket(c *Conn, firstTime bool, data []by if !ok { return "", "", nil, vterrors.Errorf(vtrpc.Code_INTERNAL, "parseClientHandshakePacket: can't read characterSet") } - c.CharacterSet = characterSet + c.CharacterSet = collations.ID(characterSet) // 23x reserved zero bytes. pos += 23 diff --git a/go/test/endtoend/vtgate/reservedconn/main_test.go b/go/test/endtoend/vtgate/reservedconn/main_test.go index c0364c93f49..56c28ed313a 100644 --- a/go/test/endtoend/vtgate/reservedconn/main_test.go +++ b/go/test/endtoend/vtgate/reservedconn/main_test.go @@ -120,7 +120,7 @@ func TestMain(m *testing.M) { SchemaSQL: sqlSchema, VSchema: vSchema, } - clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-transaction-timeout", "5"} + clusterInstance.VtTabletExtraArgs = []string{"-queryserver-config-transaction-timeout", "5", "-mysql_server_version", "5.7.0"} if err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false); err != nil { return 1 } diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index c896684a500..030d6c15467 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -75,7 +75,6 @@ type DBConfigs struct { Host string `json:"host,omitempty"` Port int `json:"port,omitempty"` Charset string `json:"charset,omitempty"` - Collation string `json:"collation,omitempty"` Flags uint64 `json:"flags,omitempty"` Flavor string `json:"flavor,omitempty"` SslMode vttls.SslMode `json:"sslMode,omitempty"` @@ -129,7 +128,6 @@ func registerBaseFlags() { flag.StringVar(&GlobalDBConfigs.Host, "db_host", "", "The host name for the tcp connection.") flag.IntVar(&GlobalDBConfigs.Port, "db_port", 0, "tcp port") flag.StringVar(&GlobalDBConfigs.Charset, "db_charset", "utf8mb4", "Character set used for this tablet.") - flag.StringVar(&GlobalDBConfigs.Collation, "db_collation", "", "Collation used for this tablet. If this flag is empty, the default collation for the character set will be used.") flag.Uint64Var(&GlobalDBConfigs.Flags, "db_flags", 0, "Flag values as defined by MySQL.") flag.StringVar(&GlobalDBConfigs.Flavor, "db_flavor", "", "Flavor overrid. Valid value is FilePos.") flag.Var(&GlobalDBConfigs.SslMode, "db_ssl_mode", "SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity.") @@ -358,19 +356,10 @@ func (dbcfgs *DBConfigs) InitWithSocket(defaultSocketFile string) { } // If the connection params has a charset defined, it will not be overridden by the - // global configuration, same thing applies to the collation field. - // At a later stage, when we establish a connection with MySQL, we will receive the - // server name back, and we will be able to create a collation environment which is - // required to figure out the default collation of a charset or if a collation is valid. - // This collation environment will be stored directly in the connection parameters as - // only the individual connection parameters are used to talk with MySQL, not the global - // connection parameter (DBConfigs). + // global configuration. if dbcfgs.Charset != "" && cp.Charset == "" { cp.Charset = dbcfgs.Charset } - if dbcfgs.Collation != "" && cp.Collation == "" { - cp.Collation = dbcfgs.Collation - } if dbcfgs.Flags != 0 { cp.Flags = dbcfgs.Flags @@ -444,7 +433,6 @@ func NewTestDBConfigs(genParams, appDebugParams mysql.ConnParams, dbname string) replParams: genParams, externalReplParams: genParams, DBName: dbname, - Collation: "utf8mb4_general_ci", - Charset: "utf8mb4", + Charset: "utf8mb4_general_ci", } } diff --git a/go/vt/dbconfigs/dbconfigs_test.go b/go/vt/dbconfigs/dbconfigs_test.go index 47d095ed8c7..1d415a4e5ce 100644 --- a/go/vt/dbconfigs/dbconfigs_test.go +++ b/go/vt/dbconfigs/dbconfigs_test.go @@ -46,7 +46,6 @@ func TestInit(t *testing.T) { Port: 1, Socket: "b", Charset: "utf8mb4", - Collation: "utf8mb4_general_ci", Flags: 2, Flavor: "flavor", SslCa: "d", @@ -82,7 +81,6 @@ func TestInit(t *testing.T) { Pass: "apppass", UnixSocket: "b", Charset: "utf8mb4", - Collation: "utf8mb4_general_ci", Flags: 2, Flavor: "flavor", ConnectTimeoutMs: 250, @@ -94,7 +92,6 @@ func TestInit(t *testing.T) { Port: 1, UnixSocket: "b", Charset: "utf8mb4", - Collation: "utf8mb4_general_ci", Flags: 2, Flavor: "flavor", SslCa: "d", @@ -111,7 +108,6 @@ func TestInit(t *testing.T) { Pass: "dbapass", UnixSocket: "b", Charset: "utf8mb4", - Collation: "utf8mb4_general_ci", Flags: 2, Flavor: "flavor", SslCa: "d", @@ -148,7 +144,6 @@ func TestInit(t *testing.T) { appParams: mysql.ConnParams{ UnixSocket: "socket", Charset: "utf8mb4", - Collation: "", // should be transformed to the default collation for its charset }, dbaParams: mysql.ConnParams{ Host: "host", @@ -236,7 +231,6 @@ func TestAccessors(t *testing.T) { replParams: mysql.ConnParams{}, DBName: "db", Charset: "utf8", - Collation: "utf8_general_ci", } if got, want := dbc.AppWithDB().connParams.DbName, "db"; got != want { t.Errorf("dbc.AppWithDB().DbName: %v, want %v", got, want) diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 97cbe8b722d..2c286e94b0a 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -26,6 +26,7 @@ package mysqlctl import ( "bufio" "bytes" + "context" "errors" "flag" "fmt" @@ -42,8 +43,6 @@ import ( rice "github.com/GeertJohan/go.rice" - "context" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/dbconnpool" @@ -663,7 +662,6 @@ func (mysqld *Mysqld) Init(ctx context.Context, cnf *Mycnf, initDBSQLFile string // user is created yet. params := &mysql.ConnParams{ Uname: "root", - Charset: "utf8mb4", UnixSocket: cnf.SocketFile, } if err = mysqld.wait(ctx, cnf, params); err != nil { diff --git a/go/vt/vtgate/evalengine/convert.go b/go/vt/vtgate/evalengine/convert.go index a30fd6adf3d..4e92764fc03 100644 --- a/go/vt/vtgate/evalengine/convert.go +++ b/go/vt/vtgate/evalengine/convert.go @@ -161,8 +161,7 @@ func getCollation(expr sqlparser.Expr, lookup ConverterLookup) collations.TypedC if lookup != nil { collation.Collation = lookup.CollationIDLookup(expr) } else { - sysdefault, _ := collations.Local().ResolveCollation("", "") - collation.Collation = sysdefault.ID() + collation.Collation = collations.ID(collations.Local().DefaultConnectionCharset()) } return collation } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index e52b0ed9efd..3d78207bd8b 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -35,7 +35,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/cache" "vitess.io/vitess/go/hack" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/sync2" @@ -1544,7 +1544,7 @@ func buildVarCharFields(names ...string) []*querypb.Field { fields[i] = &querypb.Field{ Name: v, Type: sqltypes.VarChar, - Charset: mysql.CharacterSetUtf8, + Charset: collations.CollationUtf8ID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), } } diff --git a/go/vt/vtgate/planbuilder/show.go b/go/vt/vtgate/planbuilder/show.go index 0bc12e8b3cd..11a460b7cde 100644 --- a/go/vt/vtgate/planbuilder/show.go +++ b/go/vt/vtgate/planbuilder/show.go @@ -21,12 +21,12 @@ import ( "regexp" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" topodatapb "vitess.io/vitess/go/vt/proto/topodata" "vitess.io/vitess/go/vt/vtgate/vindexes" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" @@ -284,7 +284,7 @@ func buildVarCharFields(names ...string) []*querypb.Field { fields[i] = &querypb.Field{ Name: v, Type: sqltypes.VarChar, - Charset: mysql.CharacterSetUtf8, + Charset: collations.CollationUtf8ID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), } } diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index d6249796571..a2a9e96c1bb 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -144,7 +144,6 @@ func newVCursorImpl( } // we only support collations for the new TabletGateway implementation - collationEnv := collations.Local() var connCollation collations.ID if executor != nil { if gw, isTabletGw := executor.resolver.resolver.GetGateway().(*TabletGateway); isTabletGw { @@ -152,11 +151,7 @@ func newVCursorImpl( } } if connCollation == collations.Unknown { - coll, err := collationEnv.ResolveCollation("", "") - if err != nil { - panic("should never happen: don't know how to resolve default collation") - } - connCollation = coll.ID() + connCollation = collations.ID(collations.Local().DefaultConnectionCharset()) } return &vcursorImpl{ diff --git a/go/vt/vttablet/endtoend/connkilling/main_test.go b/go/vt/vttablet/endtoend/connkilling/main_test.go index 55f16b1e181..5f1d20beffe 100644 --- a/go/vt/vttablet/endtoend/connkilling/main_test.go +++ b/go/vt/vttablet/endtoend/connkilling/main_test.go @@ -61,7 +61,7 @@ func TestMain(m *testing.M) { }, }, OnlyMySQL: true, - Collation: "utf8mb4_general_ci", + Charset: "utf8mb4_general_ci", } if err := cfg.InitSchemas("vttest", testSchema, nil); err != nil { fmt.Fprintf(os.Stderr, "InitSchemas failed: %v\n", err) diff --git a/go/vt/vttablet/endtoend/main_test.go b/go/vt/vttablet/endtoend/main_test.go index ee096480b83..efe934a08b3 100644 --- a/go/vt/vttablet/endtoend/main_test.go +++ b/go/vt/vttablet/endtoend/main_test.go @@ -61,7 +61,7 @@ func TestMain(m *testing.M) { }, }, OnlyMySQL: true, - Collation: "utf8mb4_general_ci", + Charset: "utf8mb4_general_ci", } if err := cfg.InitSchemas("vttest", testSchema, nil); err != nil { fmt.Fprintf(os.Stderr, "InitSchemas failed: %v\n", err) diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index dca6e9bf545..4b68426512a 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -233,15 +233,14 @@ func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32, d return nil, err } - var coll collations.Collation - env := collations.NewEnvironment(dbServerVersion) - if db == nil { - coll, err = env.ResolveCollation("", "") + var charset uint8 + if db != nil && db.Charset != "" { + charset, err = collations.Local().ParseConnectionCharset(db.Charset) + if err != nil { + return nil, err + } } else { - coll, err = env.ResolveCollation(db.Charset, db.Collation) - } - if err != nil { - return nil, err + charset = collations.Local().DefaultConnectionCharset() } return &topodatapb.Tablet{ @@ -258,7 +257,7 @@ func BuildTabletFromInput(alias *topodatapb.TabletAlias, port, grpcPort int32, d DbNameOverride: *initDbNameOverride, Tags: mergeTags(buildTags, initTags), DbServerVersion: dbServerVersion, - DefaultConnCollation: uint32(coll.ID()), + DefaultConnCollation: uint32(charset), }, nil } diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index 27766bc8bdc..90a138977c6 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -71,7 +71,7 @@ func TestStartBuildTabletFromInput(t *testing.T) { Tags: map[string]string{}, DbNameOverride: "aa", DbServerVersion: dbServerVersion, - DefaultConnCollation: 45, + DefaultConnCollation: 255, } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil) @@ -155,7 +155,7 @@ func TestBuildTabletFromInputWithBuildTags(t *testing.T) { Tags: servenv.AppVersion.ToStringMap(), DbNameOverride: "aa", DbServerVersion: "5.7.0", - DefaultConnCollation: 45, + DefaultConnCollation: 255, } gotTablet, err := BuildTabletFromInput(alias, port, grpcport, dbServerVersion, nil) diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index bb548dce080..d4d1475b7d9 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -95,6 +95,7 @@ func Init() (*Env, error) { }, }, OnlyMySQL: true, + Charset: "utf8mb4_general_ci", } te.cluster = &vttest.LocalCluster{ Config: cfg, diff --git a/go/vt/vttest/local_cluster.go b/go/vt/vttest/local_cluster.go index 85c1cc6e642..2ab16143432 100644 --- a/go/vt/vttest/local_cluster.go +++ b/go/vt/vttest/local_cluster.go @@ -79,11 +79,6 @@ type Config struct { // Charset is the default charset used by MySQL Charset string - // Collation is the default collation used by MySQL - // If Collation is set and Charset is not set, Collation will be used - // to define the value of Charset - Collation string - // PlannerVersion is the planner version to use for the vtgate. // Choose between V3, Gen4, Gen4Greedy and Gen4Fallback PlannerVersion string @@ -225,7 +220,9 @@ type LocalCluster struct { // This connection should be used for debug/introspection purposes; normal // cluster access should be performed through the vtgate port. func (db *LocalCluster) MySQLConnParams() mysql.ConnParams { - return db.mysql.Params(db.DbName()) + connParams := db.mysql.Params(db.DbName()) + connParams.Charset = db.Config.Charset + return connParams } // MySQLAppDebugConnParams returns a mysql.ConnParams struct that can be used diff --git a/go/vt/vttest/mysqlctl.go b/go/vt/vttest/mysqlctl.go index 5c55e51d02b..a3bc45a7a85 100644 --- a/go/vt/vttest/mysqlctl.go +++ b/go/vt/vttest/mysqlctl.go @@ -146,8 +146,6 @@ func (ctl *Mysqlctl) TabletDir() string { // using Vitess' mysql client. func (ctl *Mysqlctl) Params(dbname string) mysql.ConnParams { return mysql.ConnParams{ - Charset: DefaultCharset, - Collation: "utf8mb4_general_ci", DbName: dbname, Uname: "vt_dba", UnixSocket: ctl.UnixSocket(), diff --git a/go/vt/vttest/vtprocess.go b/go/vt/vttest/vtprocess.go index 566dedd88b5..a5a039f90c8 100644 --- a/go/vt/vttest/vtprocess.go +++ b/go/vt/vttest/vtprocess.go @@ -174,9 +174,6 @@ func (vtp *VtProcess) WaitStart() (err error) { const ( // DefaultCharset is the default charset used by MySQL instances DefaultCharset = "utf8mb4" - - // DefaultCollation is the default collation used between VTTablet and MySQL - DefaultCollation = "utf8mb4_general_ci" ) // QueryServerArgs are the default arguments passed to all Vitess query servers @@ -211,15 +208,10 @@ func VtcomboProcess(env Environment, args *Config, mysql MySQLManager) *VtProces if charset == "" { charset = DefaultCharset } - collation := args.Collation - if collation == "" { - collation = DefaultCollation - } protoTopo, _ := prototext.Marshal(args.Topology) vt.ExtraArgs = append(vt.ExtraArgs, []string{ "-db_charset", charset, - "-db_collation", collation, "-db_app_user", user, "-db_app_password", pass, "-db_dba_user", user,