From d9c6ff5be64b3ed5fc32f6395ee2f4bb8400eccf Mon Sep 17 00:00:00 2001 From: hanchuanchuan Date: Thu, 26 Dec 2019 14:51:00 +0800 Subject: [PATCH 1/2] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96alter=E5=AD=90?= =?UTF-8?q?=E5=8F=A5=E6=8B=86=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/go.yml | 44 +++++---- .travis.yml | 4 + circle.yml | 2 + server/server.go | 6 ++ session/osc.go | 62 ++++++++++++- session/session_inception_common_test.go | 11 ++- session/session_inception_exec_test.go | 113 +++++++++++++++++++++-- 7 files changed, 207 insertions(+), 35 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5bb8b639..6a7ed079 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -5,7 +5,7 @@ on: - master - release/* pull_request: - + jobs: build: @@ -15,10 +15,10 @@ jobs: # - name: show mysql info # run: ls -l /etc/my* || true - + - name: Set MySQL Config run: | - sudo service mysql stop + sudo service mysql stop sudo mkdir -p /etc/mysql/ || true echo '[mysqld]' | sudo tee /etc/mysql/my.cnf echo 'server-id=111' | sudo tee -a /etc/mysql/my.cnf @@ -28,17 +28,17 @@ jobs: echo 'enforce_gtid_consistency = ON' | sudo tee -a /etc/mysql/my.cnf echo 'lower_case_table_names = 1' | sudo tee -a /etc/mysql/my.cnf echo 'character_set_server = utf8' | sudo tee -a /etc/mysql/my.cnf - + # cat /etc/mysql/my.cnf || true - name: Startup MySQL run: | - sudo service mysql start || true + sudo service mysql start || true sudo tail -100 /var/log/mysql/error.log - name: Show MySQL Variables run: mysql -uroot -p'root' -e "show variables where Variable_name in ('server_id','log_bin','lower_case_table_names','version');" - + # - name: Show MySQL buffer_pool # run: mysql -uroot -p'root' -e "show variables like '%buffer_pool%'" @@ -47,36 +47,36 @@ jobs: mysql -uroot -p'root' -e "create database if not exists test DEFAULT CHARACTER SET utf8;create database if not exists test_inc DEFAULT CHARACTER SET utf8;" mysql -uroot -p'root' -e "grant all on *.* to test@'127.0.0.1' identified by 'test';FLUSH PRIVILEGES;" mysql -uroot -p'root' -e "show databases;show variables like 'explicit_defaults_for_timestamp';" - + # - name: Setup MySQL # uses: samin/mysql-action@v1 # with: # host port: 3306 # Optional, default value is 3306. The port of host # container name: mysql # container label: mysql - # container port: 3306 + # container port: 3306 # character set server: 'utf8mb4' - # collation server: 'utf8_general_ci' + # collation server: 'utf8_general_ci' # lower case table names: 1 # log bin: 1 # server id: 111 # binlog format: 'row' # gtid mode: 1 - # bind address: '*' + # bind address: '*' # enforce gtid consistency: 1 # skip name resolve: 1 - # mysql version: '5.7' - # mysql database: 'test' - # mysql root password: 'root' - # mysql user: 'test' - # mysql password: 'test' + # mysql version: '5.7' + # mysql database: 'test' + # mysql root password: 'root' + # mysql user: 'test' + # mysql password: 'test' # - name: docker ps # run: | # date "+%Y-%m-%d %H:%M:%S" - # docker ps - # # docker logs $(docker ps -q) - + # docker ps + # # docker logs $(docker ps -q) + - name: Waiting for MySQL to be ready run: | sleep 2 @@ -87,7 +87,7 @@ jobs: sleep 2 done echo Failed waiting for MySQL && exit 1 - + - name: Set up Go 1.13 uses: actions/setup-go@v1 with: @@ -97,6 +97,12 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v1 + - name: Install pt-online-schema-change + run: | + sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change + sudo chmod +x /usr/local/bin/pt-online-schema-change + sudo apt-get install libdbi-perl libdbd-mysql-perl + - name: "Build & Test" run: | rm -f go.sum diff --git a/.travis.yml b/.travis.yml index 29b9a633..e4ebb0fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,10 @@ before_install: - ps -ef|grep mysql script: + - sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change + - sudo chmod +x /usr/local/bin/pt-online-schema-change + - sudo apt-get install -y libdbi-perl libdbd-mysql-perl + - rm -f go.sum # - docker ps - travis_wait 30 make dev upload-coverage diff --git a/circle.yml b/circle.yml index 993566cc..ef5a3072 100644 --- a/circle.yml +++ b/circle.yml @@ -38,6 +38,8 @@ jobs: name: mysql init command: mysql -h 127.0.0.1 -u root -e "select version();create database if not exists test DEFAULT CHARACTER SET utf8;create database if not exists test_inc DEFAULT CHARACTER SET utf8;grant all on *.* to test@'127.0.0.1' identified by 'test';FLUSH PRIVILEGES;show databases;show variables like 'explicit_defaults_for_timestamp';" - run: rm -f go.sum + - run: sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change + - run: sudo chmod +x /usr/local/bin/pt-online-schema-change - run: sudo chmod +x cmd/explaintest/run-tests.sh - run: name: "Build & Test" diff --git a/server/server.go b/server/server.go index 510f26d7..3a01fb81 100644 --- a/server/server.go +++ b/server/server.go @@ -133,6 +133,12 @@ func (s *Server) skipAuth() bool { return s.cfg.Socket != "" } +func (s *Server) InitOscProcessList() { + if s.oscProcessList == nil { + s.oscProcessList = make(map[string]*util.OscProcessInfo, 0) + } +} + // NewServer creates a new Server. func NewServer(cfg *config.Config, driver IDriver) (*Server, error) { s := &Server{ diff --git a/session/osc.go b/session/osc.go index 1767a2ce..8c09671c 100644 --- a/session/osc.go +++ b/session/osc.go @@ -36,17 +36,17 @@ import ( "time" // "github.com/hanchuanchuan/goInception/ast" - // "github.com/hanchuanchuan/goInception/server" "github.com/hanchuanchuan/goInception/config" "github.com/hanchuanchuan/goInception/util" "github.com/hanchuanchuan/goInception/util/auth" - // "github.com/pingcap/errors" - "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/logic" ghostlog "github.com/outbrain/golib/log" log "github.com/sirupsen/logrus" + + "github.com/hanchuanchuan/goInception/ast" + "github.com/hanchuanchuan/goInception/format" ) var ( @@ -754,6 +754,21 @@ func (s *session) mysqlAnalyzeGhostOutput(out string, p *util.OscProcessInfo) { func (s *session) getAlterTablePostPart(sql string, isPtOSC bool) string { + sql = strings.Replace(sql, "\\", "\\\\", -1) + part, ok := s.getAlterPartSql(sql) + if !ok { + return "" + } + + // gh-ost不需要处理`,pt-osc需要处理 + if isPtOSC { + part = strings.Replace(part, "\"", "\\\"", -1) + part = strings.Replace(part, "`", "\\`", -1) + part = strings.Replace(part, "$", "\\$", -1) + } + + return part + var buf []string for _, line := range strings.Split(sql, "\n") { line = strings.TrimSpace(line) @@ -826,6 +841,47 @@ func (s *session) getAlterTablePostPart(sql string, isPtOSC bool) string { return sql } +// getAlterPartSql 获取alter子句部分 +func (s *session) getAlterPartSql(sql string) (string, bool) { + sql = strings.Replace(sql, "\n", " ", -1) + sql = strings.Replace(sql, "\r", " ", -1) + + charsetInfo, collation := s.sessionVars.GetCharsetInfo() + stmtNodes, _, err := s.parser.Parse(sql, charsetInfo, collation) + if err != nil { + s.AppendErrorMessage(err.Error()) + return "", false + } + var builder strings.Builder + var columns []string + + for _, stmtNode := range stmtNodes { + switch node := stmtNode.(type) { + case *ast.AlterTableStmt: + for _, alter := range node.Specs { + builder.Reset() + err = alter.Restore(format.NewRestoreCtx(format.DefaultRestoreFlags, &builder)) + if err != nil { + s.AppendErrorMessage(err.Error()) + return "", false + } + restoreSQL := builder.String() + columns = append(columns, restoreSQL) + } + default: + s.AppendErrorMessage(fmt.Sprintf("无效类型: %v", stmtNode)) + return "", false + } + } + + if len(columns) > 0 { + return strings.Join(columns, ", "), true + } + + s.AppendErrorMessage(fmt.Sprintf("未正确解析SQL: %s", sql)) + return "", false +} + // truncateString: 根据指定长度做字符串截断,长度溢出时转换为hash值以避免重复 func truncateString(str string, length int) string { if len(str) <= length { diff --git a/session/session_inception_common_test.go b/session/session_inception_common_test.go index 4c480707..3760f641 100644 --- a/session/session_inception_common_test.go +++ b/session/session_inception_common_test.go @@ -23,9 +23,12 @@ import ( "testing" _ "github.com/go-sql-driver/mysql" + "github.com/hanchuanchuan/goInception/ast" "github.com/hanchuanchuan/goInception/config" "github.com/hanchuanchuan/goInception/domain" "github.com/hanchuanchuan/goInception/kv" + "github.com/hanchuanchuan/goInception/parser" + "github.com/hanchuanchuan/goInception/server" "github.com/hanchuanchuan/goInception/session" "github.com/hanchuanchuan/goInception/store/mockstore" "github.com/hanchuanchuan/goInception/store/mockstore/mocktikv" @@ -34,9 +37,6 @@ import ( "github.com/jinzhu/gorm" . "github.com/pingcap/check" log "github.com/sirupsen/logrus" - - "github.com/hanchuanchuan/goInception/ast" - "github.com/hanchuanchuan/goInception/parser" ) var _ = Suite(&testCommon{}) @@ -88,6 +88,7 @@ func (s *testCommon) initSetUp(c *C) { s.store = store session.SetSchemaLease(0) session.SetStatsLease(0) + s.dom, err = session.BootstrapSession(s.store) c.Assert(err, IsNil) @@ -95,6 +96,10 @@ func (s *testCommon) initSetUp(c *C) { s.tk = testkit.NewTestKitWithInit(c, s.store) } + server := &server.Server{} + server.InitOscProcessList() + s.tk.Se.SetSessionManager(server) + cfg := config.GetGlobalConfig() _, localFile, _, _ := runtime.Caller(0) localFile = path.Dir(localFile) diff --git a/session/session_inception_exec_test.go b/session/session_inception_exec_test.go index 6f4d70a3..d4564ae8 100644 --- a/session/session_inception_exec_test.go +++ b/session/session_inception_exec_test.go @@ -54,7 +54,7 @@ func (s *testSessionIncExecSuite) TearDownTest(c *C) { s.tearDownTest(c) } -func (s *testSessionIncExecSuite) testErrorCode(c *C, sql string, errors ...*session.SQLError) { +func (s *testSessionIncExecSuite) testErrorCode(c *C, sql string, errors ...*session.SQLError) [][]interface{} { if s.tk == nil { s.tk = testkit.NewTestKitWithInit(c, s.store) } @@ -82,15 +82,16 @@ func (s *testSessionIncExecSuite) testErrorCode(c *C, sql string, errors ...*ses c.Assert(row[4], Equals, strings.Join(errMsgs, "\n"), Commentf("%v", row)) } - c.Assert(row[2], Equals, strconv.Itoa(errCode), Commentf("%v", row)) + c.Assert(row[2], Equals, strconv.Itoa(errCode), Commentf("%v", res.Rows())) // 无错误时需要校验结果是否标记为已执行 if errCode == 0 { - if !strings.Contains(row[3].(string), "Execute Successfully") { - fmt.Println(res.Rows()) + c.Assert(strings.Contains(row[3].(string), "Execute Successfully"), Equals, true, Commentf("%v", res.Rows())) + for _, row := range res.Rows() { + c.Assert(row[2], Not(Equals), "2", Commentf("%v", res.Rows())) } - c.Assert(strings.Contains(row[3].(string), "Execute Successfully"), Equals, true) - // c.Assert(row[4].(string), IsNil) } + + return res.Rows() } func (s *testSessionIncExecSuite) TestCreateTable(c *C) { @@ -1243,18 +1244,110 @@ func (s *testSessionIncExecSuite) TestAlterTable(c *C) { config.GetGlobalConfig().Inc.CheckTableComment = false config.GetGlobalConfig().Inc.EnableDropTable = true sql := "" + + sql = "drop table if exists t1;create table t1(id int auto_increment primary key,c1 int);" + s.mustRunExec(c, sql) + // 删除后添加列 - sql = "drop table if exists t1;create table t1(id int,c1 int);alter table t1 drop column c1;alter table t1 add column c1 varchar(20);" + sql = "alter table t1 drop column c1;alter table t1 add column c1 varchar(20);" s.testErrorCode(c, sql) - sql = "drop table if exists t1;create table t1(id int,c1 int);alter table t1 drop column c1,add column c1 varchar(20);" + sql = "alter table t1 drop column c1,add column c1 varchar(20);" s.testErrorCode(c, sql) // 删除后添加索引 - sql = "drop table if exists t1;create table t1(id int,c1 int,key ix(c1));alter table t1 drop index ix;alter table t1 add index ix(c1);" + sql = "drop table if exists t1;create table t1(id int primary key,c1 int,key ix(c1));" + s.mustRunExec(c, sql) + + sql = "alter table t1 drop index ix;alter table t1 add index ix(c1);" + s.testErrorCode(c, sql) + + sql = "alter table t1 drop index ix,add index ix(c1);" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column c2 varchar(20) comment '!@#$%^&*()_+[]{}\\|;:\",.<>/?';" s.testErrorCode(c, sql) - sql = "drop table if exists t1;create table t1(id int,c1 int,key ix(c1));alter table t1 drop index ix,add index ix(c1);" + sql = "alter table t1 add column c3 varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" s.testErrorCode(c, sql) + sql = "alter table t1 add column `c4` varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" + s.testErrorCode(c, sql) + +} + +func (s *testSessionIncExecSuite) TestAlterTablePtOSC(c *C) { + saved := config.GetGlobalConfig().Inc + savedOsc := config.GetGlobalConfig().Osc + defer func() { + config.GetGlobalConfig().Inc = saved + config.GetGlobalConfig().Osc = savedOsc + }() + + config.GetGlobalConfig().Inc.CheckColumnComment = false + config.GetGlobalConfig().Inc.CheckTableComment = false + config.GetGlobalConfig().Inc.EnableDropTable = true + config.GetGlobalConfig().Osc.OscOn = true + config.GetGlobalConfig().Ghost.GhostOn = false + config.GetGlobalConfig().Osc.OscMinTableSize = 0 + + sql := "drop table if exists t1;create table t1(id int auto_increment primary key,c1 int);" + s.mustRunExec(c, sql) + + // 删除后添加列 + sql = "alter table t1 drop column c1;alter table t1 add column c1 varchar(20);" + s.testErrorCode(c, sql) + + sql = "alter table t1 drop column c1,add column c1 varchar(20);" + s.testErrorCode(c, sql) + + sql = "alter table t1 drop column c1,add column c1 varchar(20) comment '123';" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column c2 varchar(20) comment '!@#$%^&*()_+[]{}\\|;:\",.<>/?';" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column c3 varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column `c4` varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" + s.testErrorCode(c, sql) +} + +func (s *testSessionIncExecSuite) TestAlterTableGhost(c *C) { + saved := config.GetGlobalConfig().Inc + savedOsc := config.GetGlobalConfig().Osc + defer func() { + config.GetGlobalConfig().Inc = saved + config.GetGlobalConfig().Osc = savedOsc + }() + + config.GetGlobalConfig().Inc.CheckColumnComment = false + config.GetGlobalConfig().Inc.CheckTableComment = false + config.GetGlobalConfig().Inc.EnableDropTable = true + config.GetGlobalConfig().Osc.OscOn = false + config.GetGlobalConfig().Ghost.GhostOn = true + config.GetGlobalConfig().Osc.OscMinTableSize = 0 + + sql := "drop table if exists t1;create table t1(id int auto_increment primary key,c1 int);" + s.mustRunExec(c, sql) + + // 删除后添加列 + sql = "alter table t1 drop column c1;alter table t1 add column c1 varchar(20);" + s.testErrorCode(c, sql) + + sql = "alter table t1 drop column c1,add column c1 varchar(20);" + s.testErrorCode(c, sql) + + sql = "alter table t1 drop column c1,add column c1 varchar(20) comment '123';" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column c2 varchar(20) comment '!@#$%^&*()_+[]{}\\|;:\",.<>/?';" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column c3 varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" + s.testErrorCode(c, sql) + + sql = "alter table t1 add column `c4` varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" + s.testErrorCode(c, sql) } From 049eef399a2b074717c3a1ec667817caa9a19ff4 Mon Sep 17 00:00:00 2001 From: hanchuanchuan Date: Thu, 26 Dec 2019 15:21:37 +0800 Subject: [PATCH 2/2] =?UTF-8?q?update:=20=E6=B7=BB=E5=8A=A0=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E6=A0=A1=E9=AA=8C,=E9=81=BF=E5=85=8D=E8=AF=AD?= =?UTF-8?q?=E6=B3=95=E8=BF=98=E5=8E=9F=E4=B8=8D=E5=AE=8C=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- session/osc.go | 12 ++++++++++++ session/session_inception_exec_test.go | 3 +++ 2 files changed, 15 insertions(+) diff --git a/session/osc.go b/session/osc.go index 8c09671c..6fc7fbb6 100644 --- a/session/osc.go +++ b/session/osc.go @@ -760,6 +760,18 @@ func (s *session) getAlterTablePostPart(sql string, isPtOSC bool) string { return "" } + // 解析后的语句长度不能小于解析前! + sqlParts := strings.Fields(sql) + if len(sqlParts) >= 4 { + newSql := strings.Join(sqlParts[3:], " ") + if len(part) < len(newSql) { + log.Errorf("origin sql: %s", sql) + log.Errorf("parsed after: %s", part) + s.AppendErrorMessage("alter子句解析失败,请联系作者或自行调整!") + return "" + } + } + // gh-ost不需要处理`,pt-osc需要处理 if isPtOSC { part = strings.Replace(part, "\"", "\\\"", -1) diff --git a/session/session_inception_exec_test.go b/session/session_inception_exec_test.go index d4564ae8..2fbaecd2 100644 --- a/session/session_inception_exec_test.go +++ b/session/session_inception_exec_test.go @@ -1350,4 +1350,7 @@ func (s *testSessionIncExecSuite) TestAlterTableGhost(c *C) { sql = "alter table t1 add column `c4` varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\";" s.testErrorCode(c, sql) + + sql = "alter table t1 add column `c5` varchar(20) comment \"!@#$%^&*()_+[]{}\\|;:',.<>/?\"; -- 测试注释" + s.testErrorCode(c, sql) }