Merge pull request #142 from hanchuanchuan/patch-new-parser

优化pt-osc审核前的ALTER子句拆分

.github/workflows/go.yml

@@ -5,7 +5,7 @@ on:
     - master
     - release/*
   pull_request:
-      
+
 jobs:
 
   build:
@@ -15,10 +15,10 @@ jobs:
 
     # - name: show mysql info
     #   run: ls -l /etc/my* || true
-    
+
     - name: Set MySQL Config
       run: |
-        sudo service mysql stop        
+        sudo service mysql stop
         sudo mkdir -p /etc/mysql/ || true
         echo '[mysqld]'            | sudo tee /etc/mysql/my.cnf
         echo 'server-id=111'         | sudo tee -a /etc/mysql/my.cnf
@@ -28,17 +28,17 @@ jobs:
         echo 'enforce_gtid_consistency = ON' | sudo tee -a /etc/mysql/my.cnf
         echo 'lower_case_table_names = 1' | sudo tee -a /etc/mysql/my.cnf
         echo 'character_set_server = utf8' | sudo tee -a /etc/mysql/my.cnf
-                        
+
         # cat /etc/mysql/my.cnf || true
 
     - name: Startup MySQL
       run: |
-        sudo service mysql start || true        
+        sudo service mysql start || true
         sudo tail -100 /var/log/mysql/error.log
 
     - name: Show MySQL Variables
       run: mysql -uroot -p'root' -e "show variables where Variable_name in ('server_id','log_bin','lower_case_table_names','version');"
-    
+
     # - name: Show MySQL buffer_pool
     #   run: mysql -uroot -p'root' -e "show variables like '%buffer_pool%'"
 
@@ -47,36 +47,36 @@ jobs:
         mysql -uroot -p'root' -e "create database if not exists test DEFAULT CHARACTER SET utf8;create database if not exists test_inc DEFAULT CHARACTER SET utf8;"
         mysql -uroot -p'root' -e "grant all on *.* to test@'127.0.0.1' identified by 'test';FLUSH PRIVILEGES;"
         mysql -uroot -p'root' -e "show databases;show variables like 'explicit_defaults_for_timestamp';"
-      
+
     # - name: Setup MySQL
     #   uses: samin/mysql-action@v1
     #   with:
     #     host port: 3306 # Optional, default value is 3306. The port of host
     #     container name: mysql
     #     container label: mysql
-    #     container port: 3306 
+    #     container port: 3306
     #     character set server: 'utf8mb4'
-    #     collation server: 'utf8_general_ci' 
+    #     collation server: 'utf8_general_ci'
     #     lower case table names: 1
     #     log bin: 1
     #     server id: 111
     #     binlog format: 'row'
     #     gtid mode: 1
-    #     bind address: '*' 
+    #     bind address: '*'
     #     enforce gtid consistency: 1
     #     skip name resolve: 1
-    #     mysql version: '5.7' 
-    #     mysql database: 'test' 
-    #     mysql root password: 'root' 
-    #     mysql user: 'test' 
-    #     mysql password: 'test' 
+    #     mysql version: '5.7'
+    #     mysql database: 'test'
+    #     mysql root password: 'root'
+    #     mysql user: 'test'
+    #     mysql password: 'test'
 
     # - name: docker ps
     #   run: |
     #     date "+%Y-%m-%d %H:%M:%S"
-    #     docker ps        
-    #     # docker logs $(docker ps -q)         
-      
+    #     docker ps
+    #     # docker logs $(docker ps -q)
+
     - name: Waiting for MySQL to be ready
       run: |
         sleep 2
@@ -87,7 +87,7 @@ jobs:
           sleep 2
         done
         echo Failed waiting for MySQL && exit 1
-     
+
     - name: Set up Go 1.13
       uses: actions/setup-go@v1
       with:
@@ -97,6 +97,12 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v1
 
+    - name: Install pt-online-schema-change
+      run: |
+        sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change
+        sudo chmod +x /usr/local/bin/pt-online-schema-change
+        sudo apt-get install libdbi-perl libdbd-mysql-perl
+
     - name: "Build & Test"
       run: |
         rm -f go.sum

.travis.yml

@@ -68,6 +68,10 @@ before_install:
   - ps -ef|grep mysql
 
 script:
+  - sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change
+  - sudo chmod +x /usr/local/bin/pt-online-schema-change
+  - sudo apt-get install -y libdbi-perl libdbd-mysql-perl
+
   - rm -f go.sum
   # - docker ps
   - travis_wait 30 make dev upload-coverage

circle.yml

@@ -38,6 +38,8 @@ jobs:
           name: mysql init
           command: mysql -h 127.0.0.1 -u root -e "select version();create database if not exists test DEFAULT CHARACTER SET utf8;create database if not exists test_inc DEFAULT CHARACTER SET utf8;grant all on *.* to test@'127.0.0.1' identified by 'test';FLUSH PRIVILEGES;show databases;show variables like 'explicit_defaults_for_timestamp';"
       - run: rm -f go.sum
+      - run: sudo wget -O /usr/local/bin/pt-online-schema-change percona.com/get/pt-online-schema-change
+      - run: sudo chmod +x /usr/local/bin/pt-online-schema-change
       - run: sudo chmod +x cmd/explaintest/run-tests.sh
       - run:
           name: "Build & Test"

server/server.go

@@ -133,6 +133,12 @@ func (s *Server) skipAuth() bool {
 	return s.cfg.Socket != ""
 }
 
+func (s *Server) InitOscProcessList() {
+	if s.oscProcessList == nil {
+		s.oscProcessList = make(map[string]*util.OscProcessInfo, 0)
+	}
+}
+
 // NewServer creates a new Server.
 func NewServer(cfg *config.Config, driver IDriver) (*Server, error) {
 	s := &Server{

session/osc.go

@@ -36,17 +36,17 @@ import (
 	"time"
 
 	// "github.com/hanchuanchuan/goInception/ast"
-	// "github.com/hanchuanchuan/goInception/server"
 	"github.com/hanchuanchuan/goInception/config"
 	"github.com/hanchuanchuan/goInception/util"
 	"github.com/hanchuanchuan/goInception/util/auth"
 
-	// "github.com/pingcap/errors"
-
 	"github.com/github/gh-ost/go/base"
 	"github.com/github/gh-ost/go/logic"
 	ghostlog "github.com/outbrain/golib/log"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/hanchuanchuan/goInception/ast"
+	"github.com/hanchuanchuan/goInception/format"
 )
 
 var (
@@ -754,6 +754,33 @@ func (s *session) mysqlAnalyzeGhostOutput(out string, p *util.OscProcessInfo) {
 
 func (s *session) getAlterTablePostPart(sql string, isPtOSC bool) string {
 
+	sql = strings.Replace(sql, "\\", "\\\\", -1)
+	part, ok := s.getAlterPartSql(sql)
+	if !ok {
+		return ""
+	}
+
+	// 解析后的语句长度不能小于解析前!
+	sqlParts := strings.Fields(sql)
+	if len(sqlParts) >= 4 {
+		newSql := strings.Join(sqlParts[3:], " ")
+		if len(part) < len(newSql) {
+			log.Errorf("origin sql: %s", sql)
+			log.Errorf("parsed after: %s", part)
+			s.AppendErrorMessage("alter子句解析失败,请联系作者或自行调整!")
+			return ""
+		}
+	}
+
+	// gh-ost不需要处理`,pt-osc需要处理
+	if isPtOSC {
+		part = strings.Replace(part, "\"", "\\\"", -1)
+		part = strings.Replace(part, "`", "\\`", -1)
+		part = strings.Replace(part, "$", "\\$", -1)
+	}
+
+	return part
+
 	var buf []string
 	for _, line := range strings.Split(sql, "\n") {
 		line = strings.TrimSpace(line)
@@ -826,6 +853,47 @@ func (s *session) getAlterTablePostPart(sql string, isPtOSC bool) string {
 	return sql
 }
 
+// getAlterPartSql 获取alter子句部分
+func (s *session) getAlterPartSql(sql string) (string, bool) {
+	sql = strings.Replace(sql, "\n", " ", -1)
+	sql = strings.Replace(sql, "\r", " ", -1)
+
+	charsetInfo, collation := s.sessionVars.GetCharsetInfo()
+	stmtNodes, _, err := s.parser.Parse(sql, charsetInfo, collation)
+	if err != nil {
+		s.AppendErrorMessage(err.Error())
+		return "", false
+	}
+	var builder strings.Builder
+	var columns []string
+
+	for _, stmtNode := range stmtNodes {
+		switch node := stmtNode.(type) {
+		case *ast.AlterTableStmt:
+			for _, alter := range node.Specs {
+				builder.Reset()
+				err = alter.Restore(format.NewRestoreCtx(format.DefaultRestoreFlags, &builder))
+				if err != nil {
+					s.AppendErrorMessage(err.Error())
+					return "", false
+				}
+				restoreSQL := builder.String()
+				columns = append(columns, restoreSQL)
+			}
+		default:
+			s.AppendErrorMessage(fmt.Sprintf("无效类型: %v", stmtNode))
+			return "", false
+		}
+	}
+
+	if len(columns) > 0 {
+		return strings.Join(columns, ", "), true
+	}
+
+	s.AppendErrorMessage(fmt.Sprintf("未正确解析SQL: %s", sql))
+	return "", false
+}
+
 // truncateString: 根据指定长度做字符串截断,长度溢出时转换为hash值以避免重复
 func truncateString(str string, length int) string {
 	if len(str) <= length {

session/session_inception_common_test.go

@@ -23,9 +23,12 @@ import (
 	"testing"
 
 	_ "github.com/go-sql-driver/mysql"
+	"github.com/hanchuanchuan/goInception/ast"
 	"github.com/hanchuanchuan/goInception/config"
 	"github.com/hanchuanchuan/goInception/domain"
 	"github.com/hanchuanchuan/goInception/kv"
+	"github.com/hanchuanchuan/goInception/parser"
+	"github.com/hanchuanchuan/goInception/server"
 	"github.com/hanchuanchuan/goInception/session"
 	"github.com/hanchuanchuan/goInception/store/mockstore"
 	"github.com/hanchuanchuan/goInception/store/mockstore/mocktikv"
@@ -34,9 +37,6 @@ import (
 	"github.com/jinzhu/gorm"
 	. "github.com/pingcap/check"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/hanchuanchuan/goInception/ast"
-	"github.com/hanchuanchuan/goInception/parser"
 )
 
 var _ = Suite(&testCommon{})
@@ -88,13 +88,18 @@ func (s *testCommon) initSetUp(c *C) {
 	s.store = store
 	session.SetSchemaLease(0)
 	session.SetStatsLease(0)
+
 	s.dom, err = session.BootstrapSession(s.store)
 	c.Assert(err, IsNil)
 
 	if s.tk == nil {
 		s.tk = testkit.NewTestKitWithInit(c, s.store)
 	}
 
+	server := &server.Server{}
+	server.InitOscProcessList()
+	s.tk.Se.SetSessionManager(server)
+
 	cfg := config.GetGlobalConfig()
 	_, localFile, _, _ := runtime.Caller(0)
 	localFile = path.Dir(localFile)