From f81c971ff2329fda601db75d310d35e242e0c106 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Mon, 4 Mar 2019 21:27:22 -0500 Subject: [PATCH 01/10] Make tests more readable --- migrate_test.go | 2 - migration_sql_test.go | 86 ++++++++----------------------------------- 2 files changed, 16 insertions(+), 72 deletions(-) diff --git a/migrate_test.go b/migrate_test.go index c14f6b59b..e1b5802b2 100644 --- a/migrate_test.go +++ b/migrate_test.go @@ -9,7 +9,6 @@ func newMigration(v int64, src string) *Migration { } func TestMigrationSort(t *testing.T) { - ms := Migrations{} // insert in any order @@ -26,7 +25,6 @@ func TestMigrationSort(t *testing.T) { } func validateMigrationSort(t *testing.T, ms Migrations, sorted []int64) { - for i, m := range ms { if sorted[i] != m.Version { t.Error("incorrect sorted version") diff --git a/migration_sql_test.go b/migration_sql_test.go index a12a82878..07a20e732 100644 --- a/migration_sql_test.go +++ b/migration_sql_test.go @@ -7,37 +7,18 @@ import ( ) func TestSemicolons(t *testing.T) { - type testData struct { line string result bool } tests := []testData{ - { - line: "END;", - result: true, - }, - { - line: "END; -- comment", - result: true, - }, - { - line: "END ; -- comment", - result: true, - }, - { - line: "END -- comment", - result: false, - }, - { - line: "END -- comment ;", - result: false, - }, - { - line: "END \" ; \" -- comment", - result: false, - }, + {line: "END;", result: true}, + {line: "END; -- comment", result: true}, + {line: "END ; -- comment", result: true}, + {line: "END -- comment", result: false}, + {line: "END -- comment ;", result: false}, + {line: "END \" ; \" -- comment", result: false}, } for _, test := range tests { @@ -49,7 +30,6 @@ func TestSemicolons(t *testing.T) { } func TestSplitStatements(t *testing.T) { - type testData struct { sql string direction bool @@ -57,26 +37,10 @@ func TestSplitStatements(t *testing.T) { } tests := []testData{ - { - sql: functxt, - direction: true, - count: 2, - }, - { - sql: functxt, - direction: false, - count: 2, - }, - { - sql: multitxt, - direction: true, - count: 2, - }, - { - sql: multitxt, - direction: false, - count: 2, - }, + {sql: functxt, direction: true, count: 2}, + {sql: functxt, direction: false, count: 2}, + {sql: multitxt, direction: true, count: 2}, + {sql: multitxt, direction: false, count: 2}, } for _, test := range tests { @@ -97,18 +61,9 @@ func TestUseTransactions(t *testing.T) { } tests := []testData{ - { - fileName: "./examples/sql-migrations/00001_create_users_table.sql", - useTransactions: true, - }, - { - fileName: "./examples/sql-migrations/00002_rename_root.sql", - useTransactions: true, - }, - { - fileName: "./examples/sql-migrations/00003_no_transaction.sql", - useTransactions: false, - }, + {fileName: "./examples/sql-migrations/00001_create_users_table.sql", useTransactions: true}, + {fileName: "./examples/sql-migrations/00002_rename_root.sql", useTransactions: true}, + {fileName: "./examples/sql-migrations/00003_no_transaction.sql", useTransactions: false}, } for _, test := range tests { @@ -133,18 +88,9 @@ func TestParsingErrors(t *testing.T) { error bool } tests := []testData{ - { - sql: statementBeginNoStatementEnd, - error: true, - }, - { - sql: unfinishedSQL, - error: true, - }, - { - sql: noUpDownAnnotations, - error: true, - }, + {sql: statementBeginNoStatementEnd, error: true}, + {sql: unfinishedSQL, error: true}, + {sql: noUpDownAnnotations, error: true}, } for _, test := range tests { _, _, err := getSQLStatements(strings.NewReader(test.sql), true) From 456f34d42d13e6483cf81da0b5cd2685a72197f9 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 01:11:22 -0500 Subject: [PATCH 02/10] Kick off new SQL parser --- migration_sql.go | 146 +------------------------------ migration_sql_test.go | 73 ++++++++-------- parser.go | 195 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 179 deletions(-) create mode 100644 parser.go diff --git a/migration_sql.go b/migration_sql.go index ddafb2c18..850c094e1 100644 --- a/migration_sql.go +++ b/migration_sql.go @@ -1,153 +1,13 @@ package goose import ( - "bufio" - "bytes" "database/sql" - "fmt" - "io" "os" "regexp" - "strings" - "sync" "github.com/pkg/errors" ) -const sqlCmdPrefix = "-- +goose " -const scanBufSize = 4 * 1024 * 1024 - -var bufferPool = sync.Pool{ - New: func() interface{} { - return make([]byte, scanBufSize) - }, -} - -// Checks the line to see if the line has a statement-ending semicolon -// or if the line contains a double-dash comment. -func endsWithSemicolon(line string) bool { - scanBuf := bufferPool.Get().([]byte) - defer bufferPool.Put(scanBuf) - - prev := "" - scanner := bufio.NewScanner(strings.NewReader(line)) - scanner.Buffer(scanBuf, scanBufSize) - scanner.Split(bufio.ScanWords) - - for scanner.Scan() { - word := scanner.Text() - if strings.HasPrefix(word, "--") { - break - } - prev = word - } - - return strings.HasSuffix(prev, ";") -} - -// Split the given sql script into individual statements. -// -// The base case is to simply split on semicolons, as these -// naturally terminate a statement. -// -// However, more complex cases like pl/pgsql can have semicolons -// within a statement. For these cases, we provide the explicit annotations -// 'StatementBegin' and 'StatementEnd' to allow the script to -// tell us to ignore semicolons. -func getSQLStatements(r io.Reader, direction bool) ([]string, bool, error) { - var buf bytes.Buffer - scanBuf := bufferPool.Get().([]byte) - defer bufferPool.Put(scanBuf) - - scanner := bufio.NewScanner(r) - scanner.Buffer(scanBuf, scanBufSize) - - // track the count of each section - // so we can diagnose scripts with no annotations - upSections := 0 - downSections := 0 - - statementEnded := false - ignoreSemicolons := false - directionIsActive := false - tx := true - stmts := []string{} - - for scanner.Scan() { - - line := scanner.Text() - - // handle any goose-specific commands - if strings.HasPrefix(line, sqlCmdPrefix) { - cmd := strings.TrimSpace(line[len(sqlCmdPrefix):]) - switch cmd { - case "Up": - directionIsActive = (direction == true) - upSections++ - break - - case "Down": - directionIsActive = (direction == false) - downSections++ - break - - case "StatementBegin": - if directionIsActive { - ignoreSemicolons = true - } - break - - case "StatementEnd": - if directionIsActive { - statementEnded = (ignoreSemicolons == true) - ignoreSemicolons = false - } - break - - case "NO TRANSACTION": - tx = false - break - } - } - - if !directionIsActive { - continue - } - - if _, err := buf.WriteString(line + "\n"); err != nil { - return nil, false, fmt.Errorf("io err: %v", err) - } - - // Wrap up the two supported cases: 1) basic with semicolon; 2) psql statement - // Lines that end with semicolon that are in a statement block - // do not conclude statement. - if (!ignoreSemicolons && endsWithSemicolon(line)) || statementEnded { - statementEnded = false - stmts = append(stmts, buf.String()) - buf.Reset() - } - } - - if err := scanner.Err(); err != nil { - return nil, false, fmt.Errorf("scanning migration: %v", err) - } - - // diagnose likely migration script errors - if ignoreSemicolons { - return nil, false, fmt.Errorf("parsing migration: saw '-- +goose StatementBegin' with no matching '-- +goose StatementEnd'") - } - - if bufferRemaining := strings.TrimSpace(buf.String()); len(bufferRemaining) > 0 { - return nil, false, fmt.Errorf("parsing migration: unexpected unfinished SQL query: %s. potential missing semicolon", bufferRemaining) - } - - if upSections == 0 && downSections == 0 { - return nil, false, fmt.Errorf("parsing migration: no Up/Down annotations found, so no statements were executed. See https://bitbucket.org/liamstask/goose/overview for details") - } - - return stmts, tx, nil -} - // Run a migration specified in raw SQL. // // Sections of the script can be annotated with a special comment, @@ -163,9 +23,9 @@ func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error } defer f.Close() - statements, useTx, err := getSQLStatements(f, direction) + statements, useTx, err := parseSQLMigrationFile(f, direction) if err != nil { - return err + return errors.Wrap(err, "failed to parse SQL migration file") } if useTx { @@ -231,7 +91,7 @@ func printInfo(s string, args ...interface{}) { var ( matchSQLComments = regexp.MustCompile(`(?m)^--.*$[\r\n]*`) - matchEmptyLines = regexp.MustCompile(`(?m)^$[\r\n]*`) + matchEmptyLines = regexp.MustCompile(`(?m)^$[\r\n]*`) // TODO: Duplicate ) func clearStatement(s string) string { diff --git a/migration_sql_test.go b/migration_sql_test.go index 07a20e732..010462683 100644 --- a/migration_sql_test.go +++ b/migration_sql_test.go @@ -83,48 +83,45 @@ func TestUseTransactions(t *testing.T) { } func TestParsingErrors(t *testing.T) { - type testData struct { - sql string - error bool - } - tests := []testData{ - {sql: statementBeginNoStatementEnd, error: true}, - {sql: unfinishedSQL, error: true}, - {sql: noUpDownAnnotations, error: true}, + tt := []string{ + statementBeginNoStatementEnd, + unfinishedSQL, + noUpDownAnnotations, + emptySQL, } - for _, test := range tests { - _, _, err := getSQLStatements(strings.NewReader(test.sql), true) + for _, sql := range tt { + _, _, err := getSQLStatements(strings.NewReader(sql), true) if err == nil { - t.Errorf("Failed transaction check. got %v, want %v", err, test.error) + t.Errorf("expected error on %q", sql) } } } var functxt = `-- +goose Up CREATE TABLE IF NOT EXISTS histories ( - id BIGSERIAL PRIMARY KEY, - current_value varchar(2000) NOT NULL, - created_at timestamp with time zone NOT NULL + id BIGSERIAL PRIMARY KEY, + current_value varchar(2000) NOT NULL, + created_at timestamp with time zone NOT NULL ); -- +goose StatementBegin CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE ) returns void AS $$ DECLARE - create_query text; + create_query text; BEGIN - FOR create_query IN SELECT - 'CREATE TABLE IF NOT EXISTS histories_' - || TO_CHAR( d, 'YYYY_MM' ) - || ' ( CHECK( created_at >= timestamp ''' - || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) - || ''' AND created_at < timestamp ''' - || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) - || ''' ) ) inherits ( histories );' - FROM generate_series( $1, $2, '1 month' ) AS d - LOOP - EXECUTE create_query; - END LOOP; -- LOOP END + FOR create_query IN SELECT + 'CREATE TABLE IF NOT EXISTS histories_' + || TO_CHAR( d, 'YYYY_MM' ) + || ' ( CHECK( created_at >= timestamp ''' + || TO_CHAR( d, 'YYYY-MM-DD 00:00:00' ) + || ''' AND created_at < timestamp ''' + || TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' ) + || ''' ) ) inherits ( histories );' + FROM generate_series( $1, $2, '1 month' ) AS d + LOOP + EXECUTE create_query; + END LOOP; -- LOOP END END; -- FUNCTION END $$ language plpgsql; @@ -138,10 +135,10 @@ drop TABLE histories; // test multiple up/down transitions in a single script var multitxt = `-- +goose Up CREATE TABLE post ( - id int NOT NULL, - title text, - body text, - PRIMARY KEY(id) + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) ); -- +goose Down @@ -149,11 +146,11 @@ DROP TABLE post; -- +goose Up CREATE TABLE fancier_post ( - id int NOT NULL, - title text, - body text, - created_on timestamp without time zone, - PRIMARY KEY(id) + id int NOT NULL, + title text, + body text, + created_on timestamp without time zone, + PRIMARY KEY(id) ); -- +goose Down @@ -200,6 +197,10 @@ ALTER TABLE post -- +goose Down ` + +var emptySQL = `-- +goose Up +-- This is just a comment` + var noUpDownAnnotations = ` CREATE TABLE post ( id int NOT NULL, diff --git a/parser.go b/parser.go new file mode 100644 index 000000000..22d2d2a0d --- /dev/null +++ b/parser.go @@ -0,0 +1,195 @@ +package goose + +import ( + "bufio" + "bytes" + "io" + "regexp" + "strings" + "sync" + + "github.com/pkg/errors" +) + +type parserState int + +const ( + start parserState = iota + gooseUp + gooseStatementBeginUp + gooseStatementEndUp + gooseDown + gooseStatementBeginDown + gooseStatementEndDown +) + +const scanBufSize = 4 * 1024 * 1024 + +var matchEmptyLines = regexp.MustCompile(`^\s*$`) + +var bufferPool = sync.Pool{ + New: func() interface{} { + return make([]byte, scanBufSize) + }, +} + +// Split given SQL script into individual statements and return +// SQL statements for given direction (up=true, down=false). +// +// The base case is to simply split on semicolons, as these +// naturally terminate a statement. +// +// However, more complex cases like pl/pgsql can have semicolons +// within a statement. For these cases, we provide the explicit annotations +// 'StatementBegin' and 'StatementEnd' to allow the script to +// tell us to ignore semicolons. +func parseSQLMigrationFile(r io.Reader, direction bool) (stmts []string, useTx bool, err error) { + var buf bytes.Buffer + scanBuf := bufferPool.Get().([]byte) + defer bufferPool.Put(scanBuf) + + scanner := bufio.NewScanner(r) + scanner.Buffer(scanBuf, scanBufSize) + + stateMachine := start + useTx = true + + for scanner.Scan() { + line := scanner.Text() + + const goosePrefix = "-- +goose " + if strings.HasPrefix(line, goosePrefix) { + cmd := strings.TrimSpace(line[len(goosePrefix):]) + + switch cmd { + case "Up": + switch stateMachine { + case start: + stateMachine = gooseUp + default: + return nil, false, errors.New("failed to parse SQL migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") + } + + case "Down": + switch stateMachine { + case gooseUp, gooseStatementBeginUp: + stateMachine = gooseDown + default: + return nil, false, errors.New("failed to parse SQL migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") + } + + case "StatementBegin": + switch stateMachine { + case gooseUp: + stateMachine = gooseStatementBeginUp + case gooseDown: + stateMachine = gooseStatementBeginDown + default: + return nil, false, errors.New("failed to parse SQL migration: '-- +goose StatementBegin' must be defined after '-- +goose Up' or '-- +goose Down' annotation, see https://github.com/pressly/goose#sql-migrations") + } + + case "StatementEnd": + switch stateMachine { + case gooseStatementBeginUp: + stateMachine = gooseStatementEndUp + case gooseStatementBeginDown: + stateMachine = gooseStatementEndDown + default: + return nil, false, errors.New("failed to parse SQL migration: '-- +goose StatementEnd' must be defined after '-- +goose StatementBegin', see https://github.com/pressly/goose#sql-migrations") + } + + case "NO TRANSACTION": + useTx = false + + default: + return nil, false, errors.Errorf("unknown annotation %q", cmd) + } + } + + // Ignore comments. + if strings.HasPrefix(line, `--`) { + continue + } + // Ignore empty lines. + if matchEmptyLines.MatchString(line) { + continue + } + + // Write SQL line to a buffer. + if _, err := buf.WriteString(line + "\n"); err != nil { + return nil, false, errors.Wrap(err, "failed to write to buf") + } + + // Read SQL body one by line, if we're in the right direction. + // + // 1) basic query with semicolon; 2) psql statement + // + // Export statement once we hit end of statement. + switch stateMachine { + case gooseUp: + if !endsWithSemicolon(line) { + return nil, false, errors.Errorf("failed to parse Up SQL migration: %q: simple query must be terminated by semicolon;", line) + } + if direction { // up + stmts = append(stmts, buf.String()) + } + case gooseDown: + if !endsWithSemicolon(line) { + return nil, false, errors.Errorf("failed to parse Down SQL migration: %q: simple query must be terminated by semicolon;", line) + } + if !direction { // down + stmts = append(stmts, buf.String()) + } + case gooseStatementEndUp: + if direction /*up*/ && endsWithSemicolon(line) { + stmts = append(stmts, buf.String()) + } + case gooseStatementEndDown: + if !direction /*down*/ && endsWithSemicolon(line) { + stmts = append(stmts, buf.String()) + } + default: + return nil, false, errors.New("failed to parse migration: unexpected state %q, see https://github.com/pressly/goose#sql-migrations") + } + buf.Reset() + } + if err := scanner.Err(); err != nil { + return nil, false, errors.Wrap(err, "failed to scan migration") + } + // EOF + + switch stateMachine { + case start: + return nil, false, errors.New("failed to parse migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") + case gooseStatementBeginUp, gooseStatementBeginDown: + return nil, false, errors.New("failed to parse migration: missing '-- +goose StatementEnd' annotation") + } + + if bufferRemaining := strings.TrimSpace(buf.String()); len(bufferRemaining) > 0 { + return nil, false, errors.Errorf("failed to parse migration: unexpected unfinished SQL query: %q: missing semicolon?", bufferRemaining) + } + + return stmts, useTx, nil +} + +// Checks the line to see if the line has a statement-ending semicolon +// or if the line contains a double-dash comment. +func endsWithSemicolon(line string) bool { + scanBuf := bufferPool.Get().([]byte) + defer bufferPool.Put(scanBuf) + + prev := "" + scanner := bufio.NewScanner(strings.NewReader(line)) + scanner.Buffer(scanBuf, scanBufSize) + scanner.Split(bufio.ScanWords) + + for scanner.Scan() { + word := scanner.Text() + if strings.HasPrefix(word, "--") { + break + } + prev = word + } + + return strings.HasSuffix(prev, ";") +} From 94c2f5149689cf483b011b9f36312c9be82496f6 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 01:12:32 -0500 Subject: [PATCH 03/10] Refactor; make the new SQL parser build --- migration.go | 44 ++++++++++++++++----- migration_sql.go | 34 ++++++---------- parser.go => sql_parser.go | 0 migration_sql_test.go => sql_parser_test.go | 0 4 files changed, 45 insertions(+), 33 deletions(-) rename parser.go => sql_parser.go (100%) rename migration_sql_test.go => sql_parser_test.go (100%) diff --git a/migration.go b/migration.go index 8b3dd34e9..c14232875 100644 --- a/migration.go +++ b/migration.go @@ -3,6 +3,7 @@ package goose import ( "database/sql" "fmt" + "os" "path/filepath" "strconv" "strings" @@ -38,7 +39,6 @@ func (m *Migration) Up(db *sql.DB) error { if err := m.run(db, true); err != nil { return err } - log.Println("OK ", filepath.Base(m.Source)) return nil } @@ -47,51 +47,75 @@ func (m *Migration) Down(db *sql.DB) error { if err := m.run(db, false); err != nil { return err } - log.Println("OK ", filepath.Base(m.Source)) return nil } func (m *Migration) run(db *sql.DB, direction bool) error { switch filepath.Ext(m.Source) { case ".sql": - if err := runSQLMigration(db, m.Source, m.Version, direction); err != nil { - return errors.Wrapf(err, "failed to run SQL migration %q", filepath.Base(m.Source)) + f, err := os.Open(m.Source) + if err != nil { + return errors.Wrapf(err, "ERROR %v: failed to open SQL migration file", filepath.Base(m.Source)) + } + defer f.Close() + + statements, useTx, err := parseSQLMigrationFile(f, direction) + if err != nil { + return errors.Wrapf(err, "ERROR %v: failed to parse SQL migration file", filepath.Base(m.Source)) + } + + if err := runSQLMigration(db, statements, useTx, m.Version, direction); err != nil { + return errors.Wrapf(err, "ERROR %v: failed to run SQL migration", filepath.Base(m.Source)) + } + + if len(statements) > 0 { + log.Println("OK ", filepath.Base(m.Source)) + } else { + log.Println("EMPTY", filepath.Base(m.Source)) } case ".go": if !m.Registered { - return errors.Errorf("failed to run Go migration %q: Go functions must be registered and built into a custom binary (see https://github.com/pressly/goose/tree/master/examples/go-migrations)", m.Source) + return errors.Errorf("ERROR %v: failed to run Go migration: Go functions must be registered and built into a custom binary (see https://github.com/pressly/goose/tree/master/examples/go-migrations)", m.Source) } tx, err := db.Begin() if err != nil { - return errors.Wrap(err, "failed to begin transaction") + return errors.Wrap(err, "ERROR failed to begin transaction") } fn := m.UpFn if !direction { fn = m.DownFn } + if fn != nil { + // Run Go migration function. if err := fn(tx); err != nil { tx.Rollback() - return errors.Wrapf(err, "failed to run Go migration %q", filepath.Base(m.Source)) + return errors.Wrapf(err, "ERROR %v: failed to run Go migration function %T", filepath.Base(m.Source), fn) } } if direction { if _, err := tx.Exec(GetDialect().insertVersionSQL(), m.Version, direction); err != nil { tx.Rollback() - return errors.Wrap(err, "failed to execute transaction") + return errors.Wrap(err, "ERROR failed to execute transaction") } } else { if _, err := tx.Exec(GetDialect().deleteVersionSQL(), m.Version); err != nil { tx.Rollback() - return errors.Wrap(err, "failed to execute transaction") + return errors.Wrap(err, "ERROR failed to execute transaction") } } if err := tx.Commit(); err != nil { - return errors.Wrap(err, "failed to commit transaction") + return errors.Wrap(err, "ERROR failed to commit transaction") + } + + if fn != nil { + log.Println("OK ", filepath.Base(m.Source)) + } else { + log.Println("EMPTY", filepath.Base(m.Source)) } return nil diff --git a/migration_sql.go b/migration_sql.go index 850c094e1..c57313d66 100644 --- a/migration_sql.go +++ b/migration_sql.go @@ -2,7 +2,6 @@ package goose import ( "database/sql" - "os" "regexp" "github.com/pkg/errors" @@ -16,22 +15,11 @@ import ( // // All statements following an Up or Down directive are grouped together // until another direction directive is found. -func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error { - f, err := os.Open(sqlFile) - if err != nil { - return errors.Wrap(err, "failed to open SQL migration file") - } - defer f.Close() - - statements, useTx, err := parseSQLMigrationFile(f, direction) - if err != nil { - return errors.Wrap(err, "failed to parse SQL migration file") - } - +func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direction bool) error { if useTx { // TRANSACTION. - printInfo("Begin transaction\n") + verboseInfo("Begin transaction\n") tx, err := db.Begin() if err != nil { @@ -39,9 +27,9 @@ func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error } for _, query := range statements { - printInfo("Executing statement: %s\n", clearStatement(query)) + verboseInfo("Executing statement: %s\n", clearStatement(query)) if _, err = tx.Exec(query); err != nil { - printInfo("Rollback transaction\n") + verboseInfo("Rollback transaction\n") tx.Rollback() return errors.Wrapf(err, "failed to execute SQL query %q", clearStatement(query)) } @@ -49,19 +37,19 @@ func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error if direction { if _, err := tx.Exec(GetDialect().insertVersionSQL(), v, direction); err != nil { - printInfo("Rollback transaction\n") + verboseInfo("Rollback transaction\n") tx.Rollback() return errors.Wrap(err, "failed to insert new goose version") } } else { if _, err := tx.Exec(GetDialect().deleteVersionSQL(), v); err != nil { - printInfo("Rollback transaction\n") + verboseInfo("Rollback transaction\n") tx.Rollback() return errors.Wrap(err, "failed to delete goose version") } } - printInfo("Commit transaction\n") + verboseInfo("Commit transaction\n") if err := tx.Commit(); err != nil { return errors.Wrap(err, "failed to commit transaction") } @@ -71,7 +59,7 @@ func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error // NO TRANSACTION. for _, query := range statements { - printInfo("Executing statement: %s\n", clearStatement(query)) + verboseInfo("Executing statement: %s\n", clearStatement(query)) if _, err := db.Exec(query); err != nil { return errors.Wrapf(err, "failed to execute SQL query %q", clearStatement(query)) } @@ -83,7 +71,7 @@ func runSQLMigration(db *sql.DB, sqlFile string, v int64, direction bool) error return nil } -func printInfo(s string, args ...interface{}) { +func verboseInfo(s string, args ...interface{}) { if verbose { log.Printf(s, args...) } @@ -91,10 +79,10 @@ func printInfo(s string, args ...interface{}) { var ( matchSQLComments = regexp.MustCompile(`(?m)^--.*$[\r\n]*`) - matchEmptyLines = regexp.MustCompile(`(?m)^$[\r\n]*`) // TODO: Duplicate + matchEmptyEOL = regexp.MustCompile(`(?m)^$[\r\n]*`) // TODO: Duplicate ) func clearStatement(s string) string { s = matchSQLComments.ReplaceAllString(s, ``) - return matchEmptyLines.ReplaceAllString(s, ``) + return matchEmptyEOL.ReplaceAllString(s, ``) } diff --git a/parser.go b/sql_parser.go similarity index 100% rename from parser.go rename to sql_parser.go diff --git a/migration_sql_test.go b/sql_parser_test.go similarity index 100% rename from migration_sql_test.go rename to sql_parser_test.go From 3472cd6ee8d5251fe04ad1a62c3e39c6512ad674 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 01:14:34 -0500 Subject: [PATCH 04/10] Embrace io.Reader --- migration.go | 2 +- sql_parser.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migration.go b/migration.go index c14232875..68e579f90 100644 --- a/migration.go +++ b/migration.go @@ -59,7 +59,7 @@ func (m *Migration) run(db *sql.DB, direction bool) error { } defer f.Close() - statements, useTx, err := parseSQLMigrationFile(f, direction) + statements, useTx, err := parseSQLMigration(f, direction) if err != nil { return errors.Wrapf(err, "ERROR %v: failed to parse SQL migration file", filepath.Base(m.Source)) } diff --git a/sql_parser.go b/sql_parser.go index 22d2d2a0d..a7820f39e 100644 --- a/sql_parser.go +++ b/sql_parser.go @@ -43,7 +43,7 @@ var bufferPool = sync.Pool{ // within a statement. For these cases, we provide the explicit annotations // 'StatementBegin' and 'StatementEnd' to allow the script to // tell us to ignore semicolons. -func parseSQLMigrationFile(r io.Reader, direction bool) (stmts []string, useTx bool, err error) { +func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, err error) { var buf bytes.Buffer scanBuf := bufferPool.Get().([]byte) defer bufferPool.Put(scanBuf) From 14668d05d8a0f8580dbc6320a6999f6f4a338852 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 01:38:45 -0500 Subject: [PATCH 05/10] Fix some failing SQL migrations --- sql_parser.go | 33 +++++++++++++++++++++------------ sql_parser_test.go | 6 +++--- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/sql_parser.go b/sql_parser.go index a7820f39e..077db57be 100644 --- a/sql_parser.go +++ b/sql_parser.go @@ -127,31 +127,40 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, // Export statement once we hit end of statement. switch stateMachine { case gooseUp: - if !endsWithSemicolon(line) { - return nil, false, errors.Errorf("failed to parse Up SQL migration: %q: simple query must be terminated by semicolon;", line) + if !direction /*down*/ { + buf.Reset() + break } - if direction { // up + if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) + buf.Reset() } case gooseDown: - if !endsWithSemicolon(line) { - return nil, false, errors.Errorf("failed to parse Down SQL migration: %q: simple query must be terminated by semicolon;", line) + if direction /*up*/ { + buf.Reset() + break } - if !direction { // down + if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) + buf.Reset() } case gooseStatementEndUp: - if direction /*up*/ && endsWithSemicolon(line) { - stmts = append(stmts, buf.String()) + if !direction /*down*/ { + buf.Reset() + break } + stmts = append(stmts, buf.String()) + buf.Reset() case gooseStatementEndDown: - if !direction /*down*/ && endsWithSemicolon(line) { - stmts = append(stmts, buf.String()) + if direction /*up*/ { + buf.Reset() + break } + stmts = append(stmts, buf.String()) + buf.Reset() default: return nil, false, errors.New("failed to parse migration: unexpected state %q, see https://github.com/pressly/goose#sql-migrations") } - buf.Reset() } if err := scanner.Err(); err != nil { return nil, false, errors.Wrap(err, "failed to scan migration") @@ -166,7 +175,7 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, } if bufferRemaining := strings.TrimSpace(buf.String()); len(bufferRemaining) > 0 { - return nil, false, errors.Errorf("failed to parse migration: unexpected unfinished SQL query: %q: missing semicolon?", bufferRemaining) + return nil, false, errors.Errorf("failed to parse migration: state %q, direction: %v: unexpected unfinished SQL query: %q: missing semicolon?", stateMachine, direction, bufferRemaining) } return stmts, useTx, nil diff --git a/sql_parser_test.go b/sql_parser_test.go index 010462683..00e86efae 100644 --- a/sql_parser_test.go +++ b/sql_parser_test.go @@ -44,7 +44,7 @@ func TestSplitStatements(t *testing.T) { } for _, test := range tests { - stmts, _, err := getSQLStatements(strings.NewReader(test.sql), test.direction) + stmts, _, err := parseSQLMigration(strings.NewReader(test.sql), test.direction) if err != nil { t.Error(err) } @@ -71,7 +71,7 @@ func TestUseTransactions(t *testing.T) { if err != nil { t.Error(err) } - _, useTx, err := getSQLStatements(f, true) + _, useTx, err := parseSQLMigration(f, true) if err != nil { t.Error(err) } @@ -90,7 +90,7 @@ func TestParsingErrors(t *testing.T) { emptySQL, } for _, sql := range tt { - _, _, err := getSQLStatements(strings.NewReader(sql), true) + _, _, err := parseSQLMigration(strings.NewReader(sql), true) if err == nil { t.Errorf("expected error on %q", sql) } From fff58a44df76f0e5a1c793aece1dd3611ca5aa07 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 02:53:57 -0500 Subject: [PATCH 06/10] Fix SQL parser tests --- sql_parser.go | 124 +++++++++++++++++++++++++-------------------- sql_parser_test.go | 63 +++++++++++++++++------ 2 files changed, 115 insertions(+), 72 deletions(-) diff --git a/sql_parser.go b/sql_parser.go index 077db57be..d8b86132b 100644 --- a/sql_parser.go +++ b/sql_parser.go @@ -14,15 +14,25 @@ import ( type parserState int const ( - start parserState = iota - gooseUp - gooseStatementBeginUp - gooseStatementEndUp - gooseDown - gooseStatementBeginDown - gooseStatementEndDown + start parserState = iota // 0 + gooseUp // 1 + gooseStatementBeginUp // 2 + gooseStatementEndUp // 3 + gooseDown // 4 + gooseStatementBeginDown // 5 + gooseStatementEndDown // 6 ) +type stateMachine parserState + +func (s *stateMachine) Get() parserState { + return parserState(*s) +} +func (s *stateMachine) Set(new parserState) { + verboseInfo("=> stateMachine: %v => %v", *s, new) + *s = stateMachine(new) +} + const scanBufSize = 4 * 1024 * 1024 var matchEmptyLines = regexp.MustCompile(`^\s*$`) @@ -51,67 +61,66 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, scanner := bufio.NewScanner(r) scanner.Buffer(scanBuf, scanBufSize) - stateMachine := start + stateMachine := stateMachine(start) useTx = true for scanner.Scan() { line := scanner.Text() - const goosePrefix = "-- +goose " - if strings.HasPrefix(line, goosePrefix) { - cmd := strings.TrimSpace(line[len(goosePrefix):]) + verboseInfo(" %v\n", line) + + if strings.HasPrefix(line, "--") { + cmd := strings.TrimSpace(strings.TrimPrefix(line, "--")) switch cmd { - case "Up": - switch stateMachine { + case "+goose Up": + switch stateMachine.Get() { case start: - stateMachine = gooseUp + stateMachine.Set(gooseUp) default: - return nil, false, errors.New("failed to parse SQL migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") + return nil, false, errors.Errorf("duplicate '-- +goose Up' annotations; stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } - case "Down": - switch stateMachine { - case gooseUp, gooseStatementBeginUp: - stateMachine = gooseDown + case "+goose Down": + switch stateMachine.Get() { + case gooseUp, gooseStatementEndUp: + stateMachine.Set(gooseDown) default: - return nil, false, errors.New("failed to parse SQL migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") + return nil, false, errors.Errorf("must start with '-- +goose Up' annotation, stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } - case "StatementBegin": - switch stateMachine { - case gooseUp: - stateMachine = gooseStatementBeginUp - case gooseDown: - stateMachine = gooseStatementBeginDown + case "+goose StatementBegin": + switch stateMachine.Get() { + case gooseUp, gooseStatementEndUp: + stateMachine.Set(gooseStatementBeginUp) + case gooseDown, gooseStatementEndDown: + stateMachine.Set(gooseStatementBeginDown) default: - return nil, false, errors.New("failed to parse SQL migration: '-- +goose StatementBegin' must be defined after '-- +goose Up' or '-- +goose Down' annotation, see https://github.com/pressly/goose#sql-migrations") + return nil, false, errors.Errorf("'-- +goose StatementBegin' must be defined after '-- +goose Up' or '-- +goose Down' annotation, stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } - case "StatementEnd": - switch stateMachine { + case "+goose StatementEnd": + switch stateMachine.Get() { case gooseStatementBeginUp: - stateMachine = gooseStatementEndUp + stateMachine.Set(gooseStatementEndUp) case gooseStatementBeginDown: - stateMachine = gooseStatementEndDown + stateMachine.Set(gooseStatementEndDown) default: - return nil, false, errors.New("failed to parse SQL migration: '-- +goose StatementEnd' must be defined after '-- +goose StatementBegin', see https://github.com/pressly/goose#sql-migrations") + return nil, false, errors.New("'-- +goose StatementEnd' must be defined after '-- +goose StatementBegin', see https://github.com/pressly/goose#sql-migrations") } - case "NO TRANSACTION": + case "+goose NO TRANSACTION": useTx = false default: - return nil, false, errors.Errorf("unknown annotation %q", cmd) + // Ignore comments. + verboseInfo("=> ignore comment") } } - // Ignore comments. - if strings.HasPrefix(line, `--`) { - continue - } // Ignore empty lines. if matchEmptyLines.MatchString(line) { + verboseInfo("=> ignore empty line") continue } @@ -125,41 +134,44 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, // 1) basic query with semicolon; 2) psql statement // // Export statement once we hit end of statement. - switch stateMachine { - case gooseUp: + switch stateMachine.Get() { + case gooseUp, gooseStatementBeginUp, gooseStatementEndUp: if !direction /*down*/ { buf.Reset() - break + verboseInfo("=> ignore down") + continue + } + case gooseDown, gooseStatementBeginDown, gooseStatementEndDown: + if direction /*up*/ { + buf.Reset() + verboseInfo("=> ignore up") + continue } + default: + return nil, false, errors.Errorf("failed to parse migration: unexpected state %q on line %q, see https://github.com/pressly/goose#sql-migrations", stateMachine, line) + } + + switch stateMachine.Get() { + case gooseUp: if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) buf.Reset() + verboseInfo("=> store simple up query") } case gooseDown: - if direction /*up*/ { - buf.Reset() - break - } if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) buf.Reset() + verboseInfo("=> store simple down query") } case gooseStatementEndUp: - if !direction /*down*/ { - buf.Reset() - break - } stmts = append(stmts, buf.String()) buf.Reset() + verboseInfo("=> store up statement") case gooseStatementEndDown: - if direction /*up*/ { - buf.Reset() - break - } stmts = append(stmts, buf.String()) buf.Reset() - default: - return nil, false, errors.New("failed to parse migration: unexpected state %q, see https://github.com/pressly/goose#sql-migrations") + verboseInfo("=> store down statement") } } if err := scanner.Err(); err != nil { @@ -167,7 +179,7 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, } // EOF - switch stateMachine { + switch stateMachine.Get() { case start: return nil, false, errors.New("failed to parse migration: must start with '-- +goose Up' annotation, see https://github.com/pressly/goose#sql-migrations") case gooseStatementBeginUp, gooseStatementBeginDown: diff --git a/sql_parser_test.go b/sql_parser_test.go index 00e86efae..6aaba5062 100644 --- a/sql_parser_test.go +++ b/sql_parser_test.go @@ -4,6 +4,8 @@ import ( "os" "strings" "testing" + + "github.com/pkg/errors" ) func TestSemicolons(t *testing.T) { @@ -30,26 +32,53 @@ func TestSemicolons(t *testing.T) { } func TestSplitStatements(t *testing.T) { + //SetVerbose(true) + type testData struct { - sql string - direction bool - count int + sql string + up int + down int } - tests := []testData{ - {sql: functxt, direction: true, count: 2}, - {sql: functxt, direction: false, count: 2}, - {sql: multitxt, direction: true, count: 2}, - {sql: multitxt, direction: false, count: 2}, + tt := []testData{ + {sql: `-- +goose Up +CREATE TABLE post ( + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) +); SELECT 1; + +-- comment +SELECT 2; +SELECT 3; SELECT 3; +SELECT 4; + +-- +goose Down +-- comment +DROP TABLE post; SELECT 1; -- comment +`, up: 4, down: 1}, + + {sql: functxt, up: 2, down: 2}, } - for _, test := range tests { - stmts, _, err := parseSQLMigration(strings.NewReader(test.sql), test.direction) + for i, test := range tt { + // up + stmts, _, err := parseSQLMigration(strings.NewReader(test.sql), true) if err != nil { - t.Error(err) + t.Error(errors.Wrapf(err, "tt[%v] unexpected error", i)) + } + if len(stmts) != test.up { + t.Errorf("tt[%v] incorrect number of up stmts. got %v (%+v), want %v", i, len(stmts), stmts, test.up) } - if len(stmts) != test.count { - t.Errorf("incorrect number of stmts. got %v, want %v", len(stmts), test.count) + + // down + stmts, _, err = parseSQLMigration(strings.NewReader(test.sql), false) + if err != nil { + t.Error(errors.Wrapf(err, "tt[%v] unexpected error", i)) + } + if len(stmts) != test.down { + t.Errorf("tt[%v] incorrect number of down stmts. got %v (%+v), want %v", i, len(stmts), stmts, test.down) } } } @@ -88,6 +117,8 @@ func TestParsingErrors(t *testing.T) { unfinishedSQL, noUpDownAnnotations, emptySQL, + multiUpDown, + downFirst, } for _, sql := range tt { _, _, err := parseSQLMigration(strings.NewReader(sql), true) @@ -132,8 +163,7 @@ drop function histories_partition_creation(DATE, DATE); drop TABLE histories; ` -// test multiple up/down transitions in a single script -var multitxt = `-- +goose Up +var multiUpDown = `-- +goose Up CREATE TABLE post ( id int NOT NULL, title text, @@ -152,8 +182,9 @@ CREATE TABLE fancier_post ( created_on timestamp without time zone, PRIMARY KEY(id) ); +` --- +goose Down +var downFirst = `-- +goose Down DROP TABLE fancier_post; ` From 3836c78d69796fbb7bebc508cebe34ab4becd3c6 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 03:23:37 -0500 Subject: [PATCH 07/10] Fix SQL parser errors --- sql_parser.go | 5 +++++ sql_parser_test.go | 56 ++++++++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/sql_parser.go b/sql_parser.go index d8b86132b..cbc3b2878 100644 --- a/sql_parser.go +++ b/sql_parser.go @@ -80,6 +80,7 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, default: return nil, false, errors.Errorf("duplicate '-- +goose Up' annotations; stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } + continue case "+goose Down": switch stateMachine.Get() { @@ -88,6 +89,7 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, default: return nil, false, errors.Errorf("must start with '-- +goose Up' annotation, stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } + continue case "+goose StatementBegin": switch stateMachine.Get() { @@ -98,6 +100,7 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, default: return nil, false, errors.Errorf("'-- +goose StatementBegin' must be defined after '-- +goose Up' or '-- +goose Down' annotation, stateMachine=%v, see https://github.com/pressly/goose#sql-migrations", stateMachine) } + continue case "+goose StatementEnd": switch stateMachine.Get() { @@ -111,10 +114,12 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, case "+goose NO TRANSACTION": useTx = false + continue default: // Ignore comments. verboseInfo("=> ignore comment") + continue } } diff --git a/sql_parser_test.go b/sql_parser_test.go index 6aaba5062..7dd5c5f94 100644 --- a/sql_parser_test.go +++ b/sql_parser_test.go @@ -32,7 +32,7 @@ func TestSemicolons(t *testing.T) { } func TestSplitStatements(t *testing.T) { - //SetVerbose(true) + // SetVerbose(true) type testData struct { sql string @@ -41,24 +41,9 @@ func TestSplitStatements(t *testing.T) { } tt := []testData{ - {sql: `-- +goose Up -CREATE TABLE post ( - id int NOT NULL, - title text, - body text, - PRIMARY KEY(id) -); SELECT 1; - --- comment -SELECT 2; -SELECT 3; SELECT 3; -SELECT 4; - --- +goose Down --- comment -DROP TABLE post; SELECT 1; -- comment -`, up: 4, down: 1}, - + {sql: multilineSQL, up: 4, down: 1}, + {sql: emptySQL, up: 0, down: 0}, + {sql: emptySQL2, up: 0, down: 0}, {sql: functxt, up: 2, down: 2}, } @@ -116,18 +101,35 @@ func TestParsingErrors(t *testing.T) { statementBeginNoStatementEnd, unfinishedSQL, noUpDownAnnotations, - emptySQL, multiUpDown, downFirst, } - for _, sql := range tt { + for i, sql := range tt { _, _, err := parseSQLMigration(strings.NewReader(sql), true) if err == nil { - t.Errorf("expected error on %q", sql) + t.Errorf("expected error on tt[%v] %q", i, sql) } } } +var multilineSQL = `-- +goose Up +CREATE TABLE post ( + id int NOT NULL, + title text, + body text, + PRIMARY KEY(id) +); SELECT 1; + +-- comment +SELECT 2; +SELECT 3; SELECT 3; +SELECT 4; + +-- +goose Down +-- comment +DROP TABLE post; SELECT 1; -- comment +` + var functxt = `-- +goose Up CREATE TABLE IF NOT EXISTS histories ( id BIGSERIAL PRIMARY KEY, @@ -232,6 +234,16 @@ ALTER TABLE post var emptySQL = `-- +goose Up -- This is just a comment` +var emptySQL2 = ` + +-- comment +-- +goose Up + +-- comment +-- +goose Down + +` + var noUpDownAnnotations = ` CREATE TABLE post ( id int NOT NULL, From 213f48bec60e1a23b3eddaa997e75e888c14af30 Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 14:57:27 -0500 Subject: [PATCH 08/10] Add test case for MySQL change delimiter #127 --- migration_sql.go | 19 ++++++++++++------- sql_parser.go | 23 ++++++++++++----------- sql_parser_test.go | 32 +++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/migration_sql.go b/migration_sql.go index c57313d66..8703ec8aa 100644 --- a/migration_sql.go +++ b/migration_sql.go @@ -19,7 +19,7 @@ func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direc if useTx { // TRANSACTION. - verboseInfo("Begin transaction\n") + verboseInfo("Begin transaction") tx, err := db.Begin() if err != nil { @@ -29,7 +29,7 @@ func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direc for _, query := range statements { verboseInfo("Executing statement: %s\n", clearStatement(query)) if _, err = tx.Exec(query); err != nil { - verboseInfo("Rollback transaction\n") + verboseInfo("Rollback transaction") tx.Rollback() return errors.Wrapf(err, "failed to execute SQL query %q", clearStatement(query)) } @@ -37,19 +37,19 @@ func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direc if direction { if _, err := tx.Exec(GetDialect().insertVersionSQL(), v, direction); err != nil { - verboseInfo("Rollback transaction\n") + verboseInfo("Rollback transaction") tx.Rollback() return errors.Wrap(err, "failed to insert new goose version") } } else { if _, err := tx.Exec(GetDialect().deleteVersionSQL(), v); err != nil { - verboseInfo("Rollback transaction\n") + verboseInfo("Rollback transaction") tx.Rollback() return errors.Wrap(err, "failed to delete goose version") } } - verboseInfo("Commit transaction\n") + verboseInfo("Commit transaction") if err := tx.Commit(); err != nil { return errors.Wrap(err, "failed to commit transaction") } @@ -59,7 +59,7 @@ func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direc // NO TRANSACTION. for _, query := range statements { - verboseInfo("Executing statement: %s\n", clearStatement(query)) + verboseInfo("Executing statement: %s", clearStatement(query)) if _, err := db.Exec(query); err != nil { return errors.Wrapf(err, "failed to execute SQL query %q", clearStatement(query)) } @@ -71,9 +71,14 @@ func runSQLMigration(db *sql.DB, statements []string, useTx bool, v int64, direc return nil } +const ( + grayColor = "\033[90m" + resetColor = "\033[00m" +) + func verboseInfo(s string, args ...interface{}) { if verbose { - log.Printf(s, args...) + log.Printf(grayColor+s+resetColor, args...) } } diff --git a/sql_parser.go b/sql_parser.go index cbc3b2878..27d146669 100644 --- a/sql_parser.go +++ b/sql_parser.go @@ -29,7 +29,7 @@ func (s *stateMachine) Get() parserState { return parserState(*s) } func (s *stateMachine) Set(new parserState) { - verboseInfo("=> stateMachine: %v => %v", *s, new) + verboseInfo("StateMachine: %v => %v", *s, new) *s = stateMachine(new) } @@ -66,8 +66,9 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, for scanner.Scan() { line := scanner.Text() - - verboseInfo(" %v\n", line) + if verbose { + log.Println(line) + } if strings.HasPrefix(line, "--") { cmd := strings.TrimSpace(strings.TrimPrefix(line, "--")) @@ -118,14 +119,14 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, default: // Ignore comments. - verboseInfo("=> ignore comment") + verboseInfo("StateMachine: ignore comment") continue } } // Ignore empty lines. if matchEmptyLines.MatchString(line) { - verboseInfo("=> ignore empty line") + verboseInfo("StateMachine: ignore empty line") continue } @@ -143,13 +144,13 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, case gooseUp, gooseStatementBeginUp, gooseStatementEndUp: if !direction /*down*/ { buf.Reset() - verboseInfo("=> ignore down") + verboseInfo("StateMachine: ignore down") continue } case gooseDown, gooseStatementBeginDown, gooseStatementEndDown: if direction /*up*/ { buf.Reset() - verboseInfo("=> ignore up") + verboseInfo("StateMachine: ignore up") continue } default: @@ -161,22 +162,22 @@ func parseSQLMigration(r io.Reader, direction bool) (stmts []string, useTx bool, if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) buf.Reset() - verboseInfo("=> store simple up query") + verboseInfo("StateMachine: store simple Up query") } case gooseDown: if endsWithSemicolon(line) { stmts = append(stmts, buf.String()) buf.Reset() - verboseInfo("=> store simple down query") + verboseInfo("StateMachine: store simple Down query") } case gooseStatementEndUp: stmts = append(stmts, buf.String()) buf.Reset() - verboseInfo("=> store up statement") + verboseInfo("StateMachine: store Up statement") case gooseStatementEndDown: stmts = append(stmts, buf.String()) buf.Reset() - verboseInfo("=> store down statement") + verboseInfo("StateMachine: store Down statement") } } if err := scanner.Err(); err != nil { diff --git a/sql_parser_test.go b/sql_parser_test.go index 7dd5c5f94..319b04c01 100644 --- a/sql_parser_test.go +++ b/sql_parser_test.go @@ -45,6 +45,7 @@ func TestSplitStatements(t *testing.T) { {sql: emptySQL, up: 0, down: 0}, {sql: emptySQL2, up: 0, down: 0}, {sql: functxt, up: 2, down: 2}, + {sql: mysqlChangeDelimiter, up: 4, down: 0}, } for i, test := range tt { @@ -118,16 +119,16 @@ CREATE TABLE post ( title text, body text, PRIMARY KEY(id) -); SELECT 1; +); -- 1st stmt -- comment -SELECT 2; -SELECT 3; SELECT 3; -SELECT 4; +SELECT 2; -- 2nd stmt +SELECT 3; SELECT 3; -- 3rd stmt +SELECT 4; -- 4th stmt -- +goose Down -- comment -DROP TABLE post; SELECT 1; -- comment +DROP TABLE post; -- 1st stmt ` var functxt = `-- +goose Up @@ -252,3 +253,24 @@ CREATE TABLE post ( PRIMARY KEY(id) ); ` + +var mysqlChangeDelimiter = ` +-- +goose Up +-- +goose StatementBegin +DELIMITER | +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE FUNCTION my_func( str CHAR(255) ) RETURNS CHAR(255) DETERMINISTIC +BEGIN + RETURN "Dummy Body"; +END | +-- +goose StatementEnd + +-- +goose StatementBegin +DELIMITER ; +-- +goose StatementEnd + +select my_func("123") from dual; +-- +goose Down +` From c587f982986c6ff65ede0b3f5c2e09d97eb07fae Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 15:00:59 -0500 Subject: [PATCH 09/10] Add test case for COPY FROM STDIN #138 --- sql_parser_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/sql_parser_test.go b/sql_parser_test.go index 319b04c01..886426a26 100644 --- a/sql_parser_test.go +++ b/sql_parser_test.go @@ -46,6 +46,7 @@ func TestSplitStatements(t *testing.T) { {sql: emptySQL2, up: 0, down: 0}, {sql: functxt, up: 2, down: 2}, {sql: mysqlChangeDelimiter, up: 4, down: 0}, + {sql: copyFromStdin, up: 1, down: 0}, } for i, test := range tt { @@ -274,3 +275,17 @@ DELIMITER ; select my_func("123") from dual; -- +goose Down ` + +var copyFromStdin = ` +-- +goose Up +-- +goose StatementBegin +COPY public.django_content_type (id, app_label, model) FROM stdin; +1 admin logentry +2 auth permission +3 auth group +4 auth user +5 contenttypes contenttype +6 sessions session +\. +-- +goose StatementEnd +` From 02bb13b385a567c088f54997f7f0ad508cf9204f Mon Sep 17 00:00:00 2001 From: Vojtech Vitek Date: Tue, 5 Mar 2019 15:53:27 -0500 Subject: [PATCH 10/10] Bump version to v2.7.0rc1 --- goose.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goose.go b/goose.go index 7ee20f7dd..83c5244ae 100644 --- a/goose.go +++ b/goose.go @@ -7,7 +7,7 @@ import ( "sync" ) -const VERSION = "v2.6.0" +const VERSION = "v2.7.0-rc1" var ( duplicateCheckOnce sync.Once