From 8b4ce58a1def3f7eafd857e1d78d597ab4640ca4 Mon Sep 17 00:00:00 2001 From: maxvw Date: Sat, 27 May 2017 08:56:05 +0200 Subject: [PATCH 1/2] adapted the sqlite driver for v3 (#165) --- Makefile | 2 +- README.md | 2 +- database/{sqlite => sqlite3}/README.md | 0 .../migration/33_create_table.down.sql | 1 + .../sqlite3/migration/33_create_table.up.sql | 3 + .../sqlite3/migration/44_alter_table.down.sql | 1 + .../sqlite3/migration/44_alter_table.up.sql | 1 + database/sqlite3/sqlite3.go | 214 ++++++++++++++++++ database/sqlite3/sqlite3_test.go | 61 +++++ 9 files changed, 283 insertions(+), 2 deletions(-) rename database/{sqlite => sqlite3}/README.md (100%) create mode 100644 database/sqlite3/migration/33_create_table.down.sql create mode 100644 database/sqlite3/migration/33_create_table.up.sql create mode 100644 database/sqlite3/migration/44_alter_table.down.sql create mode 100644 database/sqlite3/migration/44_alter_table.up.sql create mode 100644 database/sqlite3/sqlite3.go create mode 100644 database/sqlite3/sqlite3_test.go diff --git a/Makefile b/Makefile index 34e737ef..408cc30b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go-bindata github aws-s3 google-cloud-storage -DATABASE ?= postgres mysql redshift +DATABASE ?= postgres mysql redshift sqlite3 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index 2327f7d4..ae301dfd 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [Redshift](database/redshift) * [Ql](database/ql) * [Cassandra](database/cassandra) ([todo #164](https://github.com/mattes/migrate/issues/164)) - * [SQLite](database/sqlite) ([todo #165](https://github.com/mattes/migrate/issues/165)) + * [SQLite](database/sqlite3) * [MySQL/ MariaDB](database/mysql) * [Neo4j](database/neo4j) ([todo #167](https://github.com/mattes/migrate/issues/167)) * [MongoDB](database/mongodb) ([todo #169](https://github.com/mattes/migrate/issues/169)) diff --git a/database/sqlite/README.md b/database/sqlite3/README.md similarity index 100% rename from database/sqlite/README.md rename to database/sqlite3/README.md diff --git a/database/sqlite3/migration/33_create_table.down.sql b/database/sqlite3/migration/33_create_table.down.sql new file mode 100644 index 00000000..72d18c55 --- /dev/null +++ b/database/sqlite3/migration/33_create_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/migration/33_create_table.up.sql b/database/sqlite3/migration/33_create_table.up.sql new file mode 100644 index 00000000..5ad3404d --- /dev/null +++ b/database/sqlite3/migration/33_create_table.up.sql @@ -0,0 +1,3 @@ +CREATE TABLE pets ( + name string +); \ No newline at end of file diff --git a/database/sqlite3/migration/44_alter_table.down.sql b/database/sqlite3/migration/44_alter_table.down.sql new file mode 100644 index 00000000..72d18c55 --- /dev/null +++ b/database/sqlite3/migration/44_alter_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS pets; \ No newline at end of file diff --git a/database/sqlite3/migration/44_alter_table.up.sql b/database/sqlite3/migration/44_alter_table.up.sql new file mode 100644 index 00000000..f0682fcc --- /dev/null +++ b/database/sqlite3/migration/44_alter_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE pets ADD predator bool; diff --git a/database/sqlite3/sqlite3.go b/database/sqlite3/sqlite3.go new file mode 100644 index 00000000..bfd1a5b8 --- /dev/null +++ b/database/sqlite3/sqlite3.go @@ -0,0 +1,214 @@ +package sqlite3 + +import ( + "database/sql" + "fmt" + "github.com/mattes/migrate" + "github.com/mattes/migrate/database" + _ "github.com/mattn/go-sqlite3" + "io" + "io/ioutil" + nurl "net/url" + "strings" +) + +func init() { + database.Register("sqlite3", &Sqlite{}) +} + +var DefaultMigrationsTable = "schema_migrations" +var ( + ErrDatabaseDirty = fmt.Errorf("database is dirty") + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + DatabaseName string +} + +type Sqlite struct { + db *sql.DB + isLocked bool + + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + if len(config.MigrationsTable) == 0 { + config.MigrationsTable = DefaultMigrationsTable + } + + mx := &Sqlite{ + db: instance, + config: config, + } + if err := mx.ensureVersionTable(); err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) ensureVersionTable() error { + + query := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s (version uint64,dirty bool); + CREATE UNIQUE INDEX IF NOT EXISTS version_unique ON %s (version); + `, DefaultMigrationsTable, DefaultMigrationsTable) + + if _, err := m.db.Exec(query); err != nil { + return err + } + return nil +} + +func (m *Sqlite) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "sqlite3://", "", 1) + db, err := sql.Open("sqlite3", dbfile) + if err != nil { + return nil, err + } + + migrationsTable := purl.Query().Get("x-migrations-table") + if len(migrationsTable) == 0 { + migrationsTable = DefaultMigrationsTable + } + mx, err := WithInstance(db, &Config{ + DatabaseName: purl.Path, + MigrationsTable: migrationsTable, + }) + if err != nil { + return nil, err + } + return mx, nil +} + +func (m *Sqlite) Close() error { + return m.db.Close() +} + +func (m *Sqlite) Drop() error { + query := `SELECT name FROM sqlite_master WHERE type = 'table';` + tables, err := m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer tables.Close() + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + if len(tableNames) > 0 { + for _, t := range tableNames { + query := "DROP TABLE " + t + err = m.executeQuery(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + if err := m.ensureVersionTable(); err != nil { + return err + } + query := "VACUUM" + _, err = m.db.Query(query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + return nil +} + +func (m *Sqlite) Lock() error { + if m.isLocked { + return database.ErrLocked + } + m.isLocked = true + return nil +} + +func (m *Sqlite) Unlock() error { + if !m.isLocked { + return nil + } + m.isLocked = false + return nil +} + +func (m *Sqlite) Run(migration io.Reader) error { + migr, err := ioutil.ReadAll(migration) + if err != nil { + return err + } + query := string(migr[:]) + + return m.executeQuery(query) +} + +func (m *Sqlite) executeQuery(query string) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +func (m *Sqlite) SetVersion(version int, dirty bool) error { + tx, err := m.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + m.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (%d, '%t')`, m.config.MigrationsTable, version, dirty) + if _, err := tx.Exec(query); err != nil { + tx.Rollback() + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *Sqlite) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + m.config.MigrationsTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} diff --git a/database/sqlite3/sqlite3_test.go b/database/sqlite3/sqlite3_test.go new file mode 100644 index 00000000..6a5c5c86 --- /dev/null +++ b/database/sqlite3/sqlite3_test.go @@ -0,0 +1,61 @@ +package sqlite3 + +import ( + "database/sql" + "fmt" + "github.com/mattes/migrate" + dt "github.com/mattes/migrate/database/testing" + _ "github.com/mattes/migrate/source/file" + _ "github.com/mattn/go-sqlite3" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func Test(t *testing.T) { + dir, err := ioutil.TempDir("", "sqlite3-driver-test") + if err != nil { + return + } + defer func() { + os.RemoveAll(dir) + }() + fmt.Printf("DB path : %s\n", filepath.Join(dir, "sqlite3.db")) + p := &Sqlite{} + addr := fmt.Sprintf("sqlite3://%s", filepath.Join(dir, "sqlite3.db")) + d, err := p.Open(addr) + if err != nil { + t.Fatalf("%v", err) + } + + db, err := sql.Open("sqlite3", filepath.Join(dir, "sqlite3.db")) + if err != nil { + return + } + defer func() { + if err := db.Close(); err != nil { + return + } + }() + dt.Test(t, d, []byte("CREATE TABLE t (Qty int, Name string);")) + driver, err := WithInstance(db, &Config{}) + if err != nil { + t.Fatalf("%v", err) + } + if err := d.Drop(); err != nil { + t.Fatal(err) + } + + m, err := migrate.NewWithDatabaseInstance( + "file://./migration", + "ql", driver) + if err != nil { + t.Fatalf("%v", err) + } + fmt.Println("UP") + err = m.Up() + if err != nil { + t.Fatalf("%v", err) + } +} From decee4abb28da39c047b9ceb1af7e22437b9fea9 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 30 May 2017 14:06:59 +0200 Subject: [PATCH 2/2] added file cli/build_sqlite3.go --- cli/build_sqlite3.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cli/build_sqlite3.go diff --git a/cli/build_sqlite3.go b/cli/build_sqlite3.go new file mode 100644 index 00000000..48ae8ebc --- /dev/null +++ b/cli/build_sqlite3.go @@ -0,0 +1,7 @@ +// +build sqlite3 + +package main + +import ( + _ "github.com/mattes/migrate/database/sqlite3" +)