Skip to content

Commit

Permalink
add sqlite support (#55)
Browse files Browse the repository at this point in the history
* add sqlite support

* prepare sqlite database for ci

* do not fail if sqlite database does not exists when preparing

* fix file piping

* use relative path for sqlite database file
  • Loading branch information
KarnerTh authored Nov 21, 2023
1 parent 067a600 commit 5d76062
Show file tree
Hide file tree
Showing 13 changed files with 290 additions and 51 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ jobs:
uses: actions/checkout@v3
- name: Start container with test databases
run: docker-compose -f test/docker-compose.yaml up -d
- name: Prepare sqlite database
run: make prepare-sqlite
- name: Wait for docker containers to start
run: sleep 30
- name: Set up Go
Expand Down
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
GIT_TAG := $(shell git describe --tags --abbrev=0)
test_target := "./..."

.PHONY: prepare-sqlite
prepare-sqlite:
rm -f mermerd_test.db
cat test/db-table-setup.sql test/sqlite/sqlite-enum-setup.sql test/sqlite/sqlite-multiple-databases.sql | sqlite3 mermerd_test.db

.PHONY: test-coverage
test-coverage:
go test -cover -coverprofile=coverage.out ./...; go tool cover -html=coverage.out -o coverage.html; rm coverage.out
Expand Down
4 changes: 2 additions & 2 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,12 @@ connectionStringSuggestions:
assert.ElementsMatch(t,
config.RelationshipLabels(),
[]RelationshipLabel{
RelationshipLabel{
{
PkName: "schema.table1",
FkName: "schema.table2",
Label: "is_a",
},
RelationshipLabel{
{
PkName: "table-name",
FkName: "another-table-name",
Label: "has_many",
Expand Down
5 changes: 5 additions & 0 deletions database/connector_factory.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ func (connectorFactory) NewConnector(connectionString string) (Connector, error)
dbType: MsSql,
connectionString: connectionString,
}, nil
case strings.HasPrefix(connectionString, "sqlite3"):
return &sqliteConnector{
dbType: Sqlite3,
connectionString: connectionString,
}, nil
default:
return nil, errors.New("could not create connector for db")
}
Expand Down
1 change: 1 addition & 0 deletions database/connector_factory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ func TestNewConnector(t *testing.T) {
{connectionString: "postgres://user:password@localhost:5432/yourDb", expectedDbType: Postgres},
{connectionString: "mysql://root:password@tcp(127.0.0.1:3306)/yourDb", expectedDbType: MySql},
{connectionString: "sqlserver://sa:securePassword1!@localhost:1433?database=mermerd_test", expectedDbType: MsSql},
{connectionString: "sqlite3://mermerd_test.db", expectedDbType: Sqlite3},
}

for index, testCase := range testCases {
Expand Down
69 changes: 46 additions & 23 deletions database/database_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ var (
testConnectionMySql connectionParameter = connectionParameter{connectionString: "mysql://root:password@tcp(127.0.0.1:3306)/mermerd_test", schema: "mermerd_test"}
testConnectionMsSql connectionParameter = connectionParameter{connectionString: "sqlserver://sa:securePassword1!@localhost:1433?database=mermerd_test", schema: "dbo"}
testConnectionAzure connectionParameter = connectionParameter{connectionString: "sqlserver://sa:securePassword1!@localhost:1434?database=mermerd_test", schema: "dbo"}
testConnectionSqlite connectionParameter = connectionParameter{connectionString: "sqlite3://../mermerd_test.db", schema: "mermerd_test"}
)

func TestDatabaseIntegrations(t *testing.T) {
Expand Down Expand Up @@ -60,6 +61,11 @@ func TestDatabaseIntegrations(t *testing.T) {
connectionString: testConnectionAzure.connectionString,
schema: testConnectionAzure.schema,
},
{
dbType: Sqlite3,
connectionString: testConnectionSqlite.connectionString,
schema: testConnectionSqlite.schema,
},
}

for _, testCase := range testCases {
Expand Down Expand Up @@ -207,9 +213,11 @@ func TestDatabaseIntegrations(t *testing.T) {
// Assert
assert.Nil(t, err)
assert.Len(t, constraintResults, 1)
constraint := constraintResults[0]
assert.True(t, constraint.IsPrimary)
assert.False(t, constraint.HasMultiplePK)
if len(constraintResults) >= 1 {
constraint := constraintResults[0]
assert.True(t, constraint.IsPrimary)
assert.False(t, constraint.HasMultiplePK)
}
})

t.Run("Many-to-one relation #1", func(t *testing.T) {
Expand All @@ -222,9 +230,11 @@ func TestDatabaseIntegrations(t *testing.T) {
// Assert
assert.Nil(t, err)
assert.Len(t, constraintResults, 1)
constraint := constraintResults[0]
assert.False(t, constraint.IsPrimary)
assert.False(t, constraint.HasMultiplePK)
if len(constraintResults) >= 1 {
constraint := constraintResults[0]
assert.False(t, constraint.IsPrimary)
assert.False(t, constraint.HasMultiplePK)
}
})

t.Run("Many-to-one relation #2", func(t *testing.T) {
Expand All @@ -245,8 +255,10 @@ func TestDatabaseIntegrations(t *testing.T) {
}
}
assert.NotNil(t, constraint)
assert.True(t, constraint.IsPrimary)
assert.True(t, constraint.HasMultiplePK)
if constraint != nil {
assert.True(t, constraint.IsPrimary)
assert.True(t, constraint.HasMultiplePK)
}
})

// Multiple primary keys (https://github.com/KarnerTh/mermerd/issues/8)
Expand All @@ -261,16 +273,22 @@ func TestDatabaseIntegrations(t *testing.T) {
assert.Nil(t, err)
assert.NotNil(t, constraintResults)
assert.Len(t, constraintResults, 2)
assert.True(t, constraintResults[0].IsPrimary)
assert.True(t, constraintResults[0].HasMultiplePK)
assert.Equal(t, constraintResults[0].ColumnName, "aid")
assert.True(t, constraintResults[1].IsPrimary)
assert.True(t, constraintResults[1].HasMultiplePK)
assert.Equal(t, constraintResults[1].ColumnName, "bid")
if len(constraintResults) >= 2 {
assert.True(t, constraintResults[0].IsPrimary)
assert.True(t, constraintResults[0].HasMultiplePK)
assert.Equal(t, constraintResults[0].ColumnName, "aid")
assert.True(t, constraintResults[1].IsPrimary)
assert.True(t, constraintResults[1].HasMultiplePK)
assert.Equal(t, constraintResults[1].ColumnName, "bid")
}
})
})

t.Run("Multiple schemas (Issue #23)", func(t *testing.T) {
if testCase.dbType == Sqlite3 {
t.Skip("Sqlite does not support multiple schemas")
}

connector := getConnectionAndConnect(t)

t.Run("GetTables", func(t *testing.T) {
Expand Down Expand Up @@ -318,11 +336,14 @@ func TestDatabaseIntegrations(t *testing.T) {
// Assert
assert.Nil(t, err)
assert.Len(t, constraintResults, 1)
assert.False(t, constraintResults[0].IsPrimary)
assert.False(t, constraintResults[0].HasMultiplePK)
assert.Equal(t, constraintResults[0].ColumnName, "aid")
assert.Equal(t, constraintResults[0].FkTable, "test_3_b")
assert.Equal(t, constraintResults[0].PkTable, "test_3_a")

if len(constraintResults) >= 1 {
assert.False(t, constraintResults[0].IsPrimary)
assert.False(t, constraintResults[0].HasMultiplePK)
assert.Equal(t, constraintResults[0].ColumnName, "aid")
assert.Equal(t, constraintResults[0].FkTable, "test_3_b")
assert.Equal(t, constraintResults[0].PkTable, "test_3_a")
}
})

t.Run("Get schema from FK and PK table", func(t *testing.T) {
Expand All @@ -335,10 +356,12 @@ func TestDatabaseIntegrations(t *testing.T) {
// Assert
assert.Nil(t, err)
assert.Len(t, constraintResults, 1)
assert.Equal(t, constraintResults[0].FkTable, "test_3_b")
assert.Equal(t, constraintResults[0].FkSchema, "other_db")
assert.Equal(t, constraintResults[0].PkTable, "test_3_a")
assert.Equal(t, constraintResults[0].PkSchema, testCase.schema)
if len(constraintResults) >= 1 {
assert.Equal(t, constraintResults[0].FkTable, "test_3_b")
assert.Equal(t, constraintResults[0].FkSchema, "other_db")
assert.Equal(t, constraintResults[0].PkTable, "test_3_a")
assert.Equal(t, constraintResults[0].PkSchema, testCase.schema)
}
})
})
})
Expand Down
147 changes: 147 additions & 0 deletions database/sqlite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
package database

import (
"database/sql"
"fmt"
"strings"

_ "modernc.org/sqlite"
)

type sqliteConnector baseConnector

func (c *sqliteConnector) GetDbType() DbType {
return c.dbType
}

func getFilenameFromConnectionString(connectionString string) string {
return strings.Replace(connectionString, "sqlite3://", "", 1)
}

func (c *sqliteConnector) Connect() error {
db, err := sql.Open(c.dbType.String(), getFilenameFromConnectionString(c.connectionString))
if err != nil {
return err
}

if err := db.Ping(); err != nil {
return err
}

c.db = db
return nil
}

func (c *sqliteConnector) Close() {
err := c.db.Close()
if err != nil {
fmt.Println("could not close database connection", err)
}
}

func (c *sqliteConnector) GetSchemas() ([]string, error) {
fileName := getFilenameFromConnectionString(c.connectionString)
schema := strings.Replace(fileName, ".db", "", 1)
return []string{schema}, nil
}

func (c *sqliteConnector) GetTables(schemaNames []string) ([]TableDetail, error) {
rows, err := c.db.Query(`
select distinct tbl_name
from sqlite_schema
`)
if err != nil {
return nil, err
}

var tables []TableDetail
for rows.Next() {
table := TableDetail{Schema: schemaNames[0]}
if err = rows.Scan(&table.Name); err != nil {
return nil, err
}

table.Name = SanitizeValue(table.Name)
tables = append(tables, table)
}

return tables, nil
}

func (c *sqliteConnector) GetColumns(tableName TableDetail) ([]ColumnResult, error) {
rows, err := c.db.Query(`
select
t.name,
type,
pk > 0, -- first pk has 1, second has 2 ...
(case when fk.id is not null then 1 else 0 end) "isForeign",
(case when t."notnull" = true then false else true end) "nullable",
coalesce(
(select (case when i."unique" = true and i.origin = "u" then true else false end) "isUnique"
from pragma_index_list(:tableName) i
left join pragma_index_info(i.name) ii
where ii.name = t.name),
0
) "isUnique"
from pragma_table_info(:tableName) t
left join pragma_foreign_key_list(:tableName) fk on t.name = fk."from";
`, sql.Named("tableName", tableName.Name))
if err != nil {
return nil, err
}

var columns []ColumnResult
for rows.Next() {
var column ColumnResult
if err = rows.Scan(&column.Name, &column.DataType, &column.IsPrimary, &column.IsForeign, &column.IsNullable, &column.IsUnique); err != nil {
return nil, err
}

column.Name = SanitizeValue(column.Name)
column.DataType = SanitizeValue(column.DataType)

columns = append(columns, column)
}

return columns, nil
}

func (c *sqliteConnector) GetConstraints(tableName TableDetail) ([]ConstraintResult, error) {
rows, err := c.db.Query(`
select
pk."table" "pkTableName",
fk.name "pkTableName",
pk."from",
coalesce((select pk > 0 from pragma_table_info(fk.name) ti where ti.name = pk."from"), 0) "isPrimary", -- first pk has 1, second has 2 ...
coalesce((select count(*) > 1 from pragma_table_info(fk.name) ti where pk > 0), 0) "hasMultiplePK"
from sqlite_master fk
join pragma_foreign_key_list(fk.name) pk on pk."table" != fk.name
where fk.name = :tableName or pk."table" = :tableName
`, sql.Named("tableName", tableName.Name))
if err != nil {
return nil, err
}

var constraints []ConstraintResult
for rows.Next() {
constraint := ConstraintResult{
PkSchema: tableName.Schema,
FkSchema: tableName.Schema, // sqlite only has one schema
}
err = rows.Scan(
&constraint.PkTable,
&constraint.FkTable,
&constraint.ColumnName,
&constraint.IsPrimary,
&constraint.HasMultiplePK,
)

if err != nil {
return nil, err
}

constraints = append(constraints, constraint)
}

return constraints, nil
}
1 change: 1 addition & 0 deletions database/type.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ const (
Postgres DbType = "pgx"
MySql DbType = "mysql"
MsSql DbType = "sqlserver"
Sqlite3 DbType = "sqlite"
)

func (c DbType) String() string {
Expand Down
17 changes: 16 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,16 @@ require (
github.com/spf13/cobra v1.6.1
github.com/spf13/viper v1.15.0
github.com/stretchr/testify v1.8.1
modernc.org/sqlite v1.27.0
)

require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
Expand All @@ -37,16 +40,28 @@ require (
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
golang.org/x/crypto v0.7.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/tools v0.6.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.29.0 // indirect
modernc.org/mathutil v1.6.0 // indirect
modernc.org/memory v1.7.2 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
)
Loading

0 comments on commit 5d76062

Please sign in to comment.