feat: ClickHouse driver

.gitignore

@@ -27,3 +27,4 @@ _testmain.go
 .env
 /testfixtures
 /dist
+.idea

Taskfile.yml

@@ -17,6 +17,12 @@ tasks:
       - task: test-db
         vars: {DATABASE: postgresql}
 
+  test:clickhouse:
+    desc: Test ClickHouse
+    cmds:
+      - task: test-db
+        vars: { DATABASE: clickhousedb }
+
   test:mysql:
     desc: Test MySQL
     cmds:
@@ -62,4 +68,4 @@ tasks:
   docker:test:
     cmds:
       - docker-compose down -v
-      - docker-compose run testfixtures go test -v -tags 'postgresql sqlite mysql sqlserver cockroachdb'
+      - docker-compose run testfixtures go test -v -tags 'postgresql sqlite mysql sqlserver cockroachdb clickhousedb'

clickhouse.go

@@ -0,0 +1,140 @@
+package testfixtures
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/ClickHouse/clickhouse-go"
+)
+
+type clickhouse struct {
+	baseHelper
+	tables         []string
+	tablesChecksum map[string]int64
+}
+
+func (h *clickhouse) init(db *sql.DB) error {
+	var err error
+	h.tables, err = h.tableNames(db)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *clickhouse) cleanTable(tx *sql.Tx, tableName string) error {
+	if _, err := tx.Exec(fmt.Sprintf("TRUNCATE TABLE %s", tableName)); err != nil {
+		return fmt.Errorf(`testfixtures: could not clean table "%s": %w`, tableName, err)
+	}
+
+	return nil
+}
+
+func (*clickhouse) paramType() int {
+	return paramTypeQuestion
+}
+
+func (*clickhouse) quoteKeyword(str string) string {
+	return fmt.Sprintf("`%s`", str)
+}
+
+func (*clickhouse) databaseName(q queryable) (string, error) {
+	var dbName string
+	err := q.QueryRow("SELECT DATABASE()").Scan(&dbName)
+	return dbName, err
+}
+
+func (h *clickhouse) tableNames(q queryable) ([]string, error) {
+	query := `
+		SELECT name
+		FROM system.tables
+		WHERE database = ?;
+	`
+	dbName, err := h.databaseName(q)
+	if err != nil {
+		return nil, err
+	}
+
+	rows, err := q.Query(query, dbName)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var tables []string
+	for rows.Next() {
+		var table string
+		if err = rows.Scan(&table); err != nil {
+			return nil, err
+		}
+		tables = append(tables, table)
+	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
+	return tables, nil
+
+}
+
+func (h *clickhouse) disableReferentialIntegrity(db *sql.DB, loadFn loadFunction) (err error) {
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	err = loadFn(tx)
+	if err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (h *clickhouse) isTableModified(q queryable, tableName string) (bool, error) {
+	checksum, err := h.getChecksum(q, tableName)
+	if err != nil {
+		return true, err
+	}
+
+	oldChecksum := h.tablesChecksum[tableName]
+
+	return oldChecksum == 0 || checksum != oldChecksum, nil
+}
+
+func (h *clickhouse) afterLoad(q queryable) error {
+	if h.tablesChecksum != nil {
+		return nil
+	}
+
+	h.tablesChecksum = make(map[string]int64, len(h.tables))
+	for _, t := range h.tables {
+		checksum, err := h.getChecksum(q, t)
+		if err != nil {
+			return err
+		}
+		h.tablesChecksum[t] = checksum
+	}
+	return nil
+}
+
+func (h *clickhouse) getChecksum(q queryable, tableName string) (int64, error) {
+	// This is an equivalent query to get the checksum of the content of the table
+	// We divide by 2 because it returns an uint64 instead of an int64
+	query := fmt.Sprintf("SELECT toInt64(groupBitXor(cityHash64(*)) / 2) FROM %s", h.quoteKeyword(tableName))
+	var (
+		checksum sql.NullInt64
+	)
+
+	if err := q.QueryRow(query).Scan(&checksum); err != nil {
+		return 0, err
+	}
+
+	return checksum.Int64, nil
+}
+
+// splitter is a batchSplitter interface implementation. We need it for
+// ClickHouseDB because clickhouse doesn't support multi-statements.
+func (*clickhouse) splitter() []byte {
+	return []byte(";\n")
+}

clickhouse_test.go

@@ -0,0 +1,19 @@
+// +build mysql
+
+package testfixtures
+
+import (
+	"os"
+	"testing"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+func TestClickHouse(t *testing.T) {
+	testLoader(
+		t,
+		"clickhouse",
+		os.Getenv("CLICKHOUSE_CONN_STRING"),
+		"testdata/schema/clickhouse.sql",
+	)
+}

docker-compose.yml

@@ -8,12 +8,15 @@ services:
       - mysql
       - sqlserver
       - cockroachdb
+      - clickhousedb
     environment:
       PGPASSWORD: postgres
       PG_CONN_STRING: host=postgresql user=postgres dbname=testfixtures_test port=5432 sslmode=disable
 
       MYSQL_CONN_STRING: root:mysql@tcp(mysql)/testfixtures_test?multiStatements=true
 
+      CLICKHOUSE_CONN_STRING: tcp://clickhousedb:9000?debug=0&allowMultiQueries=true&parseTime=true&username=default&password=&database=testfixtures_test
+
       SQLITE_CONN_STRING: testfixtures_test.sqlite3
 
       SQLSERVER_CONN_STRING: server=sqlserver;database=master;user id=sa;password=SQL@1server;encrypt=disable
@@ -33,6 +36,11 @@ services:
       MYSQL_DATABASE: testfixtures_test
       MYSQL_ROOT_PASSWORD: mysql
 
+  clickhousedb:
+    image: yandex/clickhouse-server:20.9.2.20
+    environment:
+      CLICKHOUSE_DB: testfixtures_test
+
   sqlserver:
     image: mcr.microsoft.com/mssql/server:2019-latest
     environment:

go.mod

@@ -1,6 +1,7 @@
 module github.com/go-testfixtures/testfixtures/v3
 
 require (
+	github.com/ClickHouse/clickhouse-go v1.4.3
 	github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/jackc/pgx/v4 v4.6.0

go.sum

@@ -1,5 +1,11 @@
+github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc=
+github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
 github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -10,6 +16,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73 h1:OGNva6WhsKst5OZf7eZOklDztV3hwtTHovdrLHV+MsA=
 github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -59,6 +66,7 @@ github.com/jackc/pgx/v4 v4.6.0/go.mod h1:vPh43ZzxijXUVJ+t/EmXBtFmbFVO72cuneCT9oA
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
 github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -80,8 +88,11 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA=
 github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

helper.go

@@ -19,6 +19,7 @@ type helper interface {
 	paramType() int
 	databaseName(queryable) (string, error)
 	tableNames(queryable) ([]string, error)
+	cleanTable(*sql.Tx, string) error
 	isTableModified(queryable, string) (bool, error)
 	afterLoad(queryable) error
 	quoteKeyword(string) string
@@ -43,6 +44,7 @@ type batchSplitter interface {
 
 var (
 	_ helper = &mySQL{}
+	_ helper = &clickhouse{}
 	_ helper = &postgreSQL{}
 	_ helper = &sqlite{}
 	_ helper = &sqlserver{}

mocks_for_test.go

@@ -36,6 +36,10 @@ func (h *MockHelper) databaseName(queryable) (string, error) {
 	return h.dbName, nil
 }
 
+func (h *MockHelper) cleanTable(*sql.Tx, string) error {
+	return nil
+}
+
 // NewMockHelper returns MockHelper
 func NewMockHelper(dbName string) *MockHelper {
 	return &MockHelper{dbName: dbName}

mysql.go

@@ -11,6 +11,14 @@ type mySQL struct {
 	tablesChecksum map[string]int64
 }
 
+func (h *mySQL) cleanTable(tx *sql.Tx, tableName string) error {
+	if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", tableName)); err != nil {
+		return fmt.Errorf(`testfixtures: could not clean table "%s": %w`, tableName, err)
+	}
+
+	return nil
+}
+
 func (h *mySQL) init(db *sql.DB) error {
 	var err error
 	h.tables, err = h.tableNames(db)

postgresql.go

@@ -27,6 +27,14 @@ type pgConstraint struct {
 	definition     string
 }
 
+func (h *postgreSQL) cleanTable(tx *sql.Tx, tableName string) error {
+	if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", tableName)); err != nil {
+		return fmt.Errorf(`testfixtures: could not clean table "%s": %w`, tableName, err)
+	}
+
+	return nil
+}
+
 func (h *postgreSQL) init(db *sql.DB) error {
 	var err error
 

sqlite.go

@@ -2,6 +2,7 @@ package testfixtures
 
 import (
 	"database/sql"
+	"fmt"
 	"path/filepath"
 )
 
@@ -24,6 +25,14 @@ func (*sqlite) databaseName(q queryable) (string, error) {
 	return dbName, nil
 }
 
+func (h *sqlite) cleanTable(tx *sql.Tx, tableName string) error {
+	if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", tableName)); err != nil {
+		return fmt.Errorf(`testfixtures: could not clean table "%s": %w`, tableName, err)
+	}
+
+	return nil
+}
+
 func (*sqlite) tableNames(q queryable) ([]string, error) {
 	query := `
 		SELECT name

sqlserver.go

@@ -13,6 +13,14 @@ type sqlserver struct {
 	tables         []string
 }
 
+func (h *sqlserver) cleanTable(tx *sql.Tx, tableName string) error {
+	if _, err := tx.Exec(fmt.Sprintf("DELETE FROM %s", tableName)); err != nil {
+		return fmt.Errorf(`testfixtures: could not clean table "%s": %w`, tableName, err)
+	}
+
+	return nil
+}
+
 func (h *sqlserver) init(db *sql.DB) error {
 	var err error