feat: add migrations for Postgres and SQLite3

* feat: add migrations for Postgres and SQLite3
* feat(test): set environment for migration tests
* feat(gh-action): update test job environment

.env.local

@@ -0,0 +1,4 @@
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=postgres
+SQLITE3_DB=test.db

.github/workflows/ci.yml

@@ -17,10 +17,18 @@ jobs:
   unit:
     name: unit
     runs-on: ubuntu-20.04
+    env:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: postgres
+      SQLITE3_DB: test.db
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
+      - name: Setup PostgreSQL
+        uses: ikalnytskyi/action-setup-postgres@v4
+        id: postgres
       - run: |
           go test -race -timeout 240s ./...

Makefile

@@ -0,0 +1,8 @@
+.PHONY: test
+
+include .env.local
+
+export $(shell sed 's/=.*//' .env.local)
+
+test:
+	go test -v ./...

docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3'
+
+services:
+  postgres:
+    container_name: assetdb_postgres
+    image: postgres:latest
+    restart: always
+    env_file: .env.local
+    ports:
+      - "5432:5432"
+
+volumes:
+  postgres-db:
+    driver: local

go.mod

@@ -1,3 +1,23 @@
 module github.com/owasp-amass/asset-db
 
 go 1.20
+
+require github.com/rubenv/sql-migrate v1.4.0
+
+require (
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
+	github.com/jackc/pgx/v5 v5.3.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/mattn/go-sqlite3 v1.14.16 // indirect
+	golang.org/x/crypto v0.8.0 // indirect
+	golang.org/x/text v0.9.0 // indirect
+	gorm.io/gorm v1.25.0 // indirect
+)
+
+require (
+	github.com/go-gorp/gorp/v3 v3.1.0 // indirect
+	gorm.io/driver/postgres v1.5.2
+	gorm.io/driver/sqlite v1.5.1
+)

migrations/postgres/20230217121325-init.sql

@@ -0,0 +1,27 @@
+-- +migrate Up
+
+CREATE TABLE IF NOT EXISTS assets(
+    id SERIAL PRIMARY KEY,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    type VARCHAR(255),
+    content JSONB);
+
+CREATE TABLE IF NOT EXISTS relations(
+    id SERIAL PRIMARY KEY,
+    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    type VARCHAR(255),
+    from_asset_id INT,
+    to_asset_id INT,
+    CONSTRAINT fk_from_asset
+        FOREIGN KEY (from_asset_id)
+        REFERENCES assets(id)
+        ON DELETE CASCADE,
+    CONSTRAINT fk_to_asset
+        FOREIGN KEY (to_asset_id)
+        REFERENCES assets(id)
+        ON DELETE CASCADE);
+
+-- +migrate Down
+
+DROP TABLE relations;
+DROP TABLE assets;

migrations/postgres/doc.go

@@ -0,0 +1,4 @@
+// Integrates with migration tools
+// recognizing the standard "migrate up" and "migrate down" annotations,
+// simplifying asset database schema management and rollbacks in Postgres.
+package postgres

migrations/postgres/embed.go

@@ -0,0 +1,13 @@
+package postgres
+
+import (
+	"embed"
+)
+
+//go:embed *.sql
+var postgresMigrations embed.FS
+
+// Migrations returns the migrations for the postgres database.
+func Migrations() embed.FS {
+	return postgresMigrations
+}

migrations/postgres/example_test.go

@@ -0,0 +1,45 @@
+package postgres_test
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/owasp-amass/asset-db/migrations/postgres"
+	migrate "github.com/rubenv/sql-migrate"
+	pg "gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func ExampleMigrations() {
+	user := os.Getenv("POSTGRES_USER")
+	password := os.Getenv("POSTGRES_PASSWORD")
+	dbname := os.Getenv("POSTGRES_DB")
+
+	dsn := fmt.Sprintf("postgresql://localhost/%s?user=%s&password=%s", dbname, user, password)
+
+	db, err := gorm.Open(pg.Open(dsn), &gorm.Config{})
+	if err != nil {
+		panic("failed to connect database")
+	}
+
+	sqlDb, _ := db.DB()
+
+	migrationsSource := migrate.EmbedFileSystemMigrationSource{
+		FileSystem: postgres.Migrations(),
+		Root:       "/",
+	}
+
+	_, err = migrate.Exec(sqlDb, "postgres", migrationsSource, migrate.Up)
+	if err != nil {
+		panic(err)
+	}
+
+	tables := []string{"assets", "relations"}
+	for _, table := range tables {
+		fmt.Println(db.Migrator().HasTable(table))
+	}
+
+	// Output:
+	// true
+	// true
+}

migrations/sqlite3/20230217121325-init.sql

@@ -0,0 +1,24 @@
+-- +migrate Up
+
+-- see https://www.sqlite.org/foreignkeys.html#fk_enable about enabling foreign keys
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS assets(
+    id INTEGER PRIMARY KEY,
+    created_at DATETIME,
+    type TEXT,
+    content TEXT);
+
+CREATE TABLE IF NOT EXISTS relations(
+    id INTEGER PRIMARY KEY,
+    created_at DATETIME,
+    type TEXT,
+    from_asset_id INTEGER,
+    to_asset_id INTEGER,
+    FOREIGN KEY(from_asset_id) REFERENCES assets(id) ON DELETE CASCADE,
+    FOREIGN KEY(to_asset_id) REFERENCES assets(id) ON DELETE CASCADE);
+
+-- +migrate Down
+
+DROP TABLE relations;
+DROP TABLE assets;

migrations/sqlite3/doc.go

@@ -0,0 +1,4 @@
+// Integrates with migration tools
+// recognizing the standard "migrate up" and "migrate down" annotations,
+// simplifying asset database schema management and rollbacks in SQLite3.
+package sqlite3

migrations/sqlite3/embed.go

@@ -0,0 +1,12 @@
+package sqlite3
+
+import (
+	"embed"
+)
+
+//go:embed *.sql
+var sqlite3Migrations embed.FS
+
+func Migrations() embed.FS {
+	return sqlite3Migrations
+}

migrations/sqlite3/example_test.go

@@ -0,0 +1,46 @@
+package sqlite3_test
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/owasp-amass/asset-db/migrations/sqlite3"
+	migrate "github.com/rubenv/sql-migrate"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+func ExampleMigrations() {
+	dns := os.Getenv("SQLITE3_DB")
+
+	db, err := gorm.Open(sqlite.Open(dns), &gorm.Config{})
+	if err != nil {
+		panic("failed to connect database")
+	}
+
+	sqlDb, _ := db.DB()
+
+	migrationsSource := migrate.EmbedFileSystemMigrationSource{
+		FileSystem: sqlite3.Migrations(),
+		Root:       "/",
+	}
+
+	_, err = migrate.Exec(sqlDb, "sqlite3", migrationsSource, migrate.Up)
+	if err != nil {
+		panic(err)
+	}
+
+	tables := []string{"assets", "relations"}
+	for _, table := range tables {
+		fmt.Println(db.Migrator().HasTable(table))
+	}
+
+	err = os.Remove("./" + dns)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// true
+	// true
+}