Parallelize integration tests

.github/workflows/ci.yml

@@ -140,13 +140,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - database-backend: sqlite
-            database-uri: sqlite:///tmp/fasttrackml.db
-          - database-backend: sqlcipher
-            database-uri: sqlite:///tmp/fasttrackml.db?_key=passphrase
-          - database-backend: postgres
-            database-uri: postgres://postgres:postgres@postgres/postgres
+        database-backend: [sqlite, sqlcipher, postgres]
       fail-fast: false
     steps:
       - name: Checkout
@@ -166,7 +160,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: /tmp/go-cache/go-build.tar
-          key: go-integration-tests-image-${{ steps.cache-info.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }}
+          key: go-integration-tests-${{ matrix.database-backend }}-image-${{ steps.cache-info.outputs.go-version }}-mod-${{ hashFiles('.go-build-tags', 'go.sum') }}
 
       - name: Restore cache
         if: steps.cache.outputs.cache-hit == 'true'
@@ -176,8 +170,7 @@ jobs:
         run: make container-test
         env:
           DOCKER_BUILDKIT: 1
-          FML_DATABASE_URI: ${{ matrix.database-uri }}
-          FML_OUTPUT_DATABASE_URI: ${{ matrix.database-uri }}
+          FML_DATABASE_BACKEND: ${{ matrix.database-backend }}
 
       - name: Save cache
         if: steps.cache.outputs.cache-hit != 'true'

Makefile

@@ -141,7 +141,7 @@ test-go-unit: ## run go unit tests.
 .PHONY: test-go-integration
 test-go-integration: ## run go integration tests.
 	@echo ">>> Running integration tests."
-	@go test -p 1 -tags="$(GO_BUILDTAGS),integration" ./tests/integration/golang/...
+	@go test -tags="$(GO_BUILDTAGS),integration" ./tests/integration/golang/...
 
 .PHONY: test-python-integration
 test-python-integration: ## run all the python integration tests.

tests/integration/docker-compose.yml

@@ -48,7 +48,7 @@ services:
       GOCACHE: /cache/go-build
       GOMODCACHE: /cache/go-mod
       FML_LOG_LEVEL: ${FML_LOG_LEVEL:-fatal}
-      FML_DATABASE_URI: ${FML_DATABASE_URI:-postgres://postgres:postgres@postgres/postgres}
+      FML_POSTGRES_URI: postgres://postgres:postgres@postgres/postgres
       FML_S3_ENDPOINT_URI: http://minio:9000
       FML_GS_ENDPOINT_URI: http://google-storage:4443/storage/v1/
       AWS_ACCESS_KEY_ID: "user"

tests/integration/golang/database/import_test.go

@@ -46,8 +46,10 @@ func TestImportTestSuite(t *testing.T) {
 
 func (s *ImportTestSuite) SetupTest() {
 	// prepare input database.
+	dsn, err := helpers.GenerateDatabaseURI(s.T(), helpers.GetInputDatabaseBackend())
+	s.Require().Nil(err)
 	db, err := database.NewDBProvider(
-		helpers.GetInputDatabaseUri(),
+		dsn,
 		1*time.Second,
 		20,
 	)
@@ -63,8 +65,10 @@ func (s *ImportTestSuite) SetupTest() {
 	s.populateDB(s.inputDB)
 
 	// prepare output database.
+	dsn, err = helpers.GenerateDatabaseURI(s.T(), helpers.GetOutputDatabaseBackend())
+	s.Require().Nil(err)
 	db, err = database.NewDBProvider(
-		helpers.GetOutputDatabaseUri(),
+		dsn,
 		1*time.Second,
 		20,
 	)

tests/integration/golang/helpers/database.go

@@ -0,0 +1,57 @@
+package helpers
+
+import (
+	"database/sql"
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+
+	_ "github.com/jackc/pgx/v5/stdlib"
+	"github.com/rotisserie/eris"
+)
+
+func GenerateDatabaseURI(t *testing.T, backend string) (string, error) {
+	switch backend {
+	case "sqlite":
+		return fmt.Sprintf("sqlite://%s/test.db", t.TempDir()), nil
+	case "sqlcipher":
+		return fmt.Sprintf("sqlite://%s/test.db?_key=passphrase", t.TempDir()), nil
+	case "postgres":
+		return getPostgresDatabase(t, GetPostgresUri(),
+			strings.ToLower(
+				strings.ReplaceAll(t.TempDir(), "/", "_"),
+			),
+		)
+	default:
+		return "", fmt.Errorf("unknown backend: %s", backend)
+	}
+}
+
+func getPostgresDatabase(t *testing.T, dsn string, name string) (string, error) {
+	uri, err := url.Parse(dsn)
+	if err != nil {
+		return "", eris.Wrapf(err, "failed to parse dsn %q", dsn)
+	}
+
+	db, err := sql.Open("pgx", dsn)
+	if err != nil {
+		return "", eris.Wrapf(err, "failed to open database %q", dsn)
+	}
+
+	_, err = db.Exec("CREATE DATABASE " + name)
+	if err != nil {
+		return "", eris.Wrapf(err, "failed to create database %q on %q", name, dsn)
+	}
+
+	t.Cleanup(func() {
+		defer db.Close()
+		_, err := db.Exec("DROP DATABASE " + name + " WITH (FORCE)")
+		if err != nil {
+			t.Errorf("failed to drop database %q on %q: %v", name, dsn, err)
+		}
+	})
+
+	uri.Path = name
+	return uri.String(), nil
+}

tests/integration/golang/helpers/env.go

@@ -2,6 +2,8 @@ package helpers
 
 import "os"
 
+const defaultDatabaseBackend = "sqlite"
+
 func GetLogLevel() string {
 	level, ok := os.LookupEnv("FML_LOG_LEVEL")
 	if ok {
@@ -10,50 +12,50 @@ func GetLogLevel() string {
 	return "info"
 }
 
-func GetDatabaseUri() string {
-	uri, ok := os.LookupEnv("FML_DATABASE_URI")
+func GetDatabaseBackend() string {
+	uri, ok := os.LookupEnv("FML_DATABASE_BACKEND")
 	if ok {
 		return uri
 	}
-	return "sqlite:///tmp/fasttrackml.db"
+	return defaultDatabaseBackend
 }
 
-func GetServiceUri() string {
-	uri, ok := os.LookupEnv("FML_SERVICE_URI")
+func GetInputDatabaseBackend() string {
+	uri, ok := os.LookupEnv("FML_INPUT_DATABASE_BACKEND")
 	if ok {
 		return uri
 	}
-	return "http://localhost:5000"
+	return defaultDatabaseBackend
 }
 
-func GetS3EndpointUri() string {
-	uri, ok := os.LookupEnv("FML_S3_ENDPOINT_URI")
+func GetOutputDatabaseBackend() string {
+	uri, ok := os.LookupEnv("FML_OUTPUT_DATABASE_BACKEND")
 	if ok {
 		return uri
 	}
-	return "http://localhost:9000"
+	return defaultDatabaseBackend
 }
 
-func GetGSEndpointUri() string {
-	uri, ok := os.LookupEnv("FML_GS_ENDPOINT_URI")
+func GetPostgresUri() string {
+	uri, ok := os.LookupEnv("FML_POSTGRES_URI")
 	if ok {
 		return uri
 	}
-	return "http://localhost:4443/storage/v1/"
+	return "postgres://postgres:postgres@localhost/postgres"
 }
 
-func GetInputDatabaseUri() string {
-	uri, ok := os.LookupEnv("FML_INPUT_DATABASE_URI")
+func GetGSEndpointUri() string {
+	uri, ok := os.LookupEnv("FML_GS_ENDPOINT_URI")
 	if ok {
 		return uri
 	}
-	return "sqlite:///tmp/fasttrackml-in.db"
+	return "http://localhost:4443/storage/v1/"
 }
 
-func GetOutputDatabaseUri() string {
-	uri, ok := os.LookupEnv("FML_OUTPUT_DATABASE_URI")
+func GetS3EndpointUri() string {
+	uri, ok := os.LookupEnv("FML_S3_ENDPOINT_URI")
 	if ok {
 		return uri
 	}
-	return "sqlite:///tmp/fasttrackml-out.db"
+	return "http://localhost:9000"
 }

tests/integration/golang/helpers/test.go

@@ -61,9 +61,11 @@ func (s *BaseTestSuite) initLogger() {
 }
 
 func (s *BaseTestSuite) initDB() {
-	var err error
+	dsn, err := GenerateDatabaseURI(s.T(), GetDatabaseBackend())
+	s.Require().Nil(err)
+
 	s.db, err = database.NewDBProvider(
-		GetDatabaseUri(),
+		dsn,
 		1*time.Second,
 		20,
 	)

tests/integration/golang/mlflow/namespace/flows/artifact_test.go

@@ -33,7 +33,7 @@ type ArtifactFlowTestSuite struct {
 // - `GET /artifacts/list`
 func TestArtifactFlowTestSuite(t *testing.T) {
 	s := &ArtifactFlowTestSuite{
-		helpers.NewS3TestSuite("bucket1", "bucket2"),
+		helpers.NewS3TestSuite("flow-bucket1", "flow-bucket2"),
 	}
 	s.S3TestSuite.ResetOnSubTest = true
 	s.S3TestSuite.SkipCreateDefaultNamespace = true
@@ -102,15 +102,15 @@ func (s *ArtifactFlowTestSuite) Test_Ok() {
 
 			experiment1, err := s.ExperimentFixtures.CreateExperiment(context.Background(), &models.Experiment{
 				Name:             "Experiment1",
-				ArtifactLocation: "s3://bucket1/1",
+				ArtifactLocation: "s3://flow-bucket1/1",
 				LifecycleStage:   models.LifecycleStageActive,
 				NamespaceID:      namespace1.ID,
 			})
 			s.Require().Nil(err)
 
 			experiment2, err := s.ExperimentFixtures.CreateExperiment(context.Background(), &models.Experiment{
 				Name:             "Experiment2",
-				ArtifactLocation: "s3://bucket2/2",
+				ArtifactLocation: "s3://flow-bucket2/2",
 				LifecycleStage:   models.LifecycleStageActive,
 				NamespaceID:      namespace2.ID,
 			})
@@ -134,7 +134,7 @@ func (s *ArtifactFlowTestSuite) testRunArtifactFlow(
 	_, err := s.Client.PutObject(context.Background(), &s3.PutObjectInput{
 		Key:    aws.String(fmt.Sprintf("1/%s/artifacts/artifact1.file", run1ID)),
 		Body:   strings.NewReader("content1"),
-		Bucket: aws.String("bucket1"),
+		Bucket: aws.String("flow-bucket1"),
 	})
 	s.Require().Nil(err)
 
@@ -146,7 +146,7 @@ func (s *ArtifactFlowTestSuite) testRunArtifactFlow(
 	_, err = s.Client.PutObject(context.Background(), &s3.PutObjectInput{
 		Key:    aws.String(fmt.Sprintf("2/%s/artifacts/artifact2.file", run2ID)),
 		Body:   strings.NewReader("content2"),
-		Bucket: aws.String("bucket2"),
+		Bucket: aws.String("flow-bucket2"),
 	})
 	s.Require().Nil(err)