New migration to remove the old experiments_name_key for main branch

pkg/database/migrate.go

@@ -20,6 +20,7 @@ import (
 	"github.com/G-Research/fasttrackml/pkg/database/migrations/v_0005"
 	"github.com/G-Research/fasttrackml/pkg/database/migrations/v_0006"
 	"github.com/G-Research/fasttrackml/pkg/database/migrations/v_0007"
+	"github.com/G-Research/fasttrackml/pkg/database/migrations/v_0008"
 )
 
 var supportedAlembicVersions = []string{
@@ -41,7 +42,7 @@ func CheckAndMigrateDB(migrate bool, db *gorm.DB) error {
 		tx.First(&schemaVersion)
 	}
 
-	if !slices.Contains(supportedAlembicVersions, alembicVersion.Version) || schemaVersion.Version != v_0007.Version {
+	if !slices.Contains(supportedAlembicVersions, alembicVersion.Version) || schemaVersion.Version != v_0008.Version {
 		if !migrate && alembicVersion.Version != "" {
 			return fmt.Errorf(
 				"unsupported database schema versions alembic %s, FastTrackML %s",
@@ -172,6 +173,13 @@ func CheckAndMigrateDB(migrate bool, db *gorm.DB) error {
 				if err := v_0007.Migrate(db); err != nil {
 					return fmt.Errorf("error migrating database to FastTrackML schema %s: %w", v_0007.Version, err)
 				}
+				fallthrough
+
+			case v_0007.Version:
+				log.Infof("Migrating database to FastTrackML schema %s", v_0008.Version)
+				if err := v_0008.Migrate(db); err != nil {
+					return fmt.Errorf("error migrating database to FastTrackML schema %s: %w", v_0008.Version, err)
+				}
 
 			default:
 				return fmt.Errorf("unsupported database FastTrackML schema version %s", schemaVersion.Version)
@@ -202,7 +210,7 @@ func CheckAndMigrateDB(migrate bool, db *gorm.DB) error {
 				Version: "97727af70f4d",
 			})
 			tx.Create(&SchemaVersion{
-				Version: v_0007.Version,
+				Version: v_0008.Version,
 			})
 			tx.Commit()
 			if tx.Error != nil {

pkg/database/migrations/v_0007/migrate.go

@@ -1,25 +1,41 @@
 package v_0007
 
 import (
+	"fmt"
+
+	"gorm.io/driver/postgres"
+	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+
+	"github.com/G-Research/fasttrackml/pkg/database/migrations"
 )
 
-const Version = "2c2299e4e061"
+const Version = "cbc41c0f4fc5"
 
 func Migrate(db *gorm.DB) error {
-	return db.Transaction(func(tx *gorm.DB) error {
-		if err := tx.Migrator().AddColumn(&Metric{}, "ContextID"); err != nil {
-			return err
-		}
-		if err := tx.Migrator().AddColumn(&LatestMetric{}, "ContextID"); err != nil {
-			return err
-		}
-		if err := tx.Migrator().AutoMigrate(&Context{}); err != nil {
-			return err
-		}
-		return tx.Model(&SchemaVersion{}).
-			Where("1 = 1").
-			Update("Version", Version).
-			Error
+	// We need to run this migration without foreign key constraints to avoid
+	// the cascading delete to kick in and delete all the runs.
+	return migrations.RunWithoutForeignKeyIfNeeded(db, func() error {
+		return db.Transaction(func(tx *gorm.DB) error {
+			switch tx.Dialector.Name() {
+			case sqlite.Dialector{}.Name():
+				// SQLite no action needed
+			case postgres.Dialector{}.Name():
+				// Postgres needs to remove this constraint
+				constraint := "experiments_name_key"
+				if tx.Migrator().HasConstraint("experiments", constraint) {
+					if err := tx.Migrator().DropConstraint("experiments", constraint); err != nil {
+						return err
+					}
+				}
+			default:
+				return fmt.Errorf("unsupported database dialect %s", tx.Dialector.Name())
+			}
+
+			return tx.Model(&SchemaVersion{}).
+				Where("1 = 1").
+				Update("Version", Version).
+				Error
+		})
 	})
 }

pkg/database/migrations/v_0007/model.go

@@ -9,7 +9,6 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"gorm.io/datatypes"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
 )
@@ -32,25 +31,25 @@ const (
 )
 
 type Namespace struct {
-	ID                  uint   `gorm:"primaryKey;autoIncrement"`
-	Apps                []App  `gorm:"constraint:OnDelete:CASCADE"`
-	Code                string `gorm:"unique;index;not null"`
-	Description         string
-	CreatedAt           time.Time
-	UpdatedAt           time.Time
-	DeletedAt           gorm.DeletedAt `gorm:"index"`
-	DefaultExperimentID *int32         `gorm:"not null"`
-	Experiments         []Experiment   `gorm:"constraint:OnDelete:CASCADE"`
+	ID                  uint           `gorm:"primaryKey;autoIncrement" json:"id"`
+	Apps                []App          `gorm:"constraint:OnDelete:CASCADE" json:"apps"`
+	Code                string         `gorm:"unique;index;not null" json:"code"`
+	Description         string         `json:"description"`
+	CreatedAt           time.Time      `json:"created_at"`
+	UpdatedAt           time.Time      `json:"updated_at"`
+	DeletedAt           gorm.DeletedAt `gorm:"index" json:"deleted_at"`
+	DefaultExperimentID *int32         `gorm:"not null" json:"default_experiment_id"`
+	Experiments         []Experiment   `gorm:"constraint:OnDelete:CASCADE" json:"experiments"`
 }
 
 type Experiment struct {
 	ID               *int32         `gorm:"column:experiment_id;not null;primaryKey"`
-	Name             string         `gorm:"type:varchar(256);not null;index:idx_namespace_name,unique"`
+	Name             string         `gorm:"type:varchar(256);not null;index:,unique,composite:name"`
 	ArtifactLocation string         `gorm:"type:varchar(256)"`
 	LifecycleStage   LifecycleStage `gorm:"type:varchar(32);check:lifecycle_stage IN ('active', 'deleted')"`
 	CreationTime     sql.NullInt64  `gorm:"type:bigint"`
 	LastUpdateTime   sql.NullInt64  `gorm:"type:bigint"`
-	NamespaceID      uint           `gorm:"index:idx_namespace_name,unique"`
+	NamespaceID      uint           `gorm:"index:,unique,composite:name"`
 	Namespace        Namespace
 	Tags             []ExperimentTag `gorm:"constraint:OnDelete:CASCADE"`
 	Runs             []Run           `gorm:"constraint:OnDelete:CASCADE"`
@@ -133,8 +132,6 @@ type Metric struct {
 	Step      int64   `gorm:"default:0;not null;primaryKey"`
 	IsNan     bool    `gorm:"default:false;not null;primaryKey"`
 	Iter      int64   `gorm:"index"`
-	ContextID *uint
-	Context   *Context
 }
 
 type LatestMetric struct {
@@ -145,13 +142,6 @@ type LatestMetric struct {
 	IsNan     bool   `gorm:"not null"`
 	RunID     string `gorm:"column:run_uuid;not null;primaryKey;index"`
 	LastIter  int64
-	ContextID *uint
-	Context   *Context
-}
-
-type Context struct {
-	ID   uint           `gorm:"primaryKey;autoIncrement"`
-	Json datatypes.JSON `gorm:"not null;unique;index"`
 }
 
 type AlembicVersion struct {
@@ -209,10 +199,10 @@ func (d Dashboard) MarshalJSON() ([]byte, error) {
 
 type App struct {
 	Base
-	Type        string   `gorm:"not null" json:"type"`
-	State       AppState `json:"state"`
-	Namespace   Namespace
-	NamespaceID uint `gorm:"column:namespace_id"`
+	Type        string    `gorm:"not null" json:"type"`
+	State       AppState  `json:"state"`
+	Namespace   Namespace `json:"-"`
+	NamespaceID uint      `gorm:"column:namespace_id" json:"-"`
 }
 
 type AppState map[string]any

pkg/database/migrations/v_0008/migrate.go

@@ -0,0 +1,25 @@
+package v_0008
+
+import (
+	"gorm.io/gorm"
+)
+
+const Version = "2c2299e4e061"
+
+func Migrate(db *gorm.DB) error {
+	return db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Migrator().AddColumn(&Metric{}, "ContextID"); err != nil {
+			return err
+		}
+		if err := tx.Migrator().AddColumn(&LatestMetric{}, "ContextID"); err != nil {
+			return err
+		}
+		if err := tx.Migrator().AutoMigrate(&Context{}); err != nil {
+			return err
+		}
+		return tx.Model(&SchemaVersion{}).
+			Where("1 = 1").
+			Update("Version", Version).
+			Error
+	})
+}