diff --git a/CHANGELOG.md b/CHANGELOG.md index 51948cdcd..489c7ca6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * The Custom::ECSService custom resource now waits for newly created ECS services to stabilize [#878](https://github.com/remind101/empire/pull/878) * The CloudFormation backend now uses the Custom::ECSService resource instead of AWS::ECS::Service, by default [#877](https://github.com/remind101/empire/pull/877) +* The database schema version is now checked at boot, as well as in the http health checks. [#893](https://github.com/remind101/empire/pull/893) **Bugs** diff --git a/cmd/empire/server.go b/cmd/empire/server.go index 254d343d9..1dbfb27f1 100644 --- a/cmd/empire/server.go +++ b/cmd/empire/server.go @@ -37,6 +37,18 @@ func runServer(c *cli.Context) { log.Fatal(err) } + // Do a preliminary health check to make sure everything is good at + // boot. + if err := e.IsHealthy(); err != nil { + if err, ok := err.(*empire.IncompatibleSchemaError); ok { + log.Fatal(fmt.Errorf("%v. You can resolve this error by running the migrations with `empire migrate` or with the `--automigrate` flag", err)) + } + + log.Fatal(err) + } else { + log.Println("Health checks passed") + } + if c.String(FlagCustomResourcesQueue) != "" { p, err := newCloudFormationCustomResourceProvisioner(db, c) if err != nil { diff --git a/db.go b/db.go index 8a2383831..b0c82e988 100644 --- a/db.go +++ b/db.go @@ -10,6 +10,18 @@ import ( "github.com/remind101/migrate" ) +// IncompatibleSchemaError is an error that gets returned from +// CheckSchemaVersion. +type IncompatibleSchemaError struct { + SchemaVersion int + ExpectedSchemaVersion int +} + +// Error implements the error interface. +func (e *IncompatibleSchemaError) Error() string { + return fmt.Sprintf("expected database schema to be at version %d, but was %d", e.ExpectedSchemaVersion, e.SchemaVersion) +} + // DB wraps a gorm.DB and provides the datastore layer for Empire. type DB struct { *gorm.DB @@ -74,8 +86,43 @@ func (db *DB) Reset() error { } // IsHealthy checks that we can connect to the database. -func (db *DB) IsHealthy() bool { - return db.DB.DB().Ping() == nil +func (db *DB) IsHealthy() error { + if err := db.DB.DB().Ping(); err != nil { + return err + } + + if err := db.CheckSchemaVersion(); err != nil { + return err + } + + return nil +} + +// CheckSchemaVersion verifies that the actual database schema matches the +// version that this version of Empire expects. +func (db *DB) CheckSchemaVersion() error { + schemaVersion, err := db.SchemaVersion() + if err != nil { + return fmt.Errorf("error fetching schema version: %v", err) + } + + expectedSchemaVersion := latestSchema() + if schemaVersion != expectedSchemaVersion { + return &IncompatibleSchemaError{ + SchemaVersion: schemaVersion, + ExpectedSchemaVersion: expectedSchemaVersion, + } + } + + return nil +} + +// SchemaVersion returns the current schema version. +func (db *DB) SchemaVersion() (int, error) { + sql := `select version from schema_migrations order by version desc limit 1` + var schemaVersion int + err := db.DB.DB().QueryRow(sql).Scan(&schemaVersion) + return schemaVersion, err } // Debug puts the db in debug mode, which logs all queries. diff --git a/empire.go b/empire.go index 0c04036b4..85dd92de6 100644 --- a/empire.go +++ b/empire.go @@ -689,7 +689,7 @@ func (e *Empire) Reset() error { // IsHealthy returns true if Empire is healthy, which means it can connect to // the services it depends on. -func (e *Empire) IsHealthy() bool { +func (e *Empire) IsHealthy() error { return e.DB.IsHealthy() } diff --git a/migrations.go b/migrations.go index ee4a5d2b4..ab85654ff 100644 --- a/migrations.go +++ b/migrations.go @@ -546,3 +546,9 @@ ALTER TABLE apps ADD COLUMN exposure TEXT NOT NULL default 'private'`, }), }, } + +// latestSchema returns the schema version that this version of Empire should be +// using. +func latestSchema() int { + return Migrations[len(Migrations)-1].ID +} diff --git a/migrations_test.go b/migrations_test.go index 2909b2501..d2455fca1 100644 --- a/migrations_test.go +++ b/migrations_test.go @@ -27,3 +27,7 @@ func TestMigrations(t *testing.T) { err = db.migrator.Exec(migrate.Up, Migrations...) assert.NoError(t, err) } + +func TestLatestSchema(t *testing.T) { + assert.Equal(t, 17, latestSchema()) +} diff --git a/server/server.go b/server/server.go index cb701c8c4..50201da39 100644 --- a/server/server.go +++ b/server/server.go @@ -1,6 +1,7 @@ package server import ( + "io" "net/http" "github.com/remind101/empire" @@ -64,7 +65,7 @@ func githubWebhook(r *http.Request) bool { // HealthHandler is an http.Handler that returns the health of empire. type HealthHandler struct { // A function that returns true if empire is healthy. - IsHealthy func() bool + IsHealthy func() error } // NewHealthHandler returns a new HealthHandler using the IsHealthy method from @@ -76,13 +77,14 @@ func NewHealthHandler(e *empire.Empire) *HealthHandler { } func (h *HealthHandler) ServeHTTPContext(_ context.Context, w http.ResponseWriter, r *http.Request) error { - var status = http.StatusOK - - if !h.IsHealthy() { - status = http.StatusServiceUnavailable + err := h.IsHealthy() + if err == nil { + w.WriteHeader(http.StatusOK) + return nil } - w.WriteHeader(status) + w.WriteHeader(http.StatusServiceUnavailable) + io.WriteString(w, err.Error()) return nil }