Add schema version check to db health check.

CHANGELOG.md

@@ -8,6 +8,7 @@
 
 * The Custom::ECSService custom resource now waits for newly created ECS services to stabilize [#878](https://github.com/remind101/empire/pull/878)
 * The CloudFormation backend now uses the Custom::ECSService resource instead of AWS::ECS::Service, by default [#877](https://github.com/remind101/empire/pull/877)
+* The database schema version is now checked at boot, as well as in the http health checks. [#893](https://github.com/remind101/empire/pull/893)
 
 **Bugs**
 

cmd/empire/server.go

@@ -37,6 +37,18 @@ func runServer(c *cli.Context) {
 		log.Fatal(err)
 	}
 
+	// Do a preliminary health check to make sure everything is good at
+	// boot.
+	if err := e.IsHealthy(); err != nil {
+		if err, ok := err.(*empire.IncompatibleSchemaError); ok {
+			log.Fatal(fmt.Errorf("%v. You can resolve this error by running the migrations with `empire migrate` or with the `--automigrate` flag", err))
+		}
+
+		log.Fatal(err)
+	} else {
+		log.Println("Health checks passed")
+	}
+
 	if c.String(FlagCustomResourcesQueue) != "" {
 		p, err := newCloudFormationCustomResourceProvisioner(db, c)
 		if err != nil {

db.go

@@ -10,6 +10,18 @@ import (
 	"github.com/remind101/migrate"
 )
 
+// IncompatibleSchemaError is an error that gets returned from
+// CheckSchemaVersion.
+type IncompatibleSchemaError struct {
+	SchemaVersion         int
+	ExpectedSchemaVersion int
+}
+
+// Error implements the error interface.
+func (e *IncompatibleSchemaError) Error() string {
+	return fmt.Sprintf("expected database schema to be at version %d, but was %d", e.ExpectedSchemaVersion, e.SchemaVersion)
+}
+
 // DB wraps a gorm.DB and provides the datastore layer for Empire.
 type DB struct {
 	*gorm.DB
@@ -74,8 +86,43 @@ func (db *DB) Reset() error {
 }
 
 // IsHealthy checks that we can connect to the database.
-func (db *DB) IsHealthy() bool {
-	return db.DB.DB().Ping() == nil
+func (db *DB) IsHealthy() error {
+	if err := db.DB.DB().Ping(); err != nil {
+		return err
+	}
+
+	if err := db.CheckSchemaVersion(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CheckSchemaVersion verifies that the actual database schema matches the
+// version that this version of Empire expects.
+func (db *DB) CheckSchemaVersion() error {
+	schemaVersion, err := db.SchemaVersion()
+	if err != nil {
+		return fmt.Errorf("error fetching schema version: %v", err)
+	}
+
+	expectedSchemaVersion := latestSchema()
+	if schemaVersion != expectedSchemaVersion {
+		return &IncompatibleSchemaError{
+			SchemaVersion:         schemaVersion,
+			ExpectedSchemaVersion: expectedSchemaVersion,
+		}
+	}
+
+	return nil
+}
+
+// SchemaVersion returns the current schema version.
+func (db *DB) SchemaVersion() (int, error) {
+	sql := `select version from schema_migrations order by version desc limit 1`
+	var schemaVersion int
+	err := db.DB.DB().QueryRow(sql).Scan(&schemaVersion)
+	return schemaVersion, err
 }
 
 // Debug puts the db in debug mode, which logs all queries.

empire.go

@@ -689,7 +689,7 @@ func (e *Empire) Reset() error {
 
 // IsHealthy returns true if Empire is healthy, which means it can connect to
 // the services it depends on.
-func (e *Empire) IsHealthy() bool {
+func (e *Empire) IsHealthy() error {
 	return e.DB.IsHealthy()
 }
 

migrations.go

@@ -546,3 +546,9 @@ ALTER TABLE apps ADD COLUMN exposure TEXT NOT NULL default 'private'`,
 		}),
 	},
 }
+
+// latestSchema returns the schema version that this version of Empire should be
+// using.
+func latestSchema() int {
+	return Migrations[len(Migrations)-1].ID
+}

migrations_test.go

@@ -27,3 +27,7 @@ func TestMigrations(t *testing.T) {
 	err = db.migrator.Exec(migrate.Up, Migrations...)
 	assert.NoError(t, err)
 }
+
+func TestLatestSchema(t *testing.T) {
+	assert.Equal(t, 17, latestSchema())
+}

server/server.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"io"
 	"net/http"
 
 	"github.com/remind101/empire"
@@ -64,7 +65,7 @@ func githubWebhook(r *http.Request) bool {
 // HealthHandler is an http.Handler that returns the health of empire.
 type HealthHandler struct {
 	// A function that returns true if empire is healthy.
-	IsHealthy func() bool
+	IsHealthy func() error
 }
 
 // NewHealthHandler returns a new HealthHandler using the IsHealthy method from
@@ -76,13 +77,14 @@ func NewHealthHandler(e *empire.Empire) *HealthHandler {
 }
 
 func (h *HealthHandler) ServeHTTPContext(_ context.Context, w http.ResponseWriter, r *http.Request) error {
-	var status = http.StatusOK
-
-	if !h.IsHealthy() {
-		status = http.StatusServiceUnavailable
+	err := h.IsHealthy()
+	if err == nil {
+		w.WriteHeader(http.StatusOK)
+		return nil
 	}
 
-	w.WriteHeader(status)
+	w.WriteHeader(http.StatusServiceUnavailable)
+	io.WriteString(w, err.Error())
 
 	return nil
 }