Add consul version compatibility check on startup

agent.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/hashicorp/consul-esm/version"
 	"github.com/hashicorp/consul/api"
 	"github.com/hashicorp/go-uuid"
 )
@@ -428,3 +430,64 @@ func (a *Agent) updateLastKnownNodeStatus(node string, newStatus string) {
 	defer a.knownNodeStatusesLock.Unlock()
 	a.knownNodeStatuses[node] = lastKnownStatus{newStatus, time.Now()}
 }
+
+// VerifyConsulCompatibility queries Consul for local agent and all server versions to verify
+// compatibility with ESM.
+func (a *Agent) VerifyConsulCompatibility() error {
+	if a.client == nil {
+		return fmt.Errorf("unable to check version compatibility without Consul client initialized")
+	}
+
+	// Fetch local agent version
+	agentInfo, err := a.client.Agent().Self()
+	if err != nil {
+		// ESM blocks in NewAgent() until agent is available. At this point
+		// /agent/self endpoint should be available and an error would not be useful
+		// to retry the request.
+		return fmt.Errorf("unable to check version compatibility with Consul agent: %s", err)
+	}
+	agentVersionRaw, ok := agentInfo["Config"]["Version"]
+	if !ok {
+		return fmt.Errorf("unable to check version compatibility with Consul agent")
+	}
+	agentVersion := agentVersionRaw.(string)
+
+VERIFYCONSULSERVER:
+	select {
+	case <-a.shutdownCh:
+		return nil
+	default:
+	}
+
+	// Fetch server versions
+	resp, err := a.client.Operator().AutopilotServerHealth(nil)
+	if err != nil {
+		if strings.Contains(err.Error(), "429") {
+			// 429 is a warning that something is unhealthy. This may occur when ESM
+			// races with Consul servers first starting up, so this is safe to retry.
+			a.logger.Printf("[ERR] Failed to query for Consul server versions (will retry): %v", err)
+			time.Sleep(retryTime)
+			goto VERIFYCONSULSERVER
+		}
+
+		return fmt.Errorf("unable to check version compatibility with Consul servers: %s", err)
+	}
+
+	versions := []string{agentVersion}
+	uniqueVersions := map[string]bool{agentVersion: true}
+	for _, s := range resp.Servers {
+		if !uniqueVersions[s.Version] {
+			uniqueVersions[s.Version] = true
+			versions = append(versions, s.Version)
+		}
+	}
+
+	err = version.CheckConsulVersions(versions)
+	if err != nil {
+		a.logger.Printf("[ERR] Incompatible Consul versions")
+		return err
+	}
+
+	a.logger.Printf("[DEBUG] Consul agent and all servers are running compatible versions with ESM")
+	return nil
+}

agent_test.go

@@ -220,3 +220,25 @@ func TestAgent_LastKnownStatusIsExpired(t *testing.T) {
 		}
 	}
 }
+
+func TestAgent_VerifyConsulCompatibility(t *testing.T) {
+	// Smoke test to test the compatibility with the current Consul version
+	// pinned in go dependency.
+	t.Parallel()
+	s, err := NewTestServer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Stop()
+
+	agent := testAgent(t, func(c *Config) {
+		c.HTTPAddr = s.HTTPAddr
+		c.Tag = "test"
+	})
+	defer agent.Shutdown()
+
+	err = agent.VerifyConsulCompatibility()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+}

go.mod

@@ -14,8 +14,7 @@ require (
 	github.com/hashicorp/go-rootcerts v1.0.1 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/go-uuid v1.0.1
-	github.com/hashicorp/go-version v1.1.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl v1.0.0
 	github.com/hashicorp/serf v0.8.4
 	github.com/miekg/dns v1.1.22 // indirect

go.sum

@@ -165,9 +165,10 @@ github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v0.0.0-20170202080759-03c5bf6be031/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
-github.com/hashicorp/go-version v1.1.0 h1:bPIoEKD27tNdebFGGxxYwcL4nepeY4j1QP23PFRGzg0=
-github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

main.go

@@ -46,6 +46,8 @@ func main() {
 
 	if isVersion {
 		fmt.Printf("%s\n", version.GetHumanVersion())
+		fmt.Printf("Compatible with Consul versions %s\n",
+			version.GetConsulVersionConstraint())
 		os.Exit(ExitCodeOK)
 	}
 
@@ -75,6 +77,15 @@ func main() {
 		panic(err)
 	}
 
+	// Consul compatibility is only verified at startup. If new Consul servers
+	// join later with incompatible versions, inconsistent results may occur with
+	// updating health checks for external services.
+	err = agent.VerifyConsulCompatibility()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(ExitCodeError)
+	}
+
 	// Set up shutdown and signal handling.
 	signalCh := make(chan os.Signal, 10)
 	signal.Notify(signalCh)

version/version.go

@@ -2,9 +2,16 @@ package version
 
 import (
 	"fmt"
+	"log"
 	"strings"
+
+	"github.com/hashicorp/go-version"
 )
 
+// consulVersionConstraint is the compatible version constraint
+// between ESM with Consul.
+const consulVersionConstraint = ">= 1.4.1"
+
 var (
 	Name string
 
@@ -13,15 +20,26 @@ var (
 	GitCommit   string
 	GitDescribe string
 
-	// The main version number that is being run at the moment.
+	// Version is the main version number that is being run at the moment.
 	Version = "0.3.3"
 
-	// A pre-release marker for the version. If this is "" (empty string)
-	// then it means that it is a final release. Otherwise, this is a pre-release
-	// such as "dev" (in development), "beta", "rc1", etc.
+	// VersionPrerelease is a pre-release marker for the version. If this is ""
+	// (empty string) then it means that it is a final release. Otherwise, this
+	// is a pre-release such as "dev" (in development), "beta", "rc1", etc.
 	VersionPrerelease = ""
+
+	consulConstraints version.Constraints
 )
 
+func init() {
+	vc, err := version.NewConstraint(consulVersionConstraint)
+	if err != nil {
+		log.Fatalf("invalid Consul version constraint %q: %s",
+			consulVersionConstraint, err)
+	}
+	consulConstraints = vc
+}
+
 // GetHumanVersion composes the parts of the version in a way that's suitable
 // for displaying to humans.
 func GetHumanVersion() string {
@@ -44,3 +62,55 @@ func GetHumanVersion() string {
 	// Strip off any single quotes added by the git information.
 	return strings.Replace(version, "'", "", -1)
 }
+
+// GetConsulVersionConstraint returns the version constraint for Consul
+// that ESM is compatible with.
+func GetConsulVersionConstraint() string {
+	return consulVersionConstraint
+}
+
+// CheckConsulVersions checks for the compatibility of the Consul versions.
+// Valid SemVer is expected, and will consider any invalid versions to be
+// incompatible.
+func CheckConsulVersions(versions []string) error {
+	if len(versions) == 0 {
+		return fmt.Errorf("no Consul versions")
+	}
+
+	var unsupported []string
+	for _, s := range versions {
+		// Any prelease information is trimmed to simplify constraint comparision
+		// that will automatically return false for versions with prereleases.
+		// This is a spec for go-version constraints checking.
+		trimmed := strings.SplitN(s, "-", 2)[0]
+		v, err := version.NewSemver(trimmed)
+		if err != nil {
+			unsupported = append(unsupported, s)
+		}
+
+		if ok := consulConstraints.Check(v); !ok {
+			unsupported = append(unsupported, s)
+		}
+	}
+
+	if len(unsupported) != 0 {
+		return NewConsulVersionError(unsupported)
+	}
+
+	return nil
+}
+
+// NewConsulVersionError returns an error detailing the version compatibility
+// issue between ESM and Consul servers.
+func NewConsulVersionError(consulVersions []string) error {
+	versions := strings.Join(consulVersions, ", ")
+	return fmt.Errorf(versionErrorTmpl, GetHumanVersion(), versions)
+}
+
+const versionErrorTmpl = `
+Consul ESM version %s is incompatible with the running versions of the local
+Consul agent or Consul servers (%s), please refer to the documentation
+to safely upgrade Consul or change to a version of ESM that is compatible.
+
+https://www.consul.io/docs/upgrading.html
+`

version/version_test.go

@@ -0,0 +1,52 @@
+package version
+
+import (
+	"testing"
+)
+
+func TestCheckConsulVersions(t *testing.T) {
+	testCases := []struct {
+		name     string
+		versions []string
+		err      bool
+	}{
+		{
+			"nil",
+			nil,
+			true,
+		}, {
+			"empty",
+			[]string{},
+			true,
+		}, {
+			"valid one",
+			[]string{"1.4.1"},
+			false,
+		}, {
+			"valid",
+			[]string{"1.4.2-dev", "1.5.0+ent", "1.8.0"},
+			false,
+		}, {
+			"invalid one",
+			[]string{"1.4.0"},
+			true,
+		}, {
+			"invalid",
+			[]string{"1.8.1", "1.3.1", "1.4.0-dev"},
+			true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := CheckConsulVersions(tc.versions)
+			if tc.err && err == nil {
+				t.Logf("expected an error for version(s): %q", tc.versions)
+				t.Fail()
+			} else if !tc.err && err != nil {
+				t.Logf("unexpected error for versions %q: %s", tc.versions, err)
+				t.Fail()
+			}
+		})
+	}
+}