Add leave http endpoint

api/agent.go

@@ -358,6 +358,17 @@ func (a *Agent) Join(addr string, wan bool) error {
 	return nil
 }
 
+// Leave is used to have the agent gracefully leave the cluster and shutdown
+func (a *Agent) Leave() error {
+	r := a.c.newRequest("PUT", "/v1/agent/leave")
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return err
+	}
+	resp.Body.Close()
+	return nil
+}
+
 // ForceLeave is used to have the agent eject a failed node
 func (a *Agent) ForceLeave(node string) error {
 	r := a.c.newRequest("PUT", "/v1/agent/force-leave/"+node)

api/agent_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/consul/testutil"
+	"github.com/hashicorp/serf/serf"
 )
 
 func TestAgent_Self(t *testing.T) {
@@ -592,6 +593,39 @@ func TestAgent_Join(t *testing.T) {
 	}
 }
 
+func TestAgent_Leave(t *testing.T) {
+	t.Parallel()
+	c1, s1 := makeClient(t)
+	defer s1.Stop()
+
+	c2, s2 := makeClientWithConfig(t, nil, func(conf *testutil.TestServerConfig) {
+		conf.Server = false
+		conf.Bootstrap = false
+	})
+	defer s2.Stop()
+
+	if err := c2.Agent().Join(s1.LANAddr, false); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	if err := c2.Agent().Leave(); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Make sure the second agent's status is 'Left'
+	members, err := c1.Agent().Members(false)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	member := members[0]
+	if member.Name == s1.Config.NodeName {
+		member = members[1]
+	}
+	if member.Status != int(serf.StatusLeft) {
+		t.Fatalf("bad: %v", *member)
+	}
+}
+
 func TestAgent_ForceLeave(t *testing.T) {
 	t.Parallel()
 	c, s := makeClient(t)

api/catalog.go

@@ -16,8 +16,8 @@ type CatalogService struct {
 	ServiceTags              []string
 	ServicePort              int
 	ServiceEnableTagOverride bool
-	CreateIndex		 uint64
-	ModifyIndex		 uint64
+	CreateIndex              uint64
+	ModifyIndex              uint64
 }
 
 type CatalogNode struct {

command/agent/agent_endpoint.go

@@ -82,6 +82,13 @@ func (s *HTTPServer) AgentJoin(resp http.ResponseWriter, req *http.Request) (int
 	}
 }
 
+func (s *HTTPServer) AgentLeave(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	if err := s.agent.Leave(); err != nil {
+		return nil, err
+	}
+	return nil, s.agent.Shutdown()
+}
+
 func (s *HTTPServer) AgentForceLeave(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
 	addr := strings.TrimPrefix(req.URL.Path, "/v1/agent/force-leave/")
 	return nil, s.agent.ForceLeave(addr)

command/agent/agent_endpoint_test.go

@@ -235,6 +235,49 @@ func TestHTTPAgentJoin_WAN(t *testing.T) {
 	})
 }
 
+func TestHTTPAgentLeave(t *testing.T) {
+	dir, srv := makeHTTPServer(t)
+	defer os.RemoveAll(dir)
+	defer srv.Shutdown()
+	defer srv.agent.Shutdown()
+
+	dir2, srv2 := makeHTTPServerWithConfig(t, func(c *Config) {
+		c.Server = false
+		c.Bootstrap = false
+	})
+	defer os.RemoveAll(dir2)
+	defer srv2.Shutdown()
+
+	// Join first
+	addr := fmt.Sprintf("127.0.0.1:%d", srv2.agent.config.Ports.SerfLan)
+	_, err := srv.agent.JoinLAN([]string{addr})
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Graceful leave now
+	req, err := http.NewRequest("PUT", "/v1/agent/leave", nil)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	obj, err := srv2.AgentLeave(nil, req)
+	if err != nil {
+		t.Fatalf("Err: %v", err)
+	}
+	if obj != nil {
+		t.Fatalf("Err: %v", obj)
+	}
+
+	testutil.WaitForResult(func() (bool, error) {
+		m := srv.agent.LANMembers()
+		success := m[1].Status == serf.StatusLeft
+		return success, errors.New(m[1].Status.String())
+	}, func(err error) {
+		t.Fatalf("member status is %v, should be left", err)
+	})
+}
+
 func TestHTTPAgentForceLeave(t *testing.T) {
 	dir, srv := makeHTTPServer(t)
 	defer os.RemoveAll(dir)

command/agent/http.go

@@ -256,6 +256,7 @@ func (s *HTTPServer) registerHandlers(enableDebug bool) {
 	s.handleFuncMetrics("/v1/agent/checks", s.wrap(s.AgentChecks))
 	s.handleFuncMetrics("/v1/agent/members", s.wrap(s.AgentMembers))
 	s.handleFuncMetrics("/v1/agent/join/", s.wrap(s.AgentJoin))
+	s.handleFuncMetrics("/v1/agent/leave", s.wrap(s.AgentLeave))
 	s.handleFuncMetrics("/v1/agent/force-leave/", s.wrap(s.AgentForceLeave))
 	s.handleFuncMetrics("/v1/agent/check/register", s.wrap(s.AgentRegisterCheck))
 	s.handleFuncMetrics("/v1/agent/check/deregister/", s.wrap(s.AgentDeregisterCheck))

website/source/docs/agent/http/agent.html.markdown

@@ -23,6 +23,7 @@ The following endpoints are supported:
 * [`/v1/agent/reload`](#agent_reload) : Causes the local agent to reload its configuration
 * [`/v1/agent/maintenance`](#agent_maintenance) : Manages node maintenance mode
 * [`/v1/agent/join/<address>`](#agent_join) : Triggers the local agent to join a node
+* [`/v1/agent/leave`](#agent_leave): Triggers the local agent to gracefully shutdown and leave the cluster
 * [`/v1/agent/force-leave/<node>`](#agent_force_leave): Forces removal of a node
 * [`/v1/agent/check/register`](#agent_check_register) : Registers a new local check
 * [`/v1/agent/check/deregister/<checkID>`](#agent_check_deregister) : Deregisters a local check
@@ -234,6 +235,19 @@ The return code is 200 on success.
 
 ### <a name="agent_force_leave"></a> /v1/agent/force-leave/\<node\>
 
+This endpoint is hit with a GET and is used to trigger a graceful leave and shutdown
+of the agent. It is used to ensure other nodes see the agent as "left" instead of
+"failed". Nodes that leave will not attempt to re-join the cluster on restarting
+with a snapshot.
+
+For nodes in server mode, the node is removed from the Raft peer set in a graceful
+manner. This is critical, as in certain situations a non-graceful leave can affect
+cluster availability.
+
+The return code is 200 on success.
+
+### <a name="agent_force_leave"></a> /v1/agent/force-leave/\<node\>
+
 This endpoint is hit with a GET and is used to instruct the agent to force a node into the `left` state.
 If a node fails unexpectedly, then it will be in a `failed` state. Once in the `failed` state, Consul will
 attempt to reconnect, and the services and checks belonging to that node will not be