Add monitor http endpoint

api/agent.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bufio"
 	"fmt"
 )
 
@@ -410,3 +411,38 @@ func (a *Agent) DisableNodeMaintenance() error {
 	resp.Body.Close()
 	return nil
 }
+
+// Monitor returns a channel which will receive streaming logs from the agent
+// Providing a non-nil stopCh can be used to close the connection and stop the
+// log stream
+func (a *Agent) Monitor(loglevel string, stopCh chan struct{}, q *QueryOptions) (chan string, error) {
+	r := a.c.newRequest("GET", "/v1/agent/monitor")
+	r.setQueryOptions(q)
+	if loglevel != "" {
+		r.params.Add("loglevel", loglevel)
+	}
+	_, resp, err := requireOK(a.c.doRequest(r))
+	if err != nil {
+		return nil, err
+	}
+
+	logCh := make(chan string, 64)
+	go func() {
+		defer resp.Body.Close()
+
+		scanner := bufio.NewScanner(resp.Body)
+		for {
+			select {
+			case <-stopCh:
+				close(logCh)
+				return
+			default:
+			}
+			if scanner.Scan() {
+				logCh <- scanner.Text()
+			}
+		}
+	}()
+
+	return logCh, nil
+}

api/agent_test.go

@@ -3,6 +3,7 @@ package api
 import (
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestAgent_Self(t *testing.T) {
@@ -558,6 +559,29 @@ func TestAgent_ForceLeave(t *testing.T) {
 	}
 }
 
+func TestAgent_Monitor(t *testing.T) {
+	t.Parallel()
+	c, s := makeClient(t)
+	defer s.Stop()
+
+	agent := c.Agent()
+
+	logCh, err := agent.Monitor("info", nil, nil)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Wait for the first log message and validate it
+	select {
+	case log := <-logCh:
+		if !strings.Contains(log, "[INFO] raft: Initial configuration") {
+			t.Fatalf("bad: %q", log)
+		}
+	case <-time.After(10 * time.Second):
+		t.Fatalf("failed to get a log message")
+	}
+}
+
 func TestServiceMaintenance(t *testing.T) {
 	t.Parallel()
 	c, s := makeClient(t)

command/agent/agent.go

@@ -19,6 +19,7 @@ import (
 	"github.com/hashicorp/consul/consul/state"
 	"github.com/hashicorp/consul/consul/structs"
 	"github.com/hashicorp/consul/lib"
+	"github.com/hashicorp/consul/logger"
 	"github.com/hashicorp/consul/types"
 	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/serf/coordinate"
@@ -66,6 +67,9 @@ type Agent struct {
 	// Output sink for logs
 	logOutput io.Writer
 
+	// Used for streaming logs to
+	logWriter *logger.LogWriter
+
 	// We have one of a client or a server, depending
 	// on our configuration
 	server *consul.Server
@@ -121,7 +125,7 @@ type Agent struct {
 
 // Create is used to create a new Agent. Returns
 // the agent or potentially an error.
-func Create(config *Config, logOutput io.Writer) (*Agent, error) {
+func Create(config *Config, logOutput io.Writer, logWriter *logger.LogWriter) (*Agent, error) {
 	// Ensure we have a log sink
 	if logOutput == nil {
 		logOutput = os.Stderr
@@ -175,6 +179,7 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) {
 		config:         config,
 		logger:         log.New(logOutput, "", log.LstdFlags),
 		logOutput:      logOutput,
+		logWriter:      logWriter,
 		checkReapAfter: make(map[types.CheckID]time.Duration),
 		checkMonitors:  make(map[types.CheckID]*CheckMonitor),
 		checkTTLs:      make(map[types.CheckID]*CheckTTL),

command/agent/agent_endpoint.go

@@ -2,12 +2,15 @@ package agent
 
 import (
 	"fmt"
+	"log"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/logger"
 	"github.com/hashicorp/consul/types"
+	"github.com/hashicorp/logutils"
 	"github.com/hashicorp/serf/coordinate"
 	"github.com/hashicorp/serf/serf"
 )
@@ -393,6 +396,74 @@ func (s *HTTPServer) AgentNodeMaintenance(resp http.ResponseWriter, req *http.Re
 	return nil, nil
 }
 
+func (s *HTTPServer) AgentMonitor(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
+	// Only GET supported
+	if req.Method != "GET" {
+		resp.WriteHeader(405)
+		return nil, nil
+	}
+
+	var args structs.DCSpecificRequest
+	args.Datacenter = s.agent.config.Datacenter
+	s.parseToken(req, &args.Token)
+	// Validate that the given token has operator permissions
+	var reply structs.RaftConfigurationResponse
+	if err := s.agent.RPC("Operator.RaftGetConfiguration", &args, &reply); err != nil {
+		return nil, err
+	}
+
+	// Get the provided loglevel
+	logLevel := req.URL.Query().Get("loglevel")
+	if logLevel == "" {
+		logLevel = "INFO"
+	}
+
+	// Upper case the log level
+	logLevel = strings.ToUpper(logLevel)
+
+	// Create a level filter
+	filter := logger.LevelFilter()
+	filter.MinLevel = logutils.LogLevel(logLevel)
+	if !logger.ValidateLevelFilter(filter.MinLevel, filter) {
+		resp.WriteHeader(400)
+		resp.Write([]byte(fmt.Sprintf("Unknown log level: %s", filter.MinLevel)))
+		return nil, nil
+	}
+
+	flusher, ok := resp.(http.Flusher)
+	if !ok {
+		return nil, fmt.Errorf("Streaming not supported")
+	}
+
+	// Set up a log handler
+	handler := &httpLogHandler{
+		filter: filter,
+		logCh:  make(chan string, 512),
+		logger: s.logger,
+	}
+	s.agent.logWriter.RegisterHandler(handler)
+	defer s.agent.logWriter.DeregisterHandler(handler)
+
+	notify := resp.(http.CloseNotifier).CloseNotify()
+
+	// Stream logs until the connection is closed
+	for {
+		select {
+		case <-notify:
+			s.agent.logWriter.DeregisterHandler(handler)
+			if handler.droppedCount > 0 {
+				s.agent.logger.Printf("[WARN] agent: Dropped %d logs during monitor request", handler.droppedCount)
+			}
+			return nil, nil
+		case log := <-handler.logCh:
+			resp.Write([]byte(log + "\n"))
+			flusher.Flush()
+		}
+	}
+
+	return nil, nil
+}
+
 // syncChanges is a helper function which wraps a blocking call to sync
 // services and checks to the server. If the operation fails, we only
 // only warn because the write did succeed and anti-entropy will sync later.
@@ -401,3 +472,26 @@ func (s *HTTPServer) syncChanges() {
 		s.logger.Printf("[ERR] agent: failed to sync changes: %v", err)
 	}
 }
+
+type httpLogHandler struct {
+	filter       *logutils.LevelFilter
+	logCh        chan string
+	logger       *log.Logger
+	droppedCount int
+}
+
+func (h *httpLogHandler) HandleLog(log string) {
+	// Check the log level
+	if !h.filter.Check([]byte(log)) {
+		return
+	}
+
+	// Do a non-blocking send
+	select {
+	case h.logCh <- log:
+	default:
+		// Just increment a counter for dropped logs to this handler; we can't log now
+		// because the lock is already held by the LogWriter invoking this
+		h.droppedCount += 1
+	}
+}

command/agent/agent_endpoint_test.go

@@ -1,8 +1,11 @@
 package agent
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -12,6 +15,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/consul/consul/structs"
+	"github.com/hashicorp/consul/logger"
 	"github.com/hashicorp/consul/testutil"
 	"github.com/hashicorp/consul/types"
 	"github.com/hashicorp/serf/serf"
@@ -1019,3 +1023,99 @@ func TestHTTPAgentRegisterServiceCheck(t *testing.T) {
 		t.Fatalf("bad: %#v", result["memcached_check2"])
 	}
 }
+
+func TestHTTPAgent_Monitor(t *testing.T) {
+	logWriter := logger.NewLogWriter(512)
+	expectedLogs := bytes.Buffer{}
+	logger := io.MultiWriter(os.Stdout, &expectedLogs, logWriter)
+
+	dir, srv := makeHTTPServerWithConfigLog(t, nil, logger, logWriter)
+	srv.agent.logWriter = logWriter
+	defer os.RemoveAll(dir)
+	defer srv.Shutdown()
+	defer srv.agent.Shutdown()
+
+	// Try passing an invalid log level
+	req, _ := http.NewRequest("GET", "/v1/agent/monitor?loglevel=invalid", nil)
+	resp := newClosableRecorder()
+	if _, err := srv.AgentMonitor(resp, req); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if resp.Code != 400 {
+		t.Fatalf("bad: %v", resp.Code)
+	}
+	body, _ := ioutil.ReadAll(resp.Body)
+	if !strings.Contains(string(body), "Unknown log level") {
+		t.Fatalf("bad: %s", body)
+	}
+
+	// Begin streaming logs from the monitor endpoint
+	req, _ = http.NewRequest("GET", "/v1/agent/monitor?loglevel=debug", nil)
+	resp = newClosableRecorder()
+	go func() {
+		if _, err := srv.AgentMonitor(resp, req); err != nil {
+			t.Fatalf("err: %s", err)
+		}
+	}()
+
+	// Write the incoming logs from http to a channel for comparison
+	logCh := make(chan string, 5)
+
+	// Block until the first log entry from http
+	testutil.WaitForResult(func() (bool, error) {
+		line, err := resp.Body.ReadString('\n')
+		if err != nil && err != io.EOF {
+			return false, fmt.Errorf("err: %v", err)
+		}
+		if line == "" {
+			return false, fmt.Errorf("blank line")
+		}
+		logCh <- line
+		return true, nil
+	}, func(err error) {
+		t.Fatal(err)
+	})
+
+	go func() {
+		for {
+			line, err := resp.Body.ReadString('\n')
+			if err != nil && err != io.EOF {
+				t.Fatalf("err: %v", err)
+			}
+			if line != "" {
+				logCh <- line
+			}
+		}
+	}()
+
+	// Verify that the first 5 logs we get match the expected stream
+	for i := 0; i < 5; i++ {
+		select {
+		case log := <-logCh:
+			expected, err := expectedLogs.ReadString('\n')
+			if err != nil {
+				t.Fatalf("err: %v", err)
+			}
+			if log != expected {
+				t.Fatalf("bad: %q %q", expected, log)
+			}
+		case <-time.After(10 * time.Second):
+			t.Fatalf("failed to get log within timeout")
+		}
+	}
+}
+
+type closableRecorder struct {
+	*httptest.ResponseRecorder
+	closer chan bool
+}
+
+func newClosableRecorder() *closableRecorder {
+	r := httptest.NewRecorder()
+	closer := make(chan bool)
+	return &closableRecorder{r, closer}
+}
+
+func (r *closableRecorder) CloseNotify() <-chan bool {
+	return r.closer
+}