Add cluster-tracing option to meta config

Useful for troubleshooting but too verbose for regular use.

cmd/influxd/restore/restore.go

@@ -229,8 +229,8 @@ restore uses a snapshot of a data node to rebuild a cluster.
 
 // Config represents a partial config for rebuilding the server.
 type Config struct {
-	Meta meta.Config `toml:"meta"`
-	Data tsdb.Config `toml:"data"`
+	Meta *meta.Config `toml:"meta"`
+	Data tsdb.Config  `toml:"data"`
 }
 
 type nopListener struct {

cmd/influxd/run/config.go

@@ -24,7 +24,7 @@ import (
 
 // Config represents the configuration format for the influxd binary.
 type Config struct {
-	Meta       meta.Config       `toml:"meta"`
+	Meta       *meta.Config      `toml:"meta"`
 	Data       tsdb.Config       `toml:"data"`
 	Cluster    cluster.Config    `toml:"cluster"`
 	Retention  retention.Config  `toml:"retention"`

meta/config.go

@@ -37,17 +37,18 @@ type Config struct {
 	HeartbeatTimeout    toml.Duration `toml:"heartbeat-timeout"`
 	LeaderLeaseTimeout  toml.Duration `toml:"leader-lease-timeout"`
 	CommitTimeout       toml.Duration `toml:"commit-timeout"`
+	ClusterTracing      bool          `toml:"cluster-tracing"`
 
 	// The join command-line argument
 	join string
 }
 
-func (c Config) SetJoin(join string) {
+func (c *Config) SetJoin(join string) {
 	c.join = join
 }
 
-func NewConfig() Config {
-	return Config{
+func NewConfig() *Config {
+	return &Config{
 		Hostname:            DefaultHostname,
 		BindAddress:         DefaultBindAddress,
 		RetentionAutoCreate: true,

meta/rpc.go

@@ -16,7 +16,8 @@ import (
 
 // RPC handles request/response style messaging between cluster nodes
 type RPC struct {
-	Logger *log.Logger
+	Logger         *log.Logger
+	tracingEnabled bool
 
 	store interface {
 		cachedData() *Data
@@ -66,7 +67,8 @@ func (r *RPC) handleRPCConn(conn net.Conn) {
 	// RPC connections should execute on the leader.  If we are not the leader,
 	// proxy the connection to the leader so that clients an connect to any node
 	// in the cluster.
-	r.Logger.Printf("rpc connection from: %v", conn.RemoteAddr())
+	r.traceCluster("rpc connection from: %v", conn.RemoteAddr())
+
 	if !r.store.IsLeader() {
 		r.proxyLeader(conn.(*net.TCPConn))
 		return
@@ -90,7 +92,7 @@ func (r *RPC) handleRPCConn(conn net.Conn) {
 		rpcType := internal.RPCType(btou64(buf[0:8]))
 		buf = buf[8:]
 
-		r.Logger.Printf("recv %v request from: %v", rpcType, conn.RemoteAddr())
+		r.traceCluster("recv %v request on: %v", rpcType, conn.RemoteAddr())
 		switch rpcType {
 		case internal.RPCType_FetchData:
 			var req internal.FetchDataRequest
@@ -147,6 +149,7 @@ func (r *RPC) sendResponse(conn net.Conn, typ internal.RPCType, resp proto.Messa
 }
 
 func (r *RPC) sendError(conn net.Conn, msg string) {
+	r.traceCluster(msg)
 	resp := &internal.ErrorResponse{
 		Header: &internal.ResponseHeader{
 			OK:    proto.Bool(false),
@@ -195,7 +198,7 @@ func (r *RPC) handleFetchData(req *internal.FetchDataRequest) (*internal.FetchDa
 
 // handleJoinRequest handles a request to join the cluster
 func (r *RPC) handleJoinRequest(req *internal.JoinRequest) (*internal.JoinResponse, error) {
-	r.Logger.Printf("recv join request from: %v", *req.Addr)
+	r.traceCluster("join request from: %v", *req.Addr)
 
 	node, err := func() (*NodeInfo, error) {
 		// attempt to create the node
@@ -401,6 +404,12 @@ func (r *RPC) call(dest string, req proto.Message) (proto.Message, error) {
 	return resp, nil
 }
 
+func (r *RPC) traceCluster(msg string, args ...interface{}) {
+	if r.tracingEnabled {
+		r.Logger.Printf("rpc error: "+msg, args...)
+	}
+}
+
 func u64tob(v uint64) []byte {
 	b := make([]byte, 8)
 	binary.BigEndian.PutUint64(b, v)

meta/store.go

@@ -82,6 +82,10 @@ type Store struct {
 	wg      sync.WaitGroup
 	changed chan struct{}
 
+	// clusterTracingEnabled controls whether low-level cluster communcation is logged.
+	// Useful for troubleshooting
+	clusterTracingEnabled bool
+
 	retentionAutoCreate bool
 
 	// The listeners to accept raft and remote exec connections from.
@@ -123,7 +127,7 @@ type authUser struct {
 }
 
 // NewStore returns a new instance of Store.
-func NewStore(c Config) *Store {
+func NewStore(c *Config) *Store {
 	s := &Store{
 		path:  c.Dir,
 		peers: c.Peers,
@@ -135,7 +139,8 @@ func NewStore(c Config) *Store {
 		closing: make(chan struct{}),
 		changed: make(chan struct{}),
 
-		retentionAutoCreate: c.RetentionAutoCreate,
+		clusterTracingEnabled: c.ClusterTracing,
+		retentionAutoCreate:   c.RetentionAutoCreate,
 
 		HeartbeatTimeout:   time.Duration(c.HeartbeatTimeout),
 		ElectionTimeout:    time.Duration(c.ElectionTimeout),
@@ -150,8 +155,9 @@ func NewStore(c Config) *Store {
 
 	s.raftState = &localRaft{store: s}
 	s.rpc = &RPC{
-		store:  s,
-		Logger: s.Logger,
+		store:          s,
+		tracingEnabled: c.ClusterTracing,
+		Logger:         s.Logger,
 	}
 	return s
 }

meta/store_test.go

@@ -817,7 +817,7 @@ type Store struct {
 }
 
 // NewStore returns a new test wrapper for Store.
-func NewStore(c meta.Config) *Store {
+func NewStore(c *meta.Config) *Store {
 	s := &Store{
 		Store: meta.NewStore(c),
 	}
@@ -881,8 +881,8 @@ func (s *Store) Close() error {
 }
 
 // NewConfig returns the default test configuration.
-func NewConfig(path string) meta.Config {
-	return meta.Config{
+func NewConfig(path string) *meta.Config {
+	return &meta.Config{
 		Dir:                path,
 		Hostname:           "localhost",
 		BindAddress:        "127.0.0.1:0",