Merge pull request #816 from pepov/master

Support different advertise address for WAN gossip

command/agent/agent.go

@@ -127,6 +127,15 @@ func Create(config *Config, logOutput io.Writer) (*Agent, error) {
 		config.AdvertiseAddr = ip.String()
 	}
 
+	// Try to get an advertise address for the wan
+	if config.AdvertiseAddrWan != "" {
+		if ip := net.ParseIP(config.AdvertiseAddrWan); ip == nil {
+			return nil, fmt.Errorf("Failed to parse advertise address for wan: %v", config.AdvertiseAddrWan)
+		}
+	} else {
+		config.AdvertiseAddrWan = config.AdvertiseAddr
+	}
+
 	agent := &Agent{
 		config:        config,
 		logger:        log.New(logOutput, "", log.LstdFlags),
@@ -225,7 +234,11 @@ func (a *Agent) consulConfig() *consul.Config {
 	}
 	if a.config.AdvertiseAddr != "" {
 		base.SerfLANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr
-		base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr
+		if a.config.AdvertiseAddrWan != "" {
+			base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddrWan
+		} else {
+			base.SerfWANConfig.MemberlistConfig.AdvertiseAddr = a.config.AdvertiseAddr
+		}
 		base.RPCAdvertise = &net.TCPAddr{
 			IP:   net.ParseIP(a.config.AdvertiseAddr),
 			Port: a.config.Ports.Server,

command/agent/command.go

@@ -81,6 +81,7 @@ func (c *Command) readConfig() *Config {
 	cmdFlags.StringVar(&cmdConfig.ClientAddr, "client", "", "address to bind client listeners to (DNS, HTTP, HTTPS, RPC)")
 	cmdFlags.StringVar(&cmdConfig.BindAddr, "bind", "", "address to bind server listeners to")
 	cmdFlags.StringVar(&cmdConfig.AdvertiseAddr, "advertise", "", "address to advertise instead of bind addr")
+	cmdFlags.StringVar(&cmdConfig.AdvertiseAddrWan, "advertise-wan", "", "address to advertise on wan instead of bind or advertise addr")
 
 	cmdFlags.StringVar(&cmdConfig.AtlasInfrastructure, "atlas", "", "infrastructure name in Atlas")
 	cmdFlags.StringVar(&cmdConfig.AtlasToken, "atlas-token", "", "authentication token for Atlas")

command/agent/command_test.go

@@ -106,6 +106,32 @@ func TestRetryJoin(t *testing.T) {
 	})
 }
 
+func TestReadCliConfig(t *testing.T) {
+
+	shutdownCh := make(chan struct{})
+	defer close(shutdownCh)
+
+	tmpDir, err := ioutil.TempDir("", "consul")
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	cmd := &Command{
+		args: []string{
+			"-data-dir", tmpDir,
+			"-node", `"a"`,
+			"-advertise-wan", "1.2.3.4",
+		},
+		ShutdownCh: shutdownCh,
+		Ui:         new(cli.MockUi),
+	}
+
+	config := cmd.readConfig()
+	if config.AdvertiseAddrWan != "1.2.3.4" {
+		t.Fatalf("expected -advertise-addr-wan 1.2.3.4 got %s", config.AdvertiseAddrWan)
+	}
+}
+
 func TestRetryJoinFail(t *testing.T) {
 	conf := nextConfig()
 	tmpDir, err := ioutil.TempDir("", "consul")

command/agent/config.go

@@ -142,6 +142,10 @@ type Config struct {
 	// and Consul RPC IP. If not specified, bind address is used.
 	AdvertiseAddr string `mapstructure:"advertise_addr"`
 
+	// AdvertiseAddrWan is the address we use for advertising our
+	// Serf WAN IP. If not specified, the general advertise address is used.
+	AdvertiseAddrWan string `mapstructure:"advertise_addr_wan"`
+
 	// Port configurations
 	Ports PortConfig
 
@@ -804,6 +808,9 @@ func MergeConfig(a, b *Config) *Config {
 	if b.AdvertiseAddr != "" {
 		result.AdvertiseAddr = b.AdvertiseAddr
 	}
+	if b.AdvertiseAddrWan != "" {
+		result.AdvertiseAddrWan = b.AdvertiseAddrWan
+	}
 	if b.Server == true {
 		result.Server = b.Server
 	}

command/agent/config_test.go

@@ -190,10 +190,27 @@ func TestDecodeConfig(t *testing.T) {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	if config.AdvertiseAddrWan != "" {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	if config.Ports.Server != 8000 {
 		t.Fatalf("bad: %#v", config)
 	}
 
+	// Advertise address for wan
+	input = `{"advertise_addr_wan": "127.0.0.5"}`
+	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	if config.AdvertiseAddr != "" {
+		t.Fatalf("bad: %#v", config)
+	}
+	if config.AdvertiseAddrWan != "127.0.0.5" {
+		t.Fatalf("bad: %#v", config)
+	}
+
 	// leave_on_terminate
 	input = `{"leave_on_terminate": true}`
 	config, err = DecodeConfig(bytes.NewReader([]byte(input)))
@@ -1063,12 +1080,13 @@ func TestMergeConfig(t *testing.T) {
 			MaxStale:       30 * time.Second,
 			EnableTruncate: true,
 		},
-		Domain:        "other",
-		LogLevel:      "info",
-		NodeName:      "baz",
-		ClientAddr:    "127.0.0.1",
-		BindAddr:      "127.0.0.1",
-		AdvertiseAddr: "127.0.0.1",
+		Domain:           "other",
+		LogLevel:         "info",
+		NodeName:         "baz",
+		ClientAddr:       "127.0.0.2",
+		BindAddr:         "127.0.0.2",
+		AdvertiseAddr:    "127.0.0.2",
+		AdvertiseAddrWan: "127.0.0.2",
 		Ports: PortConfig{
 			DNS:     1,
 			HTTP:    2,

consul/server_test.go

@@ -216,6 +216,105 @@ func TestServer_JoinWAN(t *testing.T) {
 	})
 }
 
+func TestServer_JoinSeparateLanAndWanAddresses(t *testing.T) {
+	dir1, s1 := testServer(t)
+	defer os.RemoveAll(dir1)
+	defer s1.Shutdown()
+
+	dir2, s2 := testServerWithConfig(t, func(c *Config) {
+		c.NodeName = "s2"
+		c.Datacenter = "dc2"
+		// This wan address will be expected to be seen on s1
+		c.SerfWANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.2"
+		// This lan address will be expected to be seen on s3
+		c.SerfLANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.3"
+	})
+
+	defer os.RemoveAll(dir2)
+	defer s2.Shutdown()
+
+	dir3, s3 := testServerDC(t, "dc2")
+	defer os.RemoveAll(dir3)
+	defer s3.Shutdown()
+
+	// Join s2 to s1 on wan
+	addrs1 := fmt.Sprintf("127.0.0.1:%d",
+		s1.config.SerfWANConfig.MemberlistConfig.BindPort)
+	if _, err := s2.JoinWAN([]string{addrs1}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Join s3 to s2 on lan
+	addrs2 := fmt.Sprintf("127.0.0.1:%d",
+		s2.config.SerfLANConfig.MemberlistConfig.BindPort)
+	if _, err := s3.JoinLAN([]string{addrs2}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Check the WAN members on s1
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s1.WANMembers()) == 2, nil
+	}, func(err error) {
+		t.Fatalf("bad len")
+	})
+
+	// Check the WAN members on s2
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.WANMembers()) == 2, nil
+	}, func(err error) {
+		t.Fatalf("bad len")
+	})
+
+	// Check the LAN members on s2
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s2.LANMembers()) == 2, nil
+	}, func(err error) {
+		t.Fatalf("bad len")
+	})
+
+	// Check the LAN members on s3
+	testutil.WaitForResult(func() (bool, error) {
+		return len(s3.LANMembers()) == 2, nil
+	}, func(err error) {
+		t.Fatalf("bad len")
+	})
+
+	// Check the remoteConsuls has both
+	if len(s1.remoteConsuls) != 2 {
+		t.Fatalf("remote consul missing")
+	}
+
+	if len(s2.remoteConsuls) != 2 {
+		t.Fatalf("remote consul missing")
+	}
+
+	if len(s2.localConsuls) != 2 {
+		t.Fatalf("local consul fellow s3 for s2 missing")
+	}
+
+	// Get and check the wan address of s2 from s1
+	var s2WanAddr string
+	for _, member := range s1.WANMembers() {
+		if member.Name == "s2.dc2" {
+			s2WanAddr = member.Addr.String()
+		}
+	}
+	if s2WanAddr != "127.0.0.2" {
+		t.Fatalf("s1 sees s2 on a wrong address: %s, expecting: %s", s2WanAddr, "127.0.0.2")
+	}
+
+	// Get and check the lan address of s2 from s3
+	var s2LanAddr string
+	for _, lanmember := range s3.LANMembers() {
+		if lanmember.Name == "s2" {
+			s2LanAddr = lanmember.Addr.String()
+		}
+	}
+	if s2LanAddr != "127.0.0.3" {
+		t.Fatalf("s3 sees s2 on a wrong address: %s, expecting: %s", s2LanAddr, "127.0.0.3")
+	}
+}
+
 func TestServer_LeaveLeader(t *testing.T) {
 	dir1, s1 := testServer(t)
 	defer os.RemoveAll(dir1)

website/source/docs/agent/options.html.markdown

@@ -41,6 +41,14 @@ The options below are all specified on the command-line.
   If this address is not routable, the node will be in a constant flapping state
   as other nodes will treat the non-routability as a failure.
 
+* <a name="_advertise-wan"></a><a href="#_advertise-wan">`-advertise-wan`</a> - The advertise wan
+  address is used to change the address that we advertise to server nodes joining
+  through the WAN. By default, the [`-advertise`](#_advertise) address is advertised.
+  However, in some cases all members of all datacenters cannot be on the same
+  physical or virtual network, especially on hybrid setups mixing cloud and private datacenters.
+  This flag enables server nodes gossiping through the public network for the WAN while using
+  private VLANs for gossiping to each other and their client agents.
+
 * <a name="_atlas"></a><a href="#_atlas">`-atlas`</a> - This flag
   enables [Atlas](https://atlas.hashicorp.com) integration.
   It is used to provide the Atlas infrastructure name and the SCADA connection.
@@ -312,6 +320,9 @@ definitions support being updated during a reload.
 * <a name="advertise_addr"></a><a href="#advertise_addr">`advertise_addr`</a> Equivalent to
   the [`-advertise` command-line flag](#_advertise).
 
+* <a name="advertise_addr_wan"></a><a href="#advertise_addr_wan">`advertise_addr_wan`</a> Equivalent to
+  the [`-advertise-wan` command-line flag](#_advertise-wan).
+
 * <a name="atlas_acl_token"></a><a href="#atlas_acl_token">`atlas_acl_token`</a> When provided,
   any requests made by Atlas will use this ACL token unless explicitly overriden. When not provided
   the [`acl_token`](#acl_token) is used. This can be set to 'anonymous' to reduce permission below