[PIP 95][Issue 12040][broker] Multiple bind addresses for Pulsar prot…

…ocol (#12056)

Master Issue: #12040

### Motivation

Add a new configuration setting `bindAddresses` to open additional server ports on the broker.  Note that these are in addition to `brokerServicePort` / `brokerServicePortTls` for compatibility reasons.

Each new-style bind address is associated with an advertised listener, to be used as the default listener for topic lookup requests.   See #12040 for details.  A given listener may be associated with numerous bindings.

The scheme indicates the protocol handler and whether to use TLS on the server channel.  This PR is focused on the Pulsar protocol handler, but it is anticipated that other protocols may be supported in future.

For example:
```
bindAddresses=external:pulsar://0.0.0.0:6652,external:pulsar+ssl://0.0.0.0:6653
bindAddress=0.0.0.0
brokerServicePort=6650
brokerServicePortTls=6651
advertisedListeners=cluster:pulsar://broker-1.local:6650,cluster:pulsar+ssl://broker-1.local:6651,external:pulsar://broker-1.example.dev:6652,external:pulsar+ssl://broker-1.example.dev:6653
internalListenerName=cluster
```

The above would produce three server sockets, with `6650` having no associated listener name (thus retaining existing lookup behavior of returning the internal listener), and with `6652` and `6653` having an association with listener name `external`.  Given a lookup request on `6652` or `6653`, the `external` listener address would be returned.

### Modifications

- added configuration property `bindAddresses`
- implementing parsing and validation logic
- factored some utility code for formatting broker/web addresses

conf/broker.conf

@@ -40,6 +40,9 @@ webServicePortTls=
 # Hostname or IP address the service binds on, default is 0.0.0.0.
 bindAddress=0.0.0.0
 
+# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+bindAddresses=
+
 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
 advertisedAddress=
 

conf/standalone.conf

@@ -33,6 +33,9 @@ webServicePort=8080
 # Hostname or IP address the service binds on, default is 0.0.0.0.
 bindAddress=0.0.0.0
 
+# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
+bindAddresses=
+
 # Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
 advertisedAddress=
 

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfiguration.java

@@ -121,13 +121,16 @@ public class ServiceConfiguration implements PulsarConfiguration {
     private String configurationStoreServers;
     @FieldContext(
         category = CATEGORY_SERVER,
-        doc = "The port for serving binary protobuf requests"
+        doc = "The port for serving binary protobuf requests."
+            + " If set, defines a server binding for bindAddress:brokerServicePort."
+            + " The Default value is 6650."
     )
 
     private Optional<Integer> brokerServicePort = Optional.of(6650);
     @FieldContext(
         category = CATEGORY_SERVER,
-        doc = "The port for serving tls secured binary protobuf requests"
+        doc = "The port for serving TLS-secured binary protobuf requests."
+            + " If set, defines a server binding for bindAddress:brokerServicePortTls."
     )
     private Optional<Integer> brokerServicePortTls = Optional.empty();
     @FieldContext(
@@ -168,6 +171,15 @@ public class ServiceConfiguration implements PulsarConfiguration {
                     + "The Default value is absent, the broker uses the first listener as the internal listener.")
     private String internalListenerName;
 
+    @FieldContext(category=CATEGORY_SERVER,
+            doc = "Used to specify additional bind addresses for the broker."
+                    + " The value must format as <listener_name>:<scheme>://<host>:<port>,"
+                    + " multiple bind addresses should be separated with commas."
+                    + " Associates each bind address with an advertised listener and protocol handler."
+                    + " Note that the brokerServicePort, brokerServicePortTls, webServicePort, and"
+                    + " webServicePortTls properties define additional bindings.")
+    private String bindAddresses;
+
     @FieldContext(category=CATEGORY_SERVER,
             doc = "Enable or disable the proxy protocol.")
     private boolean haProxyProtocolEnabled;

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/ServiceConfigurationUtils.java

@@ -80,4 +80,19 @@ public static String getAppliedAdvertisedAddress(ServiceConfiguration configurat
         return getDefaultOrConfiguredAddress(advertisedAddress);
     }
 
+    public static String brokerUrl(String host, int port) {
+        return String.format("pulsar://%s:%d", host, port);
+    }
+
+    public static String brokerUrlTls(String host, int port) {
+        return String.format("pulsar+ssl://%s:%d", host, port);
+    }
+
+    public static String webServiceUrl(String host, int port) {
+        return String.format("http://%s:%d", host, port);
+    }
+
+    public static String webServiceUrlTls(String host, int port) {
+        return String.format("https://%s:%d", host, port);
+    }
 }

pulsar-broker-common/src/main/java/org/apache/pulsar/broker/validator/BindAddressValidator.java

@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.validator;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.ArrayList;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.broker.ServiceConfigurationUtils;
+import org.apache.pulsar.common.configuration.BindAddress;
+
+/**
+ * Validates bind address configurations.
+ */
+public class BindAddressValidator {
+
+    private static final Pattern BIND_ADDRESSES_PATTERN = Pattern.compile("(?<name>\\w+):(?<url>.+)$");
+
+    /**
+     * Validate the configuration of `bindAddresses`.
+     * @param config the pulsar broker configure.
+     * @param schemes a filter on the schemes of the bind addresses, or null to not apply a filter.
+     * @return a list of bind addresses.
+     */
+    public static List<BindAddress> validateBindAddresses(ServiceConfiguration config, Collection<String> schemes) {
+        // migrate the existing configuration properties
+        List<BindAddress> addresses = migrateBindAddresses(config);
+
+        // parse the list of additional bind addresses
+        Arrays
+                .stream(StringUtils.split(StringUtils.defaultString(config.getBindAddresses()), ","))
+                .map(s -> {
+                    Matcher m = BIND_ADDRESSES_PATTERN.matcher(s);
+                    if (!m.matches()) {
+                        throw new IllegalArgumentException("bindAddresses: malformed: " + s);
+                    }
+                    return m;
+                })
+                .map(m -> new BindAddress(m.group("name"), URI.create(m.group("url"))))
+                .forEach(addresses::add);
+
+        // apply the filter
+        if (schemes != null) {
+            addresses.removeIf(a -> !schemes.contains(a.getAddress().getScheme()));
+        }
+
+        return addresses;
+    }
+
+    /**
+     * Generates bind addresses based on legacy configuration properties.
+     */
+    private static List<BindAddress> migrateBindAddresses(ServiceConfiguration config) {
+        List<BindAddress> addresses = new ArrayList<>(2);
+        if (config.getBrokerServicePort().isPresent()) {
+            addresses.add(new BindAddress(null, URI.create(
+                    ServiceConfigurationUtils.brokerUrl(config.getBindAddress(), config.getBrokerServicePort().get()))));
+        }
+        if (config.getBrokerServicePortTls().isPresent()) {
+            addresses.add(new BindAddress(null, URI.create(
+                    ServiceConfigurationUtils.brokerUrlTls(config.getBindAddress(), config.getBrokerServicePortTls().get()))));
+        }
+        if (config.getWebServicePort().isPresent()) {
+            addresses.add(new BindAddress(null, URI.create(
+                    ServiceConfigurationUtils.webServiceUrl(config.getBindAddress(), config.getWebServicePort().get()))));
+        }
+        if (config.getWebServicePortTls().isPresent()) {
+            addresses.add(new BindAddress(null, URI.create(
+                    ServiceConfigurationUtils.webServiceUrlTls(config.getBindAddress(), config.getWebServicePortTls().get()))));
+        }
+        return addresses;
+    }
+}

pulsar-broker-common/src/main/java/org/apache/pulsar/common/configuration/BindAddress.java

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.common.configuration;
+
+import java.net.URI;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import lombok.Setter;
+import lombok.ToString;
+
+/**
+ * A bind address for the broker as a non-TLS and TLS pair.
+ */
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@EqualsAndHashCode
+@ToString
+public class BindAddress {
+
+    /**
+     * The listener name associated with the bind address, or null if no listener is associated.
+     */
+    @Getter
+    @Setter
+    private String listenerName;
+
+    /**
+     * The bind address.
+     */
+    @Getter
+    @Setter
+    @NonNull
+    private URI address;
+}

pulsar-broker-common/src/test/java/org/apache/pulsar/broker/validator/BindAddressValidatorTest.java

@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pulsar.broker.validator;
+
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import org.apache.pulsar.broker.ServiceConfiguration;
+import org.apache.pulsar.common.configuration.BindAddress;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.assertEquals;
+
+/**
+ * testcase for BindAddressValidator.
+ */
+public class BindAddressValidatorTest {
+
+    /**
+     * Provides a configuration with no bind addresses specified.
+     */
+    private ServiceConfiguration newEmptyConfiguration() {
+        ServiceConfiguration config = new ServiceConfiguration();
+        config.setBrokerServicePort(Optional.empty());  // default: 6650
+        config.setBrokerServicePortTls(Optional.empty());
+        config.setWebServicePort(Optional.empty());     // default: 8080
+        config.setWebServicePortTls(Optional.empty());
+        return config;
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testMalformed() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBindAddresses("internal:");
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(0, addresses.size());
+    }
+
+    @Test
+    public void testOneListenerMultipleAddresses() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBindAddresses("internal:pulsar://0.0.0.0:6650,internal:pulsar+ssl://0.0.0.0:6651");
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress("internal", URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress("internal", URI.create("pulsar+ssl://0.0.0.0:6651"))), addresses);
+    }
+
+    @Test
+    public void testMultiListener() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBindAddresses("internal:pulsar://0.0.0.0:6650,external:pulsar+ssl://0.0.0.0:6651");
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress("internal", URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress("external", URI.create("pulsar+ssl://0.0.0.0:6651"))), addresses);
+    }
+
+    @Test
+    public void testMigrationWithAllOptions() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBrokerServicePort(Optional.of(6650));
+        config.setBrokerServicePortTls(Optional.of(6651));
+        config.setWebServicePort(Optional.of(8080));
+        config.setWebServicePortTls(Optional.of(443));
+        config.setBindAddress("0.0.0.0");
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress(null, URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress(null, URI.create("pulsar+ssl://0.0.0.0:6651")),
+                new BindAddress(null, URI.create("http://0.0.0.0:8080")),
+                new BindAddress(null, URI.create("https://0.0.0.0:443"))), addresses);
+    }
+
+    @Test
+    public void testMigrationWithDefaults() {
+        ServiceConfiguration config = new ServiceConfiguration();
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress(null, URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress(null, URI.create("http://0.0.0.0:8080"))), addresses);
+    }
+
+    @Test
+    public void testMigrationWithExtra() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBrokerServicePort(Optional.of(6650));
+        config.setBindAddresses("extra:pulsar://0.0.0.0:6652");
+        List<BindAddress> addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress(null, URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress("extra", URI.create("pulsar://0.0.0.0:6652"))), addresses);
+    }
+
+    @Test
+    public void testSchemeFilter() {
+        ServiceConfiguration config = newEmptyConfiguration();
+        config.setBrokerServicePort(Optional.of(6650));
+        config.setBrokerServicePortTls(Optional.of(6651));
+        config.setBindAddresses("extra:pulsar://0.0.0.0:6652,extra:http://0.0.0.0:8080");
+
+        List<BindAddress> addresses;
+        addresses = BindAddressValidator.validateBindAddresses(config, null);
+        assertEquals(Arrays.asList(
+                new BindAddress(null, URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress(null, URI.create("pulsar+ssl://0.0.0.0:6651")),
+                new BindAddress("extra", URI.create("pulsar://0.0.0.0:6652")),
+                new BindAddress("extra", URI.create("http://0.0.0.0:8080"))), addresses);
+
+        addresses = BindAddressValidator.validateBindAddresses(config, Arrays.asList("pulsar", "pulsar+ssl"));
+        assertEquals(Arrays.asList(
+                new BindAddress(null, URI.create("pulsar://0.0.0.0:6650")),
+                new BindAddress(null, URI.create("pulsar+ssl://0.0.0.0:6651")),
+                new BindAddress("extra", URI.create("pulsar://0.0.0.0:6652"))), addresses);
+
+        addresses = BindAddressValidator.validateBindAddresses(config, Collections.singletonList("http"));
+        assertEquals(Collections.singletonList(
+                new BindAddress("extra", URI.create("http://0.0.0.0:8080"))), addresses);
+    }
+}

pulsar-broker/src/main/java/org/apache/pulsar/broker/PulsarService.java

@@ -1407,7 +1407,7 @@ protected String brokerUrl(ServiceConfiguration config) {
     }
 
     public static String brokerUrl(String host, int port) {
-        return String.format("pulsar://%s:%d", host, port);
+        return ServiceConfigurationUtils.brokerUrl(host, port);
     }
 
     public String brokerUrlTls(ServiceConfiguration config) {
@@ -1420,7 +1420,7 @@ public String brokerUrlTls(ServiceConfiguration config) {
     }
 
     public static String brokerUrlTls(String host, int port) {
-        return String.format("pulsar+ssl://%s:%d", host, port);
+        return ServiceConfigurationUtils.brokerUrlTls(host, port);
     }
 
     public String webAddress(ServiceConfiguration config) {