From 63abfc2a642e24aba779d726eb00d6c3bf4427ec Mon Sep 17 00:00:00 2001 From: cvictory Date: Mon, 14 Jan 2019 18:21:21 +0800 Subject: [PATCH 1/2] just for modify comments and imports --- .../support/AbstractZookeeperTransporter.java | 156 ++++++++++++ .../AbstractZookeeperTransporterTest.java | 238 ++++++++++++++++++ 2 files changed, 394 insertions(+) create mode 100644 dubbo-remoting/dubbo-remoting-zookeeper/src/main/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporter.java create mode 100644 dubbo-remoting/dubbo-remoting-zookeeper/src/test/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporterTest.java diff --git a/dubbo-remoting/dubbo-remoting-zookeeper/src/main/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporter.java b/dubbo-remoting/dubbo-remoting-zookeeper/src/main/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporter.java new file mode 100644 index 00000000000..e5f4f1600ae --- /dev/null +++ b/dubbo-remoting/dubbo-remoting-zookeeper/src/main/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporter.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.remoting.zookeeper.support; + +import org.apache.dubbo.common.Constants; +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.logger.Logger; +import org.apache.dubbo.common.logger.LoggerFactory; +import org.apache.dubbo.remoting.zookeeper.ZookeeperClient; +import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * AbstractZookeeperTransporter is abstract implements of ZookeeperTransporter. + *

+ * If you want to extends this, implements createZookeeperClient. + */ +public abstract class AbstractZookeeperTransporter implements ZookeeperTransporter { + private static final Logger logger = LoggerFactory.getLogger(ZookeeperTransporter.class); + private final Map zookeeperClientMap = new ConcurrentHashMap<>(); + + /** + * share connnect for registry, metadata, etc.. + *

+ * Make sure the connection is connected. + * + * @param url + * @return + */ + public ZookeeperClient connect(URL url) { + ZookeeperClient zookeeperClient; + List addressList = getURLBackupAddress(url); + // The field define the zookeeper server , including protocol, host, port, username, password + if ((zookeeperClient = fetchAndUpdateZookeeperClientCache(addressList)) != null && zookeeperClient.isConnected()) { + logger.info("find valid zookeeper client from the cache for address: " + url); + return zookeeperClient; + } + // avoid creating too many connections， so add lock + synchronized (zookeeperClientMap) { + + if ((zookeeperClient = fetchAndUpdateZookeeperClientCache(addressList)) != null && zookeeperClient.isConnected()) { + logger.info("find valid zookeeper client from the cache for address: " + url); + return zookeeperClient; + } + + zookeeperClient = createZookeeperClient(toClientURL(url)); + logger.info("No valid zookeeper client found from cache, therefore create a new client for url. " + url); + writeToClientMap(addressList, zookeeperClient); + } + return zookeeperClient; + } + + /** + * @param url the url that will create zookeeper connection . + * The url in AbstractZookeeperTransporter#connect parameter is rewritten by this one. + * such as: zookeeper://127.0.0.1:2181/org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter + * @return + */ + protected abstract ZookeeperClient createZookeeperClient(URL url); + + /** + * get the ZookeeperClient from cache, the ZookeeperClient must be connected. + *

+ * It is not private method for unit test. + * + * @param addressList + * @return + */ + ZookeeperClient fetchAndUpdateZookeeperClientCache(List addressList) { + + ZookeeperClient zookeeperClient = null; + for (String address : addressList) { + if ((zookeeperClient = zookeeperClientMap.get(address)) != null && zookeeperClient.isConnected()) { + break; + } + } + if (zookeeperClient != null && zookeeperClient.isConnected()) { + writeToClientMap(addressList, zookeeperClient); + } + return zookeeperClient; + } + + /** + * get all zookeeper urls (such as :zookeeper://127.0.0.1:2181?127.0.0.1:8989,127.0.0.1:9999) + * + * @param url such as:zookeeper://127.0.0.1:2181?127.0.0.1:8989,127.0.0.1:9999 + * @return such as 127.0.0.1:2181,127.0.0.1:8989,127.0.0.1:9999 + */ + List getURLBackupAddress(URL url) { + List addressList = new ArrayList(); + addressList.add(url.getAddress()); + + addressList.addAll(url.getParameter(Constants.BACKUP_KEY, Collections.EMPTY_LIST)); + return addressList; + } + + /** + * write address-ZookeeperClient relationship to Map + * + * @param addressList + * @param zookeeperClient + */ + void writeToClientMap(List addressList, ZookeeperClient zookeeperClient) { + for (String address : addressList) { + zookeeperClientMap.put(address, zookeeperClient); + } + } + + /** + * redefine the url for zookeeper. just keep protocol, username, password, host, port, and individual parameter. + * + * @param url + * @return + */ + URL toClientURL(URL url) { + Map parameterMap = new HashMap<>(); + // for CuratorZookeeperClient + if (url.getParameter(Constants.TIMEOUT_KEY) != null) { + parameterMap.put(Constants.TIMEOUT_KEY, url.getParameter(Constants.TIMEOUT_KEY)); + } + if (url.getParameter(Constants.BACKUP_KEY) != null) { + parameterMap.put(Constants.BACKUP_KEY, url.getParameter(Constants.BACKUP_KEY)); + } + return new URL(url.getProtocol(), url.getUsername(), url.getPassword(), url.getHost(), url.getPort(), + ZookeeperTransporter.class.getName(), parameterMap); + } + + /** + * for unit test + * + * @return + */ + Map getZookeeperClientMap() { + return zookeeperClientMap; + } +} diff --git a/dubbo-remoting/dubbo-remoting-zookeeper/src/test/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporterTest.java b/dubbo-remoting/dubbo-remoting-zookeeper/src/test/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporterTest.java new file mode 100644 index 00000000000..50c186e6654 --- /dev/null +++ b/dubbo-remoting/dubbo-remoting-zookeeper/src/test/java/org/apache/dubbo/remoting/zookeeper/support/AbstractZookeeperTransporterTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dubbo.remoting.zookeeper.support; + +import org.apache.dubbo.common.Constants; +import org.apache.dubbo.common.URL; +import org.apache.dubbo.common.utils.NetUtils; +import org.apache.dubbo.remoting.zookeeper.ZookeeperClient; +import org.apache.dubbo.remoting.zookeeper.ZookeeperTransporter; +import org.apache.dubbo.remoting.zookeeper.curator.CuratorZookeeperTransporter; + +import org.apache.curator.test.TestingServer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsNot.not; +import static org.hamcrest.core.IsNull.nullValue; + +/** + * AbstractZookeeperTransporterTest + */ +public class AbstractZookeeperTransporterTest { + private TestingServer zkServer; + private ZookeeperClient zookeeperClient; + private AbstractZookeeperTransporter abstractZookeeperTransporter; + private int zkServerPort; + + @Before + public void setUp() throws Exception { + zkServerPort = NetUtils.getAvailablePort(); + zkServer = new TestingServer(zkServerPort, true); + zookeeperClient = new CuratorZookeeperTransporter().connect(URL.valueOf("zookeeper://127.0.0.1:" + + zkServerPort + "/service")); + abstractZookeeperTransporter = new CuratorZookeeperTransporter(); + } + + + @After + public void tearDown() throws Exception { + zkServer.stop(); + } + + @Test + public void testZookeeperClient() { + assertThat(zookeeperClient, not(nullValue())); + zookeeperClient.close(); + } + + @Test + public void testCreateServerURL() { + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828&timeout=2300"); + URL newUrl = abstractZookeeperTransporter.toClientURL(url); + Assert.assertEquals(newUrl.getProtocol(), "zookeeper"); + Assert.assertEquals(newUrl.getHost(), "127.0.0.1"); + Assert.assertEquals(newUrl.getPort(), zkServerPort); + Assert.assertNull(newUrl.getUsername()); + Assert.assertNull(newUrl.getPassword()); + Assert.assertEquals(newUrl.getParameter(Constants.TIMEOUT_KEY, 5000), 2300); + Assert.assertEquals(newUrl.getParameters().size(), 1); + Assert.assertEquals(newUrl.getPath(), ZookeeperTransporter.class.getName()); + } + + + @Test + public void testCreateServerURLWhenHasUser() { + URL url = URL.valueOf("zookeeper://us2:pw2@127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + URL newUrl = abstractZookeeperTransporter.toClientURL(url); + Assert.assertEquals(newUrl.getProtocol(), "zookeeper"); + Assert.assertEquals(newUrl.getHost(), "127.0.0.1"); + Assert.assertEquals(newUrl.getPort(), zkServerPort); + Assert.assertEquals(newUrl.getUsername(), "us2"); + Assert.assertEquals(newUrl.getPassword(), "pw2"); + Assert.assertEquals(newUrl.getParameters().size(), 0); + Assert.assertEquals(newUrl.getPath(), ZookeeperTransporter.class.getName()); + } + + @Test + public void testGetURLBackupAddress() { + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?backup=127.0.0.1:" + 9099 + "&application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + List stringList = abstractZookeeperTransporter.getURLBackupAddress(url); + Assert.assertEquals(stringList.size(), 2); + Assert.assertEquals(stringList.get(0), "127.0.0.1:" + zkServerPort); + Assert.assertEquals(stringList.get(1), "127.0.0.1:9099"); + } + + @Test + public void testGetURLBackupAddressNoBack() { + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + List stringList = abstractZookeeperTransporter.getURLBackupAddress(url); + Assert.assertEquals(stringList.size(), 1); + Assert.assertEquals(stringList.get(0), "127.0.0.1:" + zkServerPort); + } + + @Test + public void testFetchAndUpdateZookeeperClientCache() throws Exception { + int zkServerPort2 = NetUtils.getAvailablePort(); + TestingServer zkServer2 = new TestingServer(zkServerPort2, true); + + int zkServerPort3 = NetUtils.getAvailablePort(); + TestingServer zkServer3 = new TestingServer(zkServerPort3, true); + + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?backup=127.0.0.1:" + zkServerPort3 + ",127.0.0.1:" + zkServerPort2 + "&application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + ZookeeperClient newZookeeperClient = abstractZookeeperTransporter.connect(url); + //just for connected + newZookeeperClient.getContent("/dubbo/test"); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 3); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + + URL url2 = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.metadata.store.MetadataReport?address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + checkFetchAndUpdateCacheNotNull(url2); + URL url3 = URL.valueOf("zookeeper://127.0.0.1:8778/org.apache.dubbo.metadata.store.MetadataReport?backup=127.0.0.1:" + zkServerPort3 + "&address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + checkFetchAndUpdateCacheNotNull(url3); + + zkServer2.stop(); + zkServer3.stop(); + } + + private void checkFetchAndUpdateCacheNotNull(URL url) { + List addressList = abstractZookeeperTransporter.getURLBackupAddress(url); + ZookeeperClient zookeeperClient = abstractZookeeperTransporter.fetchAndUpdateZookeeperClientCache(addressList); + Assert.assertNotNull(zookeeperClient); + } + + @Test + public void testRepeatConnect() { + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + URL url2 = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.metadata.store.MetadataReport?address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + ZookeeperClient newZookeeperClient = abstractZookeeperTransporter.connect(url); + //just for connected + newZookeeperClient.getContent("/dubbo/test"); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 1); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + Assert.assertTrue(newZookeeperClient.isConnected()); + + ZookeeperClient newZookeeperClient2 = abstractZookeeperTransporter.connect(url2); + //just for connected + newZookeeperClient2.getContent("/dubbo/test"); + Assert.assertEquals(newZookeeperClient, newZookeeperClient2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 1); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + } + + @Test + public void testNotRepeatConnect() throws Exception { + int zkServerPort2 = NetUtils.getAvailablePort(); + TestingServer zkServer2 = new TestingServer(zkServerPort2, true); + + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + URL url2 = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort2 + "/org.apache.dubbo.metadata.store.MetadataReport?address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + ZookeeperClient newZookeeperClient = abstractZookeeperTransporter.connect(url); + //just for connected + newZookeeperClient.getContent("/dubbo/test"); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 1); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + + ZookeeperClient newZookeeperClient2 = abstractZookeeperTransporter.connect(url2); + //just for connected + newZookeeperClient2.getContent("/dubbo/test"); + Assert.assertNotEquals(newZookeeperClient, newZookeeperClient2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort2), newZookeeperClient2); + + zkServer2.stop(); + } + + @Test + public void testRepeatConnectForBackUpAdd() throws Exception { + int zkServerPort2 = NetUtils.getAvailablePort(); + TestingServer zkServer2 = new TestingServer(zkServerPort2, true); + + int zkServerPort3 = NetUtils.getAvailablePort(); + TestingServer zkServer3 = new TestingServer(zkServerPort3, true); + + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?backup=127.0.0.1:" + zkServerPort2 + "&application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + URL url2 = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort2 + "/org.apache.dubbo.metadata.store.MetadataReport?backup=127.0.0.1:" + zkServerPort3 + "&address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + ZookeeperClient newZookeeperClient = abstractZookeeperTransporter.connect(url); + //just for connected + newZookeeperClient.getContent("/dubbo/test"); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + + ZookeeperClient newZookeeperClient2 = abstractZookeeperTransporter.connect(url2); + //just for connected + newZookeeperClient2.getContent("/dubbo/test"); + Assert.assertEquals(newZookeeperClient, newZookeeperClient2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 3); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort2), newZookeeperClient2); + + zkServer2.stop(); + zkServer3.stop(); + } + + @Test + public void testRepeatConnectForNoMatchBackUpAdd() throws Exception { + int zkServerPort2 = NetUtils.getAvailablePort(); + TestingServer zkServer2 = new TestingServer(zkServerPort2, true); + + int zkServerPort3 = NetUtils.getAvailablePort(); + TestingServer zkServer3 = new TestingServer(zkServerPort3, true); + + URL url = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort + "/org.apache.dubbo.registry.RegistryService?backup=127.0.0.1:" + zkServerPort3 + "&application=metadatareport-local-xml-provider2&dubbo=2.0.2&interface=org.apache.dubbo.registry.RegistryService&pid=47418&specVersion=2.7.0-SNAPSHOT×tamp=1547102428828"); + URL url2 = URL.valueOf("zookeeper://127.0.0.1:" + zkServerPort2 + "/org.apache.dubbo.metadata.store.MetadataReport?address=zookeeper://127.0.0.1:2181&application=metadatareport-local-xml-provider2&cycle-report=false&interface=org.apache.dubbo.metadata.store.MetadataReport&retry-period=4590&retry-times=23&sync-report=true"); + ZookeeperClient newZookeeperClient = abstractZookeeperTransporter.connect(url); + //just for connected + newZookeeperClient.getContent("/dubbo/test"); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort), newZookeeperClient); + + ZookeeperClient newZookeeperClient2 = abstractZookeeperTransporter.connect(url2); + //just for connected + newZookeeperClient2.getContent("/dubbo/test"); + Assert.assertNotEquals(newZookeeperClient, newZookeeperClient2); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().size(), 3); + Assert.assertEquals(abstractZookeeperTransporter.getZookeeperClientMap().get("127.0.0.1:" + zkServerPort2), newZookeeperClient2); + + zkServer2.stop(); + zkServer3.stop(); + } +} From ef62505207f5f9f82d3dd94b36a49a58f8b6406f Mon Sep 17 00:00:00 2001 From: cvictory Date: Wed, 16 Jan 2019 15:14:44 +0800 Subject: [PATCH 2/2] remove gson from dubbo.jar in shading mode, add dependency --- dubbo-all/pom.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dubbo-all/pom.xml b/dubbo-all/pom.xml index 0c67ab9b58f..85949214467 100644 --- a/dubbo-all/pom.xml +++ b/dubbo-all/pom.xml @@ -395,6 +395,10 @@ io.netty netty-all + + com.google.code.gson + gson + @@ -483,7 +487,6 @@ org.apache.dubbo:dubbo-metadata-definition org.apache.dubbo:dubbo-metadata-report-redis org.apache.dubbo:dubbo-metadata-report-zookeeper - com.google.code.gson:gson