diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml index c839bc10b8bf6..fa7ab17dd216b 100644 --- a/distribution/server/pom.xml +++ b/distribution/server/pom.xml @@ -113,6 +113,26 @@ org.apache.logging.log4j log4j-web + + + io.dropwizard.metrics + metrics-core + + + + io.dropwizard.metrics + metrics-graphite + + + + io.dropwizard.metrics + metrics-jvm + + + + org.xerial.snappy + snappy-java + com.fasterxml.jackson.dataformat diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b5bd7de4a17bd..d83c427f41dde 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -494,9 +494,9 @@ The Apache Software License, Version 2.0 - joda-time-2.10.1.jar - joda-time-joda-time-2.10.1.jar * Dropwizard - - io.dropwizard.metrics-metrics-core-3.1.0.jar - - io.dropwizard.metrics-metrics-graphite-3.1.0.jar - - io.dropwizard.metrics-metrics-jvm-3.1.0.jar + - io.dropwizard.metrics-metrics-core-3.2.5.jar + - io.dropwizard.metrics-metrics-graphite-3.2.5.jar + - io.dropwizard.metrics-metrics-jvm-3.2.5.jar * Prometheus - io.prometheus-simpleclient_httpserver-0.5.0.jar * Java JSON WebTokens @@ -511,7 +511,9 @@ The Apache Software License, Version 2.0 - io.vertx-vertx-core-3.5.3.jar - io.vertx-vertx-web-3.5.3.jar * Apache ZooKeeper - - org.apache.zookeeper-zookeeper-jute-3.5.7.jar + - org.apache.zookeeper-zookeeper-jute-3.6.2.jar + * Snappy Java + - org.xerial.snappy-snappy-java-1.1.7.jar BSD 3-clause "New" or "Revised" License * Google auth library diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml index 935e2c0f88cef..3a692dda9e0be 100644 --- a/managed-ledger/pom.xml +++ b/managed-ledger/pom.xml @@ -84,7 +84,16 @@ tests test - + + io.dropwizard.metrics + metrics-core + test + + + org.xerial.snappy + snappy-java + test + + 3.2.5 + 5.1.0 4.1.51.Final 2.0.33.Final 9.4.35.v20201120 @@ -296,7 +299,31 @@ flexible messaging model and an intuitive client API. - + + org.apache.zookeeper + zookeeper-jute + ${zookeeper.version} + + + io.dropwizard.metrics + metrics-core + ${dropwizardmetrics.version} + + + io.dropwizard.metrics + metrics-graphite + ${dropwizardmetrics.version} + + + io.dropwizard.metrics + metrics-jvm + ${dropwizardmetrics.version} + + + org.xerial.snappy + snappy-java + ${snappy.version} + org.apache.zookeeper zookeeper @@ -342,6 +369,18 @@ flexible messaging model and an intuitive client API. + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.zookeeper + * + + + org.apache.bookkeeper diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml index db6ab77f51f82..06242e7fca3c9 100644 --- a/pulsar-broker/pom.xml +++ b/pulsar-broker/pom.xml @@ -133,13 +133,25 @@ ${project.version} + + + io.dropwizard.metrics + metrics-core + + + + + org.xerial.snappy + snappy-java + + ${project.groupId} pulsar-zookeeper-utils ${project.version} test-jar test - + diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java index 3dd170340df2a..9da1a47c47c9a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java @@ -1,3 +1,4 @@ + /** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -33,7 +34,6 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; - import org.apache.bookkeeper.mledger.LedgerOffloader; import org.apache.bookkeeper.mledger.ManagedLedgerInfo; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; @@ -298,6 +298,9 @@ public void testOffload(boolean isPartitioned) throws Exception { when(topicOffloader.getOffloadDriverName()).thenReturn("mock"); doReturn(topicOffloader).when(pulsar).createManagedLedgerOffloader(any()); + Awaitility.await().atMost(3, TimeUnit.SECONDS) + .until(() -> pulsar.getTopicPoliciesService().cacheIsInitialized(TopicName.get(topicName))); + //4 set topic level offload policies admin.topics().setOffloadPolicies(topicName, offloadPolicies); Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(() diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml index 16783c4c8c1e7..1e6830ab54579 100644 --- a/pulsar-io/flume/pom.xml +++ b/pulsar-io/flume/pom.xml @@ -85,11 +85,30 @@ test 1.9.0 + + org.apache.curator + curator-framework + ${curator.version} + org.apache.curator curator-test - 4.2.0 + ${curator.version} + test + + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + com.github.stefanbirkner system-rules diff --git a/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java b/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java index 775225c5abce7..a93a1a451e0f2 100644 --- a/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java +++ b/pulsar-io/flume/src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java @@ -23,7 +23,6 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.NodeCache; -import org.apache.curator.framework.recipes.cache.NodeCacheListener; import org.apache.flume.FlumeException; import org.apache.flume.conf.FlumeConfiguration; import org.apache.flume.lifecycle.LifecycleAware; diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml index b6bdf7411b111..b0ef1c1a741c1 100644 --- a/pulsar-metadata/pom.xml +++ b/pulsar-metadata/pom.xml @@ -43,6 +43,20 @@ org.apache.zookeeper zookeeper + + + + io.dropwizard.metrics + metrics-core + test + + + + + org.xerial.snappy + snappy-java + test + org.apache.bookkeeper diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml index 689ba621b1c06..67958d76f80ee 100644 --- a/pulsar-package-management/bookkeeper-storage/pom.xml +++ b/pulsar-package-management/bookkeeper-storage/pom.xml @@ -57,6 +57,20 @@ test + + + io.dropwizard.metrics + metrics-core + test + + + + + org.xerial.snappy + snappy-java + test + + ${project.groupId} managed-ledger diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 60c3875bc77a0..4b29804ef0ac6 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -397,9 +397,9 @@ The Apache Software License, Version 2.0 * Objectsize - objectsize-0.0.12.jar * Dropwizard Metrics - - metrics-core-3.1.0.jar - - metrics-graphite-3.1.0.jar - - metrics-jvm-3.1.0.jar + - metrics-core-3.2.5.jar + - metrics-graphite-3.2.5.jar + - metrics-jvm-3.2.5.jar * Prometheus - simpleclient-0.5.0.jar - simpleclient_common-0.5.0.jar @@ -435,7 +435,7 @@ The Apache Software License, Version 2.0 * GSON - gson-2.8.6.jar * Snappy - - snappy-java-1.1.7.3.jar + - snappy-java-1.1.7.jar * Jackson - jackson-module-parameter-names-2.10.0.jar - jackson-module-parameter-names-2.11.1.jar @@ -458,8 +458,8 @@ The Apache Software License, Version 2.0 - memory-0.8.3.jar - sketches-core-0.8.3.jar * Apache Zookeeper - - zookeeper-3.5.7.jar - - zookeeper-jute-3.5.7.jar + - zookeeper-3.6.2.jar + - zookeeper-jute-3.6.2.jar * Apache Yetus Audience Annotations - audience-annotations-0.5.0.jar * Swagger diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml index 8a1d34ec9818e..a244e15f94237 100644 --- a/pulsar-testclient/pom.xml +++ b/pulsar-testclient/pom.xml @@ -44,7 +44,6 @@ org.apache.zookeeper zookeeper - ${zookeeper.version} diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml index d792500dfe1ef..3172fd9b2074b 100644 --- a/pulsar-zookeeper-utils/pom.xml +++ b/pulsar-zookeeper-utils/pom.xml @@ -52,9 +52,18 @@ io.grpc grpc-all + + org.apache.curator + * + + + org.apache.curator + curator-recipes + + io.grpc @@ -78,6 +87,18 @@ test + + io.dropwizard.metrics + metrics-core + test + + + + org.xerial.snappy + snappy-java + test + + ${project.groupId} testmocks diff --git a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java index 3c479bfe8abdc..48618b866cf82 100644 --- a/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java +++ b/pulsar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java @@ -43,6 +43,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; +import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.bookkeeper.bookie.BookieException.InvalidCookieException; @@ -204,7 +205,7 @@ private void runZookeeper(int maxCC) throws IOException { serverFactory.configure(new InetSocketAddress(zkPort), maxCC); serverFactory.startup(zks); - zkDataCleanupManager = new DatadirCleanupManager(zkDataDir, zkDataDir, 0, 1 /* hour */); + zkDataCleanupManager = new DatadirCleanupManager(zkDataDir, zkDataDir, 3, 1 /* hour */); zkDataCleanupManager.start(); } catch (Exception e) { LOG.error("Exception while instantiating ZooKeeper", e); @@ -227,8 +228,10 @@ private void runZookeeper(int maxCC) throws IOException { public void disconnectZookeeper(ZooKeeper zooKeeper) { ServerCnxn serverCnxn = getZookeeperServerConnection(zooKeeper); try { - Method method = serverCnxn.getClass().getMethod("close"); - method.invoke(serverCnxn); + LOG.info("disconnect ZK server side connection {}", serverCnxn); + Class disconnectReasonClass = Class.forName("org.apache.zookeeper.server.ServerCnxn$DisconnectReason"); + Method method = serverCnxn.getClass().getMethod("close", disconnectReasonClass); + method.invoke(serverCnxn, Stream.of(disconnectReasonClass.getEnumConstants()).filter(s->s.toString().equals("CONNECTION_CLOSE_FORCED")).findFirst().get()); } catch (Exception ex) { throw new RuntimeException(ex); } diff --git a/pulsar-zookeeper/pom.xml b/pulsar-zookeeper/pom.xml index a846a76156600..3b2d5d48f0935 100644 --- a/pulsar-zookeeper/pom.xml +++ b/pulsar-zookeeper/pom.xml @@ -39,6 +39,20 @@ org.apache.zookeeper zookeeper + + + org.apache.zookeeper + zookeeper-jute + + + + io.dropwizard.metrics + metrics-core + + + org.xerial.snappy + snappy-java + ${project.groupId} @@ -93,6 +107,10 @@ org.apache.zookeeper zookeeper + + org.apache.zookeeper + zookeeper-jute + diff --git a/pulsar-zookeeper/src/main/resources/findbugsExclude.xml b/pulsar-zookeeper/src/main/resources/findbugsExclude.xml index 5a9926c0e8116..088436259f73f 100644 --- a/pulsar-zookeeper/src/main/resources/findbugsExclude.xml +++ b/pulsar-zookeeper/src/main/resources/findbugsExclude.xml @@ -21,6 +21,8 @@ - + + + diff --git a/src/findbugs-exclude.xml b/src/findbugs-exclude.xml index 78a1bd7f81d45..8f289b83a7be1 100644 --- a/src/findbugs-exclude.xml +++ b/src/findbugs-exclude.xml @@ -28,6 +28,9 @@ + + + diff --git a/testmocks/pom.xml b/testmocks/pom.xml index ca46c03365e91..55e5a49522723 100644 --- a/testmocks/pom.xml +++ b/testmocks/pom.xml @@ -58,6 +58,11 @@ org.objenesis objenesis + + + org.powermock + powermock-module-testng + diff --git a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java index 423bb905ebd1f..b791803d29d3b 100644 --- a/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java +++ b/testmocks/src/main/java/org/apache/zookeeper/MockZooKeeper.java @@ -25,6 +25,9 @@ import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; import io.netty.util.concurrent.DefaultThreadFactory; + +import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.Set; @@ -38,6 +41,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiPredicate; + import org.apache.commons.lang3.tuple.Pair; import org.apache.zookeeper.AsyncCallback.Children2Callback; import org.apache.zookeeper.AsyncCallback.ChildrenCallback; @@ -47,11 +51,13 @@ import org.apache.zookeeper.AsyncCallback.VoidCallback; import org.apache.zookeeper.Watcher.Event.EventType; import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.apache.zookeeper.client.HostProvider; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; import org.objenesis.Objenesis; import org.objenesis.ObjenesisStd; import org.objenesis.instantiator.ObjectInstantiator; +import org.powermock.reflect.Whitebox; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,6 +131,8 @@ public static MockZooKeeper newInstance(ExecutorService executor, int readOpDela zk.init(executor); zk.readOpDelayMs = readOpDelayMs; zk.mutex = new ReentrantLock(); + ObjectInstantiator clientCnxnObjectInstantiator = objenesis.getInstantiatorOf(ClientCnxn.class); + Whitebox.setInternalState(zk, "cnxn", clientCnxnObjectInstantiator.newInstance()); zk.sequentialIdGenerator = new AtomicLong(); return zk; } catch (RuntimeException e) { @@ -882,6 +890,36 @@ public void delete(final String path, int version, final VoidCallback cb, final } + @Override + public void multi(Iterable ops, AsyncCallback.MultiCallback cb, Object ctx) { + try { + List res = multi(ops); + cb.processResult(KeeperException.Code.OK.intValue(), (String)null, ctx, res); + } catch (Exception e) { + cb.processResult(KeeperException.Code.APIERROR.intValue(), (String)null, ctx, null); + } + } + + @Override + public List multi(Iterable ops) throws InterruptedException, KeeperException { + List res = new ArrayList<>(); + for (org.apache.zookeeper.Op op : ops) { + switch (op.getType()) { + case ZooDefs.OpCode.create: + this.create(op.getPath(), ((org.apache.zookeeper.Op.Create)op).data, null, null); + res.add(new OpResult.CreateResult(op.getPath())); + case ZooDefs.OpCode.delete: + this.delete(op.getPath(), -1); + res.add(new OpResult.DeleteResult()); + case ZooDefs.OpCode.setData: + this.create(op.getPath(), ((org.apache.zookeeper.Op.Create)op).data, null, null); + res.add(new OpResult.SetDataResult(null)); + default: + } + } + return res; + } + @Override public void close() throws InterruptedException { }