Update ZooKeeper to 3.6.2 and Curator to 5.1.0 (#8590)

### Motivation Upgrade to latest stable ZK version 3.6.2. The new minor versions brings several advantages: Performance improvements (eg: tuning group commit on txn log) Prometheus based metrics (so that we can get rid of AspectJ hacky way to instrument ZK) New features like persistent recursive watches which would greatly simplify the logic to handle metadata cache invalidations. The possibility of rollback to previous version has also been validated. ### Modifications - Update ZooKeeper to 3.6.2 - Updated Apache Curator to 5.1.0 (that supports ZK 3.6.x), Curator was implicitly imported by BookKeeper Storage Service ### Verifying this change - This change is already covered by existing tests, such as *(please describe tests)*. - Test manually a Pulsar service and Pulsar Standalone - Test ZooKeeper metrics ### Problems Unfortunately we cannot upgrade Curator because there is an error with "pulsar standalone": **Basically we have to upgrade ZooKeeper and Curator on BookKeeper before doing this change here** ``` 09:40:36.781 [main] ERROR org.apache.bookkeeper.common.component.AbstractLifecycleComponent - Failed to start Component: zk-storage-container-manager java.lang.NoSuchMethodError: 'org.apache.curator.framework.listen.ListenerContainer org.apache.curator.framework.recipes.cache.NodeCache.getListenable()' at org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterMetadataStore.watchClusterAssignmentData(ZkClusterMetadataStore.java:169) ~[org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar:4.12.0] at org.apache.bookkeeper.stream.storage.impl.sc.ZkStorageContainerManager.doStart(ZkStorageContainerManager.java:105) ~[org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar:4.12.0] at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at org.apache.bookkeeper.stream.storage.impl.StorageContainerStoreImpl.doStart(StorageContainerStoreImpl.java:97) [org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar:4.12.0] at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at org.apache.bookkeeper.stream.server.service.StorageService.doStart(StorageService.java:47) [org.apache.bookkeeper-stream-storage-server-4.12.0.jar:4.12.0] at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at org.apache.bookkeeper.common.component.LifecycleComponentStack.lambda$start$4(LifecycleComponentStack.java:144) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at com.google.common.collect.ImmutableList.forEach(ImmutableList.java:405) [com.google.guava-guava-30.0-jre.jar:?] at org.apache.bookkeeper.common.component.LifecycleComponentStack.start(LifecycleComponentStack.java:144) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at org.apache.bookkeeper.stream.server.StreamStorageLifecycleComponent.doStart(StreamStorageLifecycleComponent.java:51) [org.apache.bookkeeper-stream-storage-server-4.12.0.jar:4.12.0] at org.apache.bookkeeper.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:83) [org.apache.bookkeeper-bookkeeper-common-4.12.0.jar:4.12.0] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.runStreamStorage(LocalBookkeeperEnsemble.java:350) [org.apache.pulsar-pulsar-zookeeper-utils-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.startStandalone(LocalBookkeeperEnsemble.java:432) [org.apache.pulsar-pulsar-zookeeper-utils-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.pulsar.PulsarStandalone.start(PulsarStandalone.java:258) [org.apache.pulsar-pulsar-broker-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.pulsar.PulsarStandaloneStarter.main(PulsarStandaloneStarter.java:115) [org.apache.pulsar-pulsar-broker-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 09:40:36.782 [main] ERROR org.apache.bookkeeper.common.component.AbstractLifecycleComponent - Failed to start Component: range-service java.lang.NoSuchMethodError: 'org.apache.curator.framework.listen.ListenerContainer org.apache.curator.framework.recipes.cache.NodeCache.getListenable()' at org.apache.bookkeeper.stream.storage.impl.cluster.ZkClusterMetadataStore.watchClusterAssignmentData(ZkClusterMetadataStore.java:169) ~[org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar:4.12.0] at org.apache.bookkeeper.stream.storage.impl.sc.ZkStorageContainerManager.doStart(ZkStorageContainerManager.java:105) ~[org.apache.bookkeeper-stream-storage-service-impl-4.12.0.jar:4.12.0] ```
apache · Feb 11, 2021 · 9c39ca0 · 9c39ca0
1 parent f65b297
commit 9c39ca0
Show file tree

Hide file tree

Showing 19 changed files with 242 additions and 22 deletions.
diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -113,6 +113,26 @@
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-web</artifactId>
     </dependency>
+
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-core</artifactId>
+    </dependency>
+
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-graphite</artifactId>
+    </dependency>
+
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-jvm</artifactId>
+    </dependency>
+
+    <dependency>
+       <groupId>org.xerial.snappy</groupId>
+       <artifactId>snappy-java</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>com.fasterxml.jackson.dataformat</groupId>

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -494,9 +494,9 @@ The Apache Software License, Version 2.0
     - joda-time-2.10.1.jar
     - joda-time-joda-time-2.10.1.jar
   * Dropwizard
-    - io.dropwizard.metrics-metrics-core-3.1.0.jar
-    - io.dropwizard.metrics-metrics-graphite-3.1.0.jar
-    - io.dropwizard.metrics-metrics-jvm-3.1.0.jar
+    - io.dropwizard.metrics-metrics-core-3.2.5.jar
+    - io.dropwizard.metrics-metrics-graphite-3.2.5.jar
+    - io.dropwizard.metrics-metrics-jvm-3.2.5.jar
   * Prometheus
     - io.prometheus-simpleclient_httpserver-0.5.0.jar
   * Java JSON WebTokens
@@ -511,7 +511,9 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-core-3.5.3.jar
     - io.vertx-vertx-web-3.5.3.jar
   * Apache ZooKeeper
-    - org.apache.zookeeper-zookeeper-jute-3.5.7.jar
+    - org.apache.zookeeper-zookeeper-jute-3.6.2.jar
+  * Snappy Java  
+    - org.xerial.snappy-snappy-java-1.1.7.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library

diff --git a/managed-ledger/pom.xml b/managed-ledger/pom.xml
@@ -84,7 +84,16 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
-
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-core</artifactId>
+       <scope>test</scope>
+    </dependency>
+    <dependency>
+       <groupId>org.xerial.snappy</groupId>
+       <artifactId>snappy-java</artifactId>
+       <scope>test</scope>
+    </dependency>
     <!--
         junit is a runtime dependency of zookeeper tests, so only
         add this dependency here.

diff --git a/pom.xml b/pom.xml
@@ -96,7 +96,10 @@ flexible messaging model and an intuitive client API.</description>
     <commons-compress.version>1.19</commons-compress.version>
 
     <bookkeeper.version>4.12.1</bookkeeper.version>
-    <zookeeper.version>3.5.7</zookeeper.version>
+    <zookeeper.version>3.6.2</zookeeper.version>
+    <snappy.version>1.1.7</snappy.version> <!-- ZooKeeper server -->
+    <dropwizardmetrics.version>3.2.5</dropwizardmetrics.version> <!-- ZooKeeper server -->
+    <curator.version>5.1.0</curator.version>
     <netty.version>4.1.51.Final</netty.version>
     <netty-tc-native.version>2.0.33.Final</netty-tc-native.version>
     <jetty.version>9.4.35.v20201120</jetty.version>
@@ -296,7 +299,31 @@ flexible messaging model and an intuitive client API.</description>
           </exclusion>
         </exclusions>
       </dependency>
-
+      <dependency>
+        <groupId>org.apache.zookeeper</groupId>
+        <artifactId>zookeeper-jute</artifactId>
+        <version>${zookeeper.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.dropwizard.metrics</groupId>
+        <artifactId>metrics-core</artifactId>
+        <version>${dropwizardmetrics.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.dropwizard.metrics</groupId>
+        <artifactId>metrics-graphite</artifactId>
+        <version>${dropwizardmetrics.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>io.dropwizard.metrics</groupId>
+        <artifactId>metrics-jvm</artifactId>
+        <version>${dropwizardmetrics.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.xerial.snappy</groupId>
+        <artifactId>snappy-java</artifactId>
+        <version>${snappy.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.zookeeper</groupId>
         <artifactId>zookeeper</artifactId>
@@ -342,6 +369,18 @@ flexible messaging model and an intuitive client API.</description>
           </exclusion>
         </exclusions>
       </dependency>
+
+      <dependency>
+         <groupId>org.apache.curator</groupId>
+         <artifactId>curator-recipes</artifactId>
+         <version>${curator.version}</version>
+         <exclusions>
+             <exclusion>
+                 <groupId>org.apache.zookeeper</groupId>
+                 <artifactId>*</artifactId>
+             </exclusion>
+         </exclusions>
+      </dependency>
 
       <dependency>
         <groupId>org.apache.bookkeeper</groupId>

diff --git a/pulsar-broker/pom.xml b/pulsar-broker/pom.xml
@@ -133,13 +133,25 @@
       <version>${project.version}</version>
     </dependency>
 
+    <!-- zookeeper server -->
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-core</artifactId>
+    </dependency>
+
+    <!-- zookeeper server -->
+    <dependency>
+       <groupId>org.xerial.snappy</groupId>
+       <artifactId>snappy-java</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>pulsar-zookeeper-utils</artifactId>
       <version>${project.version}</version>
       <type>test-jar</type>
       <scope>test</scope>
-    </dependency>
+    </dependency>   
 
     <!-- functions related dependencies (begin) -->
 

diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java
@@ -1,3 +1,4 @@
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
@@ -33,7 +34,6 @@
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
-
 import org.apache.bookkeeper.mledger.LedgerOffloader;
 import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
 import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
@@ -298,6 +298,9 @@ public void testOffload(boolean isPartitioned) throws Exception {
         when(topicOffloader.getOffloadDriverName()).thenReturn("mock");
         doReturn(topicOffloader).when(pulsar).createManagedLedgerOffloader(any());
 
+        Awaitility.await().atMost(3, TimeUnit.SECONDS)
+                .until(() -> pulsar.getTopicPoliciesService().cacheIsInitialized(TopicName.get(topicName)));
+
         //4 set topic level offload policies
         admin.topics().setOffloadPolicies(topicName, offloadPolicies);
         Awaitility.await().atMost(10, TimeUnit.SECONDS).untilAsserted(()

diff --git a/pulsar-io/flume/pom.xml b/pulsar-io/flume/pom.xml
@@ -85,11 +85,30 @@
             <scope>test</scope>
             <version>1.9.0</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-framework</artifactId>
+            <version>${curator.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-test</artifactId>
-            <version>4.2.0</version>
+            <version>${curator.version}</version>
+            <scope>test</scope>
         </dependency>
+        <!-- zookeeper server -->
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <!-- zookeeper server -->
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.github.stefanbirkner</groupId>
             <artifactId>system-rules</artifactId>

diff --git a/.../src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java b/.../src/main/java/org/apache/pulsar/io/flume/node/PollingZooKeeperConfigurationProvider.java
@@ -23,7 +23,6 @@
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.recipes.cache.ChildData;
 import org.apache.curator.framework.recipes.cache.NodeCache;
-import org.apache.curator.framework.recipes.cache.NodeCacheListener;
 import org.apache.flume.FlumeException;
 import org.apache.flume.conf.FlumeConfiguration;
 import org.apache.flume.lifecycle.LifecycleAware;

diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml
@@ -43,6 +43,20 @@
       <groupId>org.apache.zookeeper</groupId>
       <artifactId>zookeeper</artifactId>
     </dependency>
+
+    <!-- zookeeper server -->
+    <dependency>
+      <groupId>io.dropwizard.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <!-- zookeeper server -->
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.bookkeeper</groupId>

diff --git a/pulsar-package-management/bookkeeper-storage/pom.xml b/pulsar-package-management/bookkeeper-storage/pom.xml
@@ -57,6 +57,20 @@
             <scope>test</scope>
         </dependency>
 
+        <!-- zookeeper server -->
+        <dependency>
+            <groupId>io.dropwizard.metrics</groupId>
+            <artifactId>metrics-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- zookeeper server -->
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>managed-ledger</artifactId>

diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE
@@ -397,9 +397,9 @@ The Apache Software License, Version 2.0
   * Objectsize
     - objectsize-0.0.12.jar
   * Dropwizard Metrics
-    - metrics-core-3.1.0.jar
-    - metrics-graphite-3.1.0.jar
-    - metrics-jvm-3.1.0.jar
+    - metrics-core-3.2.5.jar
+    - metrics-graphite-3.2.5.jar
+    - metrics-jvm-3.2.5.jar
   * Prometheus
     - simpleclient-0.5.0.jar
     - simpleclient_common-0.5.0.jar
@@ -435,7 +435,7 @@ The Apache Software License, Version 2.0
   * GSON
     - gson-2.8.6.jar
   * Snappy
-    - snappy-java-1.1.7.3.jar
+    - snappy-java-1.1.7.jar
   * Jackson
     - jackson-module-parameter-names-2.10.0.jar
     - jackson-module-parameter-names-2.11.1.jar
@@ -458,8 +458,8 @@ The Apache Software License, Version 2.0
     - memory-0.8.3.jar
     - sketches-core-0.8.3.jar
   * Apache Zookeeper
-    - zookeeper-3.5.7.jar
-    - zookeeper-jute-3.5.7.jar
+    - zookeeper-3.6.2.jar
+    - zookeeper-jute-3.6.2.jar
   * Apache Yetus Audience Annotations
     - audience-annotations-0.5.0.jar
   * Swagger

diff --git a/pulsar-testclient/pom.xml b/pulsar-testclient/pom.xml
@@ -44,7 +44,6 @@
 		<dependency>
 			<groupId>org.apache.zookeeper</groupId>
 			<artifactId>zookeeper</artifactId>
-			<version>${zookeeper.version}</version>
 		</dependency>
 
 		<dependency>

diff --git a/pulsar-zookeeper-utils/pom.xml b/pulsar-zookeeper-utils/pom.xml
@@ -52,9 +52,18 @@
           <groupId>io.grpc</groupId>
           <artifactId>grpc-all</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.curator</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
 
+    <dependency>
+         <groupId>org.apache.curator</groupId>
+         <artifactId>curator-recipes</artifactId>
+    </dependency>
+
     <!-- `grpc-all` is excluded from `stream-storage-server` at root pom file -->
     <dependency>
       <groupId>io.grpc</groupId>
@@ -78,6 +87,18 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+       <groupId>io.dropwizard.metrics</groupId>
+       <artifactId>metrics-core</artifactId>
+       <scope>test</scope>
+    </dependency>
+
+    <dependency>
+       <groupId>org.xerial.snappy</groupId>
+       <artifactId>snappy-java</artifactId>
+       <scope>test</scope>
+    </dependency>
+
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>testmocks</artifactId>

diff --git a/...ar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java b/...ar-zookeeper-utils/src/main/java/org/apache/pulsar/zookeeper/LocalBookkeeperEnsemble.java
@@ -43,6 +43,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 import java.util.stream.StreamSupport;
 
 import org.apache.bookkeeper.bookie.BookieException.InvalidCookieException;
@@ -204,7 +205,7 @@ private void runZookeeper(int maxCC) throws IOException {
             serverFactory.configure(new InetSocketAddress(zkPort), maxCC);
             serverFactory.startup(zks);
 
-            zkDataCleanupManager = new DatadirCleanupManager(zkDataDir, zkDataDir, 0, 1 /* hour */);
+            zkDataCleanupManager = new DatadirCleanupManager(zkDataDir, zkDataDir, 3, 1 /* hour */);
             zkDataCleanupManager.start();
         } catch (Exception e) {
             LOG.error("Exception while instantiating ZooKeeper", e);
@@ -227,8 +228,10 @@ private void runZookeeper(int maxCC) throws IOException {
     public void disconnectZookeeper(ZooKeeper zooKeeper) {
         ServerCnxn serverCnxn = getZookeeperServerConnection(zooKeeper);
         try {
-            Method method = serverCnxn.getClass().getMethod("close");
-            method.invoke(serverCnxn);
+            LOG.info("disconnect ZK server side connection {}", serverCnxn);
+            Class disconnectReasonClass = Class.forName("org.apache.zookeeper.server.ServerCnxn$DisconnectReason");
+            Method method = serverCnxn.getClass().getMethod("close", disconnectReasonClass);
+            method.invoke(serverCnxn, Stream.of(disconnectReasonClass.getEnumConstants()).filter(s->s.toString().equals("CONNECTION_CLOSE_FORCED")).findFirst().get());
         } catch (Exception ex) {
             throw new RuntimeException(ex);
         }