Merge remote-tracking branch 'apache/master' into congbo/fix/redelive…

…r-epoch-consume-stuck-problem
congbobo184 · Nov 20, 2024 · 3ee01db · 3ee01db
2 parents ddc7def + 895e968
commit 3ee01db
Show file tree

Hide file tree

Showing 34 changed files with 771 additions and 164 deletions.
diff --git a/buildtools/pom.xml b/buildtools/pom.xml
@@ -47,7 +47,7 @@
     <license-maven-plugin.version>4.1</license-maven-plugin.version>
     <puppycrawl.checkstyle.version>10.14.2</puppycrawl.checkstyle.version>
     <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
-    <netty.version>4.1.104.Final</netty.version>
+    <netty.version>4.1.115.Final</netty.version>
     <guice.version>4.2.3</guice.version>
     <guava.version>32.1.2-jre</guava.version>
     <ant.version>1.10.12</ant.version>

diff --git a/distribution/server/pom.xml b/distribution/server/pom.xml
@@ -49,7 +49,6 @@
       <groupId>${project.groupId}</groupId>
       <artifactId>jetcd-core-shaded</artifactId>
       <version>${project.version}</version>
-      <classifier>shaded</classifier>
     </dependency>
 
     <dependency>

diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt
@@ -293,33 +293,33 @@ The Apache Software License, Version 2.0
     - org.apache.commons-commons-lang3-3.11.jar
     - org.apache.commons-commons-text-1.10.0.jar
  * Netty
-    - io.netty-netty-buffer-4.1.113.Final.jar
-    - io.netty-netty-codec-4.1.113.Final.jar
-    - io.netty-netty-codec-dns-4.1.113.Final.jar
-    - io.netty-netty-codec-http-4.1.113.Final.jar
-    - io.netty-netty-codec-http2-4.1.113.Final.jar
-    - io.netty-netty-codec-socks-4.1.113.Final.jar
-    - io.netty-netty-codec-haproxy-4.1.113.Final.jar
-    - io.netty-netty-common-4.1.113.Final.jar
-    - io.netty-netty-handler-4.1.113.Final.jar
-    - io.netty-netty-handler-proxy-4.1.113.Final.jar
-    - io.netty-netty-resolver-4.1.113.Final.jar
-    - io.netty-netty-resolver-dns-4.1.113.Final.jar
-    - io.netty-netty-resolver-dns-classes-macos-4.1.113.Final.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
-    - io.netty-netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
-    - io.netty-netty-transport-4.1.113.Final.jar
-    - io.netty-netty-transport-classes-epoll-4.1.113.Final.jar
-    - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
-    - io.netty-netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
-    - io.netty-netty-transport-native-unix-common-4.1.113.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
-    - io.netty-netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
-    - io.netty-netty-tcnative-classes-2.0.66.Final.jar
+    - io.netty-netty-buffer-4.1.115.Final.jar
+    - io.netty-netty-codec-4.1.115.Final.jar
+    - io.netty-netty-codec-dns-4.1.115.Final.jar
+    - io.netty-netty-codec-http-4.1.115.Final.jar
+    - io.netty-netty-codec-http2-4.1.115.Final.jar
+    - io.netty-netty-codec-socks-4.1.115.Final.jar
+    - io.netty-netty-codec-haproxy-4.1.115.Final.jar
+    - io.netty-netty-common-4.1.115.Final.jar
+    - io.netty-netty-handler-4.1.115.Final.jar
+    - io.netty-netty-handler-proxy-4.1.115.Final.jar
+    - io.netty-netty-resolver-4.1.115.Final.jar
+    - io.netty-netty-resolver-dns-4.1.115.Final.jar
+    - io.netty-netty-resolver-dns-classes-macos-4.1.115.Final.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.115.Final-osx-aarch_64.jar
+    - io.netty-netty-resolver-dns-native-macos-4.1.115.Final-osx-x86_64.jar
+    - io.netty-netty-transport-4.1.115.Final.jar
+    - io.netty-netty-transport-classes-epoll-4.1.115.Final.jar
+    - io.netty-netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar
+    - io.netty-netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar
+    - io.netty-netty-transport-native-unix-common-4.1.115.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar
+    - io.netty-netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar
+    - io.netty-netty-tcnative-classes-2.0.69.Final.jar
     - io.netty.incubator-netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
     - io.netty.incubator-netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
@@ -464,9 +464,9 @@ The Apache Software License, Version 2.0
     - org.apache.avro-avro-1.11.4.jar
     - org.apache.avro-avro-protobuf-1.11.4.jar
   * Apache Curator
-    - org.apache.curator-curator-client-5.1.0.jar
-    - org.apache.curator-curator-framework-5.1.0.jar
-    - org.apache.curator-curator-recipes-5.1.0.jar
+    - org.apache.curator-curator-client-5.7.1.jar
+    - org.apache.curator-curator-framework-5.7.1.jar
+    - org.apache.curator-curator-recipes-5.7.1.jar
   * Apache Yetus
     - org.apache.yetus-audience-annotations-0.12.0.jar
   * Kubernetes Client
@@ -481,8 +481,8 @@ The Apache Software License, Version 2.0
   * Prometheus
     - io.prometheus-simpleclient_httpserver-0.16.0.jar
   * Oxia
-    - io.streamnative.oxia-oxia-client-api-0.4.7.jar
-    - io.streamnative.oxia-oxia-client-0.4.7.jar
+    - io.streamnative.oxia-oxia-client-api-0.4.9.jar
+    - io.streamnative.oxia-oxia-client-0.4.9.jar
   * OpenHFT
     - net.openhft-zero-allocation-hashing-0.16.jar
   * Java JSON WebTokens
@@ -498,9 +498,9 @@ The Apache Software License, Version 2.0
     - io.vertx-vertx-web-4.5.10.jar
     - io.vertx-vertx-web-common-4.5.10.jar
   * Apache ZooKeeper
-    - org.apache.zookeeper-zookeeper-3.9.2.jar
-    - org.apache.zookeeper-zookeeper-jute-3.9.2.jar
-    - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.2.jar
+    - org.apache.zookeeper-zookeeper-3.9.3.jar
+    - org.apache.zookeeper-zookeeper-jute-3.9.3.jar
+    - org.apache.zookeeper-zookeeper-prometheus-metrics-3.9.3.jar
   * Snappy Java
     - org.xerial.snappy-snappy-java-1.1.10.5.jar
   * Google HTTP Client

diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt
@@ -347,35 +347,35 @@ The Apache Software License, Version 2.0
     - commons-text-1.10.0.jar
     - commons-compress-1.26.0.jar
  * Netty
-    - netty-buffer-4.1.113.Final.jar
-    - netty-codec-4.1.113.Final.jar
-    - netty-codec-dns-4.1.113.Final.jar
-    - netty-codec-http-4.1.113.Final.jar
-    - netty-codec-socks-4.1.113.Final.jar
-    - netty-codec-haproxy-4.1.113.Final.jar
-    - netty-common-4.1.113.Final.jar
-    - netty-handler-4.1.113.Final.jar
-    - netty-handler-proxy-4.1.113.Final.jar
-    - netty-resolver-4.1.113.Final.jar
-    - netty-resolver-dns-4.1.113.Final.jar
-    - netty-transport-4.1.113.Final.jar
-    - netty-transport-classes-epoll-4.1.113.Final.jar
-    - netty-transport-native-epoll-4.1.113.Final-linux-aarch_64.jar
-    - netty-transport-native-epoll-4.1.113.Final-linux-x86_64.jar
-    - netty-transport-native-unix-common-4.1.113.Final.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final-linux-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final-linux-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final-osx-aarch_64.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final-osx-x86_64.jar
-    - netty-tcnative-boringssl-static-2.0.66.Final-windows-x86_64.jar
-    - netty-tcnative-classes-2.0.66.Final.jar
+    - netty-buffer-4.1.115.Final.jar
+    - netty-codec-4.1.115.Final.jar
+    - netty-codec-dns-4.1.115.Final.jar
+    - netty-codec-http-4.1.115.Final.jar
+    - netty-codec-socks-4.1.115.Final.jar
+    - netty-codec-haproxy-4.1.115.Final.jar
+    - netty-common-4.1.115.Final.jar
+    - netty-handler-4.1.115.Final.jar
+    - netty-handler-proxy-4.1.115.Final.jar
+    - netty-resolver-4.1.115.Final.jar
+    - netty-resolver-dns-4.1.115.Final.jar
+    - netty-transport-4.1.115.Final.jar
+    - netty-transport-classes-epoll-4.1.115.Final.jar
+    - netty-transport-native-epoll-4.1.115.Final-linux-aarch_64.jar
+    - netty-transport-native-epoll-4.1.115.Final-linux-x86_64.jar
+    - netty-transport-native-unix-common-4.1.115.Final.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final-linux-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final-linux-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final-osx-aarch_64.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final-osx-x86_64.jar
+    - netty-tcnative-boringssl-static-2.0.69.Final-windows-x86_64.jar
+    - netty-tcnative-classes-2.0.69.Final.jar
     - netty-incubator-transport-classes-io_uring-0.0.24.Final.jar
     - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-aarch_64.jar
     - netty-incubator-transport-native-io_uring-0.0.24.Final-linux-x86_64.jar
-    - netty-resolver-dns-classes-macos-4.1.113.Final.jar
-    - netty-resolver-dns-native-macos-4.1.113.Final-osx-aarch_64.jar
-    - netty-resolver-dns-native-macos-4.1.113.Final-osx-x86_64.jar
+    - netty-resolver-dns-classes-macos-4.1.115.Final.jar
+    - netty-resolver-dns-native-macos-4.1.115.Final-osx-aarch_64.jar
+    - netty-resolver-dns-native-macos-4.1.115.Final-osx-x86_64.jar
  * Prometheus client
     - simpleclient-0.16.0.jar
     - simpleclient_log4j2-0.16.0.jar

diff --git a/jetcd-core-shaded/pom.xml b/jetcd-core-shaded/pom.xml
@@ -74,6 +74,7 @@
     </dependency>
   </dependencies>
   <build>
+    <finalName>${project.artifactId}-${project.version}</finalName>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
@@ -141,54 +142,6 @@
                   <file>${project.basedir}/dependency-reduced-pom.xml</file>
                 </transformer>
               </transformers>
-              <!-- required for IntelliJ support -->
-              <shadedArtifactAttached>true</shadedArtifactAttached>
-              <shadedClassifierName>shaded</shadedClassifierName>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <!-- required for IntelliJ support, for some reason shadedArtifactAttached isn't sufficient alone -->
-      <plugin>
-        <groupId>org.codehaus.mojo</groupId>
-        <artifactId>build-helper-maven-plugin</artifactId>
-        <executions>
-          <execution>
-            <id>attach-shaded-jar</id>
-            <phase>package</phase>
-            <goals>
-              <goal>attach-artifact</goal>
-            </goals>
-            <configuration>
-              <artifacts>
-                <artifact>
-                  <file>${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar</file>
-                  <type>jar</type>
-                  <classifier>shaded</classifier>
-                </artifact>
-              </artifacts>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <!-- required for running tests in subproject -->
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <version>${maven-antrun-plugin.version}</version>
-        <executions>
-          <execution>
-            <id>unpack-shaded-jar</id>
-            <phase>package</phase>
-            <goals>
-              <goal>run</goal>
-            </goals>
-            <configuration>
-              <target>
-                <unzip src="${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar"
-                       dest="${project.build.outputDirectory}"
-                       overwrite="true" />
-              </target>
             </configuration>
           </execution>
         </executions>

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedCursorImpl.java
@@ -1569,7 +1569,7 @@ public Set<? extends Position> asyncReplayEntries(Set<? extends Position> positi
         Set<Position> alreadyAcknowledgedPositions = new HashSet<>();
         lock.readLock().lock();
         try {
-            positions.stream().filter(this::isMessageDeleted).forEach(alreadyAcknowledgedPositions::add);
+            positions.stream().filter(this::internalIsMessageDeleted).forEach(alreadyAcknowledgedPositions::add);
         } finally {
             lock.readLock().unlock();
         }
@@ -2345,7 +2345,7 @@ public void asyncDelete(Iterable<Position> positions, AsyncCallbacks.DeleteCallb
                     return;
                 }
 
-                if (isMessageDeleted(position)) {
+                if (internalIsMessageDeleted(position)) {
                     if (getConfig().isDeletionAtBatchIndexLevelEnabled()) {
                         BitSetRecyclable bitSetRecyclable = batchDeletedIndexes.remove(position);
                         if (bitSetRecyclable != null) {
@@ -3543,13 +3543,19 @@ public Position processIndividuallyDeletedMessagesAndGetMarkDeletedPosition(
     public boolean isMessageDeleted(Position position) {
         lock.readLock().lock();
         try {
-            return position.compareTo(markDeletePosition) <= 0
-                    || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+            return internalIsMessageDeleted(position);
         } finally {
             lock.readLock().unlock();
         }
     }
 
+    // When this method is called while the external has already acquired a write lock or a read lock,
+    // it avoids unnecessary lock nesting.
+    private boolean internalIsMessageDeleted(Position position) {
+        return position.compareTo(markDeletePosition) <= 0
+                || individualDeletedMessages.contains(position.getLedgerId(), position.getEntryId());
+    }
+
     //this method will return a copy of the position's ack set
     @Override
     public long[] getBatchPositionAckSet(Position position) {

diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java
@@ -3930,7 +3930,8 @@ private void checkManagedLedgerIsOpen() throws ManagedLedgerException {
         }
     }
 
-    synchronized void setFenced() {
+    @VisibleForTesting
+    public synchronized void setFenced() {
         log.info("{} Moving to Fenced state", name);
         STATE_UPDATER.set(this, State.Fenced);
     }