From f58884c818e726eafa4b7e4c874843bd53d02fcb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 5 Jul 2024 21:54:23 +0300 Subject: [PATCH 1/2] [improve][broker] Use RoaringBitmap for tracking individual acks --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 ++ .../bookkeeper/mledger/impl/RangeSetWrapper.java | 2 +- pom.xml | 2 +- pulsar-common/pom.xml | 5 +++++ .../util/collections/OpenLongPairRangeSet.java | 15 ++------------- 6 files changed, 12 insertions(+), 16 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index cfbe991a8edd8..24c601b184afa 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -513,7 +513,7 @@ The Apache Software License, Version 2.0 * RxJava - io.reactivex.rxjava3-rxjava-3.0.1.jar * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-1.0.6.jar + - org.roaringbitmap-RoaringBitmap-1.1.0.jar * OpenTelemetry - io.opentelemetry-opentelemetry-api-1.38.0.jar - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 0da56c6afa8fc..2971147c2c8df 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -382,6 +382,8 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_common-0.16.0.jar - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar + * RoaringBitmap + - RoaringBitmap-1.1.0.jar * Log4J - log4j-api-2.23.1.jar - log4j-core-2.23.1.jar diff --git a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java index 299fd3dc74cb4..c193d71c64f7d 100644 --- a/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java +++ b/managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/RangeSetWrapper.java @@ -55,7 +55,7 @@ public RangeSetWrapper(LongPairConsumer rangeConverter, this.config = managedCursor.getManagedLedger().getConfig(); this.rangeConverter = rangeConverter; this.rangeSet = config.isUnackedRangesOpenCacheSetEnabled() - ? new OpenLongPairRangeSet<>(4096, rangeConverter) + ? new OpenLongPairRangeSet<>(rangeConverter) : new LongPairRangeSet.DefaultRangeSet<>(rangeConverter, rangeBoundConsumer); this.enableMultiEntry = config.isPersistentUnackedRangesWithMultipleEntriesEnabled(); } diff --git a/pom.xml b/pom.xml index d42eac2d5af59..0f7816521189d 100644 --- a/pom.xml +++ b/pom.xml @@ -317,7 +317,7 @@ flexible messaging model and an intuitive client API. 1.3 0.4 10.0.1 - 1.0.6 + 1.1.0 1.6.1 6.4.0 3.33.0 diff --git a/pulsar-common/pom.xml b/pulsar-common/pom.xml index aa7e4998e5c3e..3f73a43698ea4 100644 --- a/pulsar-common/pom.xml +++ b/pulsar-common/pom.xml @@ -252,6 +252,11 @@ awaitility test + + + org.roaringbitmap + RoaringBitmap + diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java index c053c106be206..5114675324ad7 100644 --- a/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/collections/OpenLongPairRangeSet.java @@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.concurrent.NotThreadSafe; import org.apache.commons.lang.mutable.MutableInt; +import org.roaringbitmap.RoaringBitSet; /** * A Concurrent set comprising zero or more ranges of type {@link LongPair}. This can be alternative of @@ -46,8 +47,6 @@ public class OpenLongPairRangeSet> implements LongPairRangeSet { protected final NavigableMap rangeBitSetMap = new ConcurrentSkipListMap<>(); - private boolean threadSafe = true; - private final int bitSetSize; private final LongPairConsumer consumer; // caching place-holder for cpu-optimization to avoid calculating ranges again @@ -57,16 +56,6 @@ public class OpenLongPairRangeSet> implements LongPairRa private volatile boolean updatedAfterCachedForToString = true; public OpenLongPairRangeSet(LongPairConsumer consumer) { - this(1024, true, consumer); - } - - public OpenLongPairRangeSet(int size, LongPairConsumer consumer) { - this(size, true, consumer); - } - - public OpenLongPairRangeSet(int size, boolean threadSafe, LongPairConsumer consumer) { - this.threadSafe = threadSafe; - this.bitSetSize = size; this.consumer = consumer; } @@ -416,7 +405,7 @@ private int getSafeEntry(long value) { } private BitSet createNewBitSet() { - return this.threadSafe ? new ConcurrentBitSet(bitSetSize) : new BitSet(bitSetSize); + return new RoaringBitSet(); } } From ea38d93b84a205c7a8e6dd1928f1d7a2961ba357 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Sat, 6 Jul 2024 10:16:27 +0300 Subject: [PATCH 2/2] Upgrade RoaringBitmap to 1.2.0 --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- distribution/shell/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 24c601b184afa..f46b18347c1eb 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -513,7 +513,7 @@ The Apache Software License, Version 2.0 * RxJava - io.reactivex.rxjava3-rxjava-3.0.1.jar * RoaringBitmap - - org.roaringbitmap-RoaringBitmap-1.1.0.jar + - org.roaringbitmap-RoaringBitmap-1.2.0.jar * OpenTelemetry - io.opentelemetry-opentelemetry-api-1.38.0.jar - io.opentelemetry-opentelemetry-api-incubator-1.38.0-alpha.jar diff --git a/distribution/shell/src/assemble/LICENSE.bin.txt b/distribution/shell/src/assemble/LICENSE.bin.txt index 2971147c2c8df..261fef74a102c 100644 --- a/distribution/shell/src/assemble/LICENSE.bin.txt +++ b/distribution/shell/src/assemble/LICENSE.bin.txt @@ -383,7 +383,7 @@ The Apache Software License, Version 2.0 - simpleclient_tracer_otel-0.16.0.jar - simpleclient_tracer_otel_agent-0.16.0.jar * RoaringBitmap - - RoaringBitmap-1.1.0.jar + - RoaringBitmap-1.2.0.jar * Log4J - log4j-api-2.23.1.jar - log4j-core-2.23.1.jar diff --git a/pom.xml b/pom.xml index 0f7816521189d..7767a1f626a95 100644 --- a/pom.xml +++ b/pom.xml @@ -317,7 +317,7 @@ flexible messaging model and an intuitive client API. 1.3 0.4 10.0.1 - 1.1.0 + 1.2.0 1.6.1 6.4.0 3.33.0