From dcfd40d6b4e44be65ec0e7c5d97ff4e2b8950dd4 Mon Sep 17 00:00:00 2001 From: Dave Fisher Date: Thu, 13 Oct 2022 09:23:15 -0700 Subject: [PATCH 01/32] [improve][misc] Highlight change to threading (#18025) * [improve][misc] Highlight change to threading Whenever a PR changes the threading model we need to make clear that we should track these changes * Fix typo --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1b98475838087..108e742a7f472 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -56,6 +56,7 @@ This change added tests and can be verified as follows: - [ ] The public API - [ ] The schema - [ ] The default values of configurations +- [ ] The threading model - [ ] The binary protocol - [ ] The REST endpoints - [ ] The admin CLI options From e365afe5aa647204c8b9f5862d8fadb03bc40760 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 14 Oct 2022 09:06:32 +0800 Subject: [PATCH 02/32] [fix][test] flaky AdminApi2Test.cleanup (#17861) --- .../auth/MockedPulsarServiceBaseTest.java | 5 +++-- .../client/api/NonPersistentTopicTest.java | 20 ------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index 62fd69b3811d6..ace2e8a18f9d9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -80,6 +80,7 @@ import org.apache.pulsar.tests.TestRetrySupport; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.MockZooKeeper; +import org.apache.zookeeper.MockZooKeeperSession; import org.apache.zookeeper.data.ACL; import org.mockito.Mockito; import org.mockito.internal.util.MockUtil; @@ -400,7 +401,7 @@ protected MetadataStoreExtended createLocalMetadataStore(PulsarMetadataEventSync } protected MetadataStoreExtended createLocalMetadataStore() throws MetadataStoreException { - return new ZKMetadataStore(mockZooKeeper, MetadataStoreConfig.builder() + return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeper), MetadataStoreConfig.builder() .metadataStoreName(MetadataStoreConfig.METADATA_STORE).build()); } @@ -413,7 +414,7 @@ protected MetadataStoreExtended createConfigurationMetadataStore(PulsarMetadataE } protected MetadataStoreExtended createConfigurationMetadataStore() throws MetadataStoreException { - return new ZKMetadataStore(mockZooKeeperGlobal, MetadataStoreConfig.builder() + return new ZKMetadataStore(MockZooKeeperSession.newInstance(mockZooKeeperGlobal), MetadataStoreConfig.builder() .metadataStoreName(MetadataStoreConfig.CONFIGURATION_METADATA_STORE).build()); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java index 677cca30c7ba2..d1b536a679689 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/NonPersistentTopicTest.java @@ -27,7 +27,6 @@ import static org.testng.Assert.assertTrue; import static org.testng.Assert.fail; import com.google.common.collect.Sets; -import java.lang.reflect.Field; import java.net.URL; import java.util.HashSet; import java.util.Optional; @@ -39,7 +38,6 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; import lombok.Cleanup; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; @@ -700,15 +698,6 @@ public void testLoadManagerAssignmentForNonPersistentTestAssignment(String loadM conf.setLoadManagerClassName(loadManagerName); startBroker(); - Field field = PulsarService.class.getDeclaredField("loadManager"); - field.setAccessible(true); - @SuppressWarnings("unchecked") - AtomicReference loadManagerRef = (AtomicReference) field.get(pulsar); - LoadManager manager = LoadManager.create(pulsar); - manager.start(); - LoadManager oldLoadManager = loadManagerRef.getAndSet(manager); - oldLoadManager.stop(); - NamespaceBundle fdqn = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); LoadManager loadManager = pulsar.getLoadManager().get(); ResourceUnit broker = null; @@ -789,15 +778,6 @@ public void testNonPersistentBrokerModeRejectPersistentTopic(String loadManagerN conf.setLoadManagerClassName(loadManagerName); startBroker(); - Field field = PulsarService.class.getDeclaredField("loadManager"); - field.setAccessible(true); - @SuppressWarnings("unchecked") - AtomicReference loadManagerRef = (AtomicReference) field.get(pulsar); - LoadManager manager = LoadManager.create(pulsar); - manager.start(); - LoadManager oldLoadManager = loadManagerRef.getAndSet(manager); - oldLoadManager.stop(); - NamespaceBundle fdqn = pulsar.getNamespaceService().getBundle(TopicName.get(topicName)); LoadManager loadManager = pulsar.getLoadManager().get(); ResourceUnit broker = null; From 1ecb9c70b3762fa1450e6e40bf1f97863c8a9e7b Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 14 Oct 2022 09:17:26 +0800 Subject: [PATCH 03/32] [improve][test] Improve KeySharedSubscriptionTest to reduce the execution time (#18027) --- .../client/api/KeySharedSubscriptionTest.java | 63 ++++++++++++++----- 1 file changed, 47 insertions(+), 16 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java index e2198deb294ab..0917eb7764fab 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/KeySharedSubscriptionTest.java @@ -44,6 +44,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import lombok.Cleanup; @@ -63,8 +64,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -109,7 +111,7 @@ public Object[][] topicDomainProvider() { }; } - @BeforeMethod(alwaysRun = true) + @BeforeClass(alwaysRun = true) @Override protected void setup() throws Exception { super.internalSetup(); @@ -117,12 +119,22 @@ protected void setup() throws Exception { this.conf.setSubscriptionKeySharedUseConsistentHashing(true); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); } + @AfterMethod(alwaysRun = true) + public void resetDefaultNamespace() throws Exception { + List list = admin.namespaces().getTopics("public/default"); + for (String topicName : list){ + if (!pulsar.getBrokerService().isSystemTopic(topicName)) { + admin.topics().delete(topicName, false); + } + } + } + private static final Random random = new Random(System.nanoTime()); private static final int NUMBER_OF_KEYS = 300; @@ -441,12 +453,18 @@ public void testOrderingKeyWithHashRangeExclusiveStickyKeyConsumerSelector(boole public void testDisableKeySharedSubscription() throws PulsarClientException { this.conf.getSubscriptionTypesEnabled().remove("Key_Shared"); String topic = "persistent://public/default/key_shared_disabled"; - pulsarClient.newConsumer() - .topic(topic) - .subscriptionName("key_shared") - .subscriptionType(SubscriptionType.Key_Shared) - .ackTimeout(10, TimeUnit.SECONDS) - .subscribe(); + try { + @Cleanup + Consumer c = pulsarClient.newConsumer() + .topic(topic) + .subscriptionName("key_shared") + .subscriptionType(SubscriptionType.Key_Shared) + .ackTimeout(10, TimeUnit.SECONDS) + .subscribe(); + } finally { + // reset subscription types. + this.conf.getSubscriptionTypesEnabled().add("Key_Shared"); + } } @Test @@ -481,6 +499,7 @@ public void testMakingProgressWithSlowerConsumer(boolean enableBatch) throws Exc String slowKey = "slowKey"; List clients = new ArrayList<>(); + List consumers = new ArrayList<>(); try { AtomicInteger receivedMessages = new AtomicInteger(); @@ -490,7 +509,7 @@ public void testMakingProgressWithSlowerConsumer(boolean enableBatch) throws Exc .build(); clients.add(client); - client.newConsumer(Schema.INT32) + Consumer c = client.newConsumer(Schema.INT32) .topic(topic) .subscriptionName("key_shared") .subscriptionType(SubscriptionType.Key_Shared) @@ -509,6 +528,7 @@ public void testMakingProgressWithSlowerConsumer(boolean enableBatch) throws Exc } }) .subscribe(); + consumers.add(c); } @Cleanup @@ -535,6 +555,10 @@ public void testMakingProgressWithSlowerConsumer(boolean enableBatch) throws Exc Awaitility.await().untilAsserted(() -> { assertEquals((double) receivedMessages.get(), N * 0.9, N * 0.3); }); + + for (Consumer c : consumers) { + c.close(); + } } finally { for (PulsarClient c : clients) { c.close(); @@ -1108,6 +1132,7 @@ public void testCheckConsumersWithSameName() throws Exception { .subscriptionType(SubscriptionType.Key_Shared); // Create 3 consumers with same name + @Cleanup Consumer c1 = cb.subscribe(); @Cleanup @@ -1115,6 +1140,7 @@ public void testCheckConsumersWithSameName() throws Exception { @Cleanup Consumer c3 = cb.subscribe(); + @Cleanup Producer p = pulsarClient.newProducer(Schema.STRING) .topic(topicName) .create(); @@ -1398,7 +1424,7 @@ public EncryptionKeyInfo getPrivateKey(String keyName, Map keyMe } @Test - public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, InterruptedException { + public void testStickyKeyRangesRestartConsumers() throws Exception { final String topic = TopicName.get("persistent", "public", "default", "testStickyKeyRangesRestartConsumers" + UUID.randomUUID()).toString(); @@ -1411,6 +1437,7 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, CountDownLatch count1 = new CountDownLatch(2); CountDownLatch count2 = new CountDownLatch(13); // consumer 2 usually receive the fix messages CountDownLatch count3 = new CountDownLatch(numMessages); + @Cleanup Consumer consumer1 = pulsarClient.newConsumer( Schema.STRING) .topic(topic) @@ -1431,6 +1458,7 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, }) .subscribe(); + @Cleanup Consumer consumer2 = pulsarClient.newConsumer( Schema.STRING) .topic(topic) @@ -1451,13 +1479,13 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, }) .subscribe(); - pulsar.getExecutor().submit(() -> { + Future producerFuture = pulsar.getExecutor().submit(() -> { try { try (Producer producer = pulsarClient.newProducer(Schema.STRING) .topic(topic) .enableBatching(false) - .create();) { + .create()) { for (int i = 0; i < numMessages; i++) { String key = "test" + i; @@ -1484,8 +1512,8 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, Thread.sleep(3000); // start consuming again... - - pulsarClient.newConsumer(Schema.STRING) + @Cleanup + Consumer consumer3 = pulsarClient.newConsumer(Schema.STRING) .topic(topic) .subscriptionName(subscriptionName) .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) @@ -1502,7 +1530,8 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, }); }) .subscribe(); - pulsarClient.newConsumer(Schema.STRING) + @Cleanup + Consumer consumer4 = pulsarClient.newConsumer(Schema.STRING) .topic(topic) .subscriptionName(subscriptionName) .subscriptionInitialPosition(SubscriptionInitialPosition.Earliest) @@ -1522,5 +1551,7 @@ public void testStickyKeyRangesRestartConsumers() throws PulsarClientException, // wait for all the messages to be delivered count3.await(); assertTrue(sentMessages.isEmpty(), "didn't receive " + sentMessages); + + producerFuture.get(); } } From 230e7dcc16c26ad1f9e8336871f9bb19b8fe286f Mon Sep 17 00:00:00 2001 From: HuangZeGui Date: Fri, 14 Oct 2022 09:20:38 +0800 Subject: [PATCH 04/32] [fix][broker] Update the log print content of createSubscriptions (#18024) Co-authored-by: huangzegui --- .../apache/pulsar/broker/admin/impl/PersistentTopicsBase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java index 86b242c949760..a00d90e92162d 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/PersistentTopicsBase.java @@ -4524,7 +4524,7 @@ private CompletableFuture createSubscriptions(TopicName topicName, int num }); FutureUtil.waitForAll(subscriptionFutures).thenRun(() -> { - log.info("[{}] Successfully created new partitions {}", clientAppId(), topicName); + log.info("[{}] Successfully created subscriptions on new partitions {}", clientAppId(), topicName); result.complete(null); }).exceptionally(ex -> { log.warn("[{}] Failed to create subscriptions on new partitions for {}", From bcf94b19ac9a5e4579bd6e947b3bf7258265f755 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 14 Oct 2022 09:23:22 +0800 Subject: [PATCH 05/32] [improve][test] Improve SimpleSchemaTest to reduce the execution time (#18032) --- .../pulsar/client/api/SimpleSchemaTest.java | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java index cb8b8728ceee8..244ccab1a35d1 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SimpleSchemaTest.java @@ -61,8 +61,9 @@ import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Factory; import org.testng.annotations.Test; @@ -71,6 +72,8 @@ @Slf4j public class SimpleSchemaTest extends ProducerConsumerBase { + private static final String NAMESPACE = "my-property/my-ns"; + @DataProvider(name = "batchingModes") public static Object[][] batchingModes() { return new Object[][] { @@ -111,7 +114,7 @@ public SimpleSchemaTest(boolean schemaValidationEnforced) { } - @BeforeMethod + @BeforeClass @Override protected void setup() throws Exception { conf.setSchemaValidationEnforced(schemaValidationEnforced); @@ -120,19 +123,32 @@ protected void setup() throws Exception { super.producerBaseSetup(); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); } + @AfterMethod(alwaysRun = true) + public void resetNamespace() throws Exception { + List list = admin.namespaces().getTopics(NAMESPACE); + for (String topicName : list){ + if (!pulsar.getBrokerService().isSystemTopic(topicName)) { + admin.topics().delete(topicName, false); + } + } + PulsarClientImpl pulsarClientImpl = (PulsarClientImpl) pulsarClient; + pulsarClientImpl.getSchemaProviderLoadingCache().invalidateAll(); + } + @Test public void testString() throws Exception { + final String topicName = String.format("persistent://%s/my-topic1", NAMESPACE); try (Consumer consumer = pulsarClient.newConsumer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic1") + .topic(topicName) .subscriptionName("my-subscriber-name").subscribe(); Producer producer = pulsarClient.newProducer(Schema.STRING) - .topic("persistent://my-property/my-ns/my-topic1").create()) { + .topic(topicName).create()) { int N = 10; for (int i = 0; i < N; i++) { @@ -176,7 +192,7 @@ static class IncompatibleData { @Test public void newProducerNewTopicNewSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic).create()) { p.send(new V1Data(0)); @@ -185,7 +201,7 @@ public void newProducerNewTopicNewSchema() throws Exception { @Test public void newProducerTopicExistsWithoutSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer().topic(topic).create()) { p.send(topic.getBytes(UTF_8)); } @@ -198,7 +214,7 @@ public void newProducerTopicExistsWithoutSchema() throws Exception { @Test public void newProducerTopicExistsWithSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic).create()) { p.send(new V1Data(1)); @@ -212,7 +228,7 @@ public void newProducerTopicExistsWithSchema() throws Exception { @Test public void newProducerWithoutSchemaOnTopicWithSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic).create()) { @@ -256,7 +272,7 @@ public void newProducerWithoutSchemaOnTopicWithSchema() throws Exception { @Test public void newProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; Schema v1Schema = Schema.AVRO(V1Data.class); byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema(); AvroWriter v1Writer = new AvroWriter<>( @@ -323,7 +339,7 @@ public void newProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Ex @Test public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; Schema v1Schema = Schema.AVRO(V1Data.class); byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema(); org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes)); @@ -390,7 +406,7 @@ public void newNativeAvroProducerForMessageSchemaOnTopicWithMultiVersionSchema() @Test public void newProducerForMessageOnTopicWithDifferentSchemaType() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; V1Data data1 = new V1Data(2); V2Data data2 = new V2Data(3, 5); V1Data data3 = new V1Data(8); @@ -422,7 +438,7 @@ public void newProducerForMessageOnTopicWithDifferentSchemaType() throws Excepti @Test public void newProducerForMessageSchemaOnTopicInitialWithNoSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; Schema v1Schema = Schema.AVRO(V1Data.class); byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema(); AvroWriter v1Writer = new AvroWriter<>( @@ -462,7 +478,7 @@ public void newProducerForMessageSchemaOnTopicInitialWithNoSchema() throws Excep @Test public void newNativeAvroProducerForMessageSchemaOnTopicInitialWithNoSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; Schema v1Schema = Schema.AVRO(V1Data.class); byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema(); org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes)); @@ -503,10 +519,12 @@ public void newNativeAvroProducerForMessageSchemaOnTopicInitialWithNoSchema() th @Test public void newProducerForMessageSchemaWithBatch() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; + @Cleanup Consumer c = pulsarClient.newConsumer(Schema.AVRO(V2Data.class)) .topic(topic) .subscriptionName("sub1").subscribe(); + @Cleanup Producer p = pulsarClient.newProducer(Schema.AUTO_PRODUCE_BYTES()) .topic(topic) .enableBatching(true) @@ -535,6 +553,7 @@ public void newProducerForMessageSchemaWithBatch() throws Exception { try { p.newMessage(Schema.AUTO_PRODUCE_BYTES(Schema.AVRO(IncompatibleData.class))) .value(content).send(); + fail("Expect: incompatible schema exception"); } catch (Exception e) { Assert.assertTrue(e instanceof IncompatibleSchemaException, e.getMessage()); } @@ -556,7 +575,7 @@ public void newProducerForMessageSchemaWithBatch() throws Exception { @Test public void newNativeAvroProducerForMessageSchemaWithBatch() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; Schema v1Schema = Schema.AVRO(V1Data.class); byte[] v1SchemaBytes = v1Schema.getSchemaInfo().getSchema(); org.apache.avro.Schema v1SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v1SchemaBytes)); @@ -566,9 +585,11 @@ public void newNativeAvroProducerForMessageSchemaWithBatch() throws Exception { org.apache.avro.Schema v2SchemaAvroNative = new Parser().parse(new ByteArrayInputStream(v2SchemaBytes)); AvroWriter v2Writer = new AvroWriter<>(v2SchemaAvroNative); + @Cleanup Consumer c = pulsarClient.newConsumer(Schema.BYTES) .topic(topic) .subscriptionName("sub1").subscribe(); + @Cleanup Producer p = pulsarClient.newProducer(Schema.NATIVE_AVRO(v1SchemaAvroNative)) .topic(topic) .enableBatching(true) @@ -624,7 +645,7 @@ public void newNativeAvroProducerForMessageSchemaWithBatch() throws Exception { @Test public void newProducerWithMultipleSchemaDisabled() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; AvroWriter v1DataAvroWriter = new AvroWriter<>( ReflectData.AllowNull.get().getSchema(V1Data.class)); try (Producer p = pulsarClient.newProducer() @@ -638,7 +659,7 @@ public void newProducerWithMultipleSchemaDisabled() throws Exception { @Test public void newConsumerWithSchemaOnNewTopic() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Consumer c = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) .topic(topic).subscriptionName("sub1").subscribe(); @@ -651,7 +672,7 @@ public void newConsumerWithSchemaOnNewTopic() throws Exception { @Test public void newConsumerWithSchemaOnExistingTopicWithoutSchema() { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer().topic(topic).create(); Consumer c = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) @@ -664,7 +685,7 @@ public void newConsumerWithSchemaOnExistingTopicWithoutSchema() { @Test public void newConsumerWithSchemaTopicHasSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topic).create(); Consumer c = pulsarClient.newConsumer(Schema.AVRO(V1Data.class)) @@ -677,7 +698,7 @@ public void newConsumerWithSchemaTopicHasSchema() throws Exception { @Test public void newBytesConsumerWithTopicWithSchema() throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topic).create(); Consumer c = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").subscribe()) { @@ -697,7 +718,7 @@ public void getSchemaVersionFromMessagesBatchingEnabled() throws Exception { } private void getSchemaVersionFromMessages(boolean batching) throws Exception { - String topic = "my-property/my-ns/schema-test"; + String topic = NAMESPACE + "/schema-test"; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic) @@ -718,7 +739,7 @@ private void getSchemaVersionFromMessages(boolean batching) throws Exception { @Test(dataProvider = "batchingModes") public void testAutoConsume(boolean batching) throws Exception { - String topic = "my-property/my-ns/schema-test-auto-consume-" + batching; + String topic = NAMESPACE + "/schema-test-auto-consume-" + batching; try (Producer p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic) @@ -755,7 +776,7 @@ public void testAutoConsume(boolean batching) throws Exception { @Test(dataProvider = "batchingModesAndValueEncodingType") public void testAutoKeyValueConsume(boolean batching, KeyValueEncodingType keyValueEncodingType) throws Exception { - String topic = "my-property/my-ns/schema-test-auto-keyvalue-consume-" + batching+"-"+keyValueEncodingType; + String topic = NAMESPACE + "/schema-test-auto-keyvalue-consume-" + batching+"-"+keyValueEncodingType; Schema> pojoSchema = Schema.KeyValue( Schema.AVRO(V1Data.class), @@ -1039,7 +1060,7 @@ public void testAutoKeyValueConsume(boolean batching, KeyValueEncodingType keyVa @Test public void testAutoKeyValueConsumeGenericObject() throws Exception { - String topic = "my-property/my-ns/schema-test-auto-keyvalue-consume-"+ UUID.randomUUID(); + String topic = NAMESPACE + "/schema-test-auto-keyvalue-consume-"+ UUID.randomUUID(); Schema> pojoSchema = Schema.KeyValue( Schema.AVRO(V1Data.class), @@ -1121,17 +1142,19 @@ public void testAutoKeyValueConsumeGenericObject() throws Exception { @Test public void testGetSchemaByVersion() throws PulsarClientException, PulsarAdminException, ExecutionException, InterruptedException { - final String topic = "persistent://my-property/my-ns/testGetSchemaByVersion"; + final String topic = String.format("persistent://%s/testGetSchemaByVersion", NAMESPACE); @Cleanup PulsarClientImpl httpProtocolClient = (PulsarClientImpl) PulsarClient.builder().serviceUrl(brokerUrl.toString()).build(); PulsarClientImpl binaryProtocolClient = (PulsarClientImpl) pulsarClient; - pulsarClient.newProducer(Schema.AVRO(V1Data.class)) + @Cleanup + Producer p1 = pulsarClient.newProducer(Schema.AVRO(V1Data.class)) .topic(topic) .create(); - pulsarClient.newProducer(Schema.AVRO(V2Data.class)) + @Cleanup + Producer p2 = pulsarClient.newProducer(Schema.AVRO(V2Data.class)) .topic(topic) .create(); @@ -1148,7 +1171,7 @@ public void testGetSchemaByVersion() throws PulsarClientException, PulsarAdminEx @Test public void testGetNativeSchemaWithAutoConsumeWithMultiVersion() throws Exception { - final String topic = "persistent://my-property/my-ns/testGetSchemaWithMultiVersion"; + final String topic = String.format("persistent://%s/testGetSchemaWithMultiVersion", NAMESPACE); @Cleanup Consumer consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()) @@ -1188,8 +1211,8 @@ public void testGetNativeSchemaWithAutoConsumeWithMultiVersion() throws Exceptio @Test(dataProvider = "topicDomain") public void testAutoCreatedSchema(String domain) throws Exception { - final String topic1 = domain + "my-property/my-ns/testAutoCreatedSchema-1"; - final String topic2 = domain + "my-property/my-ns/testAutoCreatedSchema-2"; + final String topic1 = domain + NAMESPACE + "/testAutoCreatedSchema-1"; + final String topic2 = domain + NAMESPACE + "/testAutoCreatedSchema-2"; pulsarClient.newProducer(Schema.BYTES).topic(topic1).create().close(); try { @@ -1223,7 +1246,7 @@ public static Object[] keyEncodingType() { @Test(dataProvider = "keyEncodingType") public void testAutoKeyValueConsumeGenericObjectNullValues(KeyValueEncodingType encodingType) throws Exception { - String topic = "my-property/my-ns/schema-test-auto-keyvalue-" + encodingType + "-null-value-consume-" + UUID.randomUUID(); + String topic = NAMESPACE + "/schema-test-auto-keyvalue-" + encodingType + "-null-value-consume-" + UUID.randomUUID(); Schema> pojoSchema = Schema.KeyValue( Schema.AVRO(V1Data.class), @@ -1275,7 +1298,7 @@ public void testConsumeAvroMessagesWithoutSchema() throws Exception { if (schemaValidationEnforced) { return; } - final String topic = "test-consume-avro-messages-without-schema-" + UUID.randomUUID(); + final String topic = NAMESPACE + "/test-consume-avro-messages-without-schema-" + UUID.randomUUID(); final Schema schema = Schema.AVRO(V1Data.class); final Consumer consumer = pulsarClient.newConsumer(schema) .topic(topic) From e960a656c43dba6cdf065ff7569c51bcaae2a9a8 Mon Sep 17 00:00:00 2001 From: Huanli Meng <48120384+Huanli-Meng@users.noreply.github.com> Date: Fri, 14 Oct 2022 09:26:28 +0800 Subject: [PATCH 06/32] [fix][doc] Fix the grammar error in docs (#18033) --- site2/docs/security-overview.md | 8 ++++---- .../version-2.1.0-incubating/security-overview.md | 2 +- .../version-2.1.1-incubating/security-overview.md | 2 +- .../version-2.10.0-deprecated/security-overview.md | 2 +- .../version-2.10.1-deprecated/security-overview.md | 2 +- .../versioned_docs/version-2.10.x/security-overview.md | 2 +- .../versioned_docs/version-2.2.0/security-overview.md | 2 +- .../versioned_docs/version-2.2.1/security-overview.md | 2 +- .../versioned_docs/version-2.3.0/security-overview.md | 2 +- .../versioned_docs/version-2.3.1/security-overview.md | 2 +- .../versioned_docs/version-2.3.2/security-overview.md | 2 +- .../versioned_docs/version-2.5.0/security-overview.md | 2 +- .../versioned_docs/version-2.5.1/security-overview.md | 2 +- .../versioned_docs/version-2.5.2/security-overview.md | 2 +- .../versioned_docs/version-2.6.0/security-overview.md | 2 +- .../versioned_docs/version-2.6.1/security-overview.md | 2 +- .../versioned_docs/version-2.6.2/security-overview.md | 2 +- .../versioned_docs/version-2.6.3/security-overview.md | 2 +- .../versioned_docs/version-2.6.4/security-overview.md | 2 +- .../versioned_docs/version-2.7.0/security-overview.md | 2 +- .../versioned_docs/version-2.7.1/security-overview.md | 2 +- .../versioned_docs/version-2.7.2/security-overview.md | 2 +- .../versioned_docs/version-2.7.3/security-overview.md | 2 +- .../versioned_docs/version-2.7.4/security-overview.md | 2 +- .../versioned_docs/version-2.7.5/security-overview.md | 2 +- .../version-2.8.0-deprecated/security-overview.md | 2 +- .../version-2.8.1-deprecated/security-overview.md | 2 +- .../version-2.8.2-deprecated/security-overview.md | 2 +- .../version-2.8.3-deprecated/security-overview.md | 2 +- .../versioned_docs/version-2.8.x/security-overview.md | 2 +- .../version-2.9.0-deprecated/security-overview.md | 2 +- .../version-2.9.1-deprecated/security-overview.md | 2 +- .../version-2.9.2-deprecated/security-overview.md | 2 +- .../version-2.9.3-deprecated/security-overview.md | 2 +- .../versioned_docs/version-2.9.x/security-overview.md | 2 +- 35 files changed, 38 insertions(+), 38 deletions(-) diff --git a/site2/docs/security-overview.md b/site2/docs/security-overview.md index 3e9a90874bd54..3008e065f98cf 100644 --- a/site2/docs/security-overview.md +++ b/site2/docs/security-overview.md @@ -7,13 +7,13 @@ sidebar_label: "Overview" As the central message bus for a business, Apache Pulsar is frequently used for storing mission-critical data. Therefore, enabling security features in Pulsar is crucial. This chapter describes the main security controls that Pulsar uses to help protect your data. Pulsar security is based on the following core pillars. -* [Encryption](#encryption) -* [Authentication](#authentication) -* [Authorization](#authorization) +- [Encryption](#encryption) +- [Authentication](#authentication) +- [Authorization](#authorization) By default, Pulsar configures no encryption, authentication, or authorization. Any clients can communicate to Pulsar via plain text service URLs. So you must ensure that Pulsar accessing via these plain text service URLs is restricted to trusted clients only. In such cases, you can use network segmentation and/or authorization ACLs to restrict access to trusted IPs. If you use neither, the state of the cluster is wide open and anyone can access the cluster. -Apache Pulsar uses an [Authentication Provider](#authentication) or an [Authentication Provider Chain](security-extending.md/#proxybroker-authentication-plugin) to establish the identity of a client and then assign a *role token* (a string like `admin` or `app1`)to that client. This role token can represent a single client or multiple clients and is then used for [Authorization](security-authorization.md) to determine what the client is authorized to do. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. +Apache Pulsar uses an [Authentication Provider](#authentication) or an [Authentication Provider Chain](security-extending.md/#proxybroker-authentication-plugin) to establish the identity of a client and then assign a *role token* (a string like `admin` or `app1`) to that client. This role token can represent a single client or multiple clients and is then used for [Authorization](security-authorization.md) to determine what the client is authorized to do. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. ## Encryption diff --git a/site2/website/versioned_docs/version-2.1.0-incubating/security-overview.md b/site2/website/versioned_docs/version-2.1.0-incubating/security-overview.md index 6507a93733990..8ed2d907d7a95 100644 --- a/site2/website/versioned_docs/version-2.1.0-incubating/security-overview.md +++ b/site2/website/versioned_docs/version-2.1.0-incubating/security-overview.md @@ -21,7 +21,7 @@ It is strongly recommended to secure the service components in your Apache Pulsa In Pulsar, a *role* is a string, like `admin` or `app1`, that can represent a single client or multiple clients. Roles are used to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and more. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign that client a *role token*. This +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign that client a *role token*. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication Providers diff --git a/site2/website/versioned_docs/version-2.1.1-incubating/security-overview.md b/site2/website/versioned_docs/version-2.1.1-incubating/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.1.1-incubating/security-overview.md +++ b/site2/website/versioned_docs/version-2.1.1-incubating/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.10.0-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.10.0-deprecated/security-overview.md index d03b8c85c38f1..93a766ee89fb8 100644 --- a/site2/website/versioned_docs/version-2.10.0-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.10.0-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.10.1-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.10.1-deprecated/security-overview.md index d03b8c85c38f1..93a766ee89fb8 100644 --- a/site2/website/versioned_docs/version-2.10.1-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.10.1-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.10.x/security-overview.md b/site2/website/versioned_docs/version-2.10.x/security-overview.md index d03b8c85c38f1..93a766ee89fb8 100644 --- a/site2/website/versioned_docs/version-2.10.x/security-overview.md +++ b/site2/website/versioned_docs/version-2.10.x/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.2.0/security-overview.md b/site2/website/versioned_docs/version-2.2.0/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.2.0/security-overview.md +++ b/site2/website/versioned_docs/version-2.2.0/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.2.1/security-overview.md b/site2/website/versioned_docs/version-2.2.1/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.2.1/security-overview.md +++ b/site2/website/versioned_docs/version-2.2.1/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.3.0/security-overview.md b/site2/website/versioned_docs/version-2.3.0/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.3.0/security-overview.md +++ b/site2/website/versioned_docs/version-2.3.0/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.3.1/security-overview.md b/site2/website/versioned_docs/version-2.3.1/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.3.1/security-overview.md +++ b/site2/website/versioned_docs/version-2.3.1/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.3.2/security-overview.md b/site2/website/versioned_docs/version-2.3.2/security-overview.md index ad9a809e9125a..759ca8d785525 100644 --- a/site2/website/versioned_docs/version-2.3.2/security-overview.md +++ b/site2/website/versioned_docs/version-2.3.2/security-overview.md @@ -21,7 +21,7 @@ It is strongly recommended to secure the service components in your Apache Pulsa In Pulsar, a *role* is a string, like `admin` or `app1`, that can represent a single client or multiple clients. Roles are used to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and more. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign that client a *role token*. This +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign that client a *role token*. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication Providers diff --git a/site2/website/versioned_docs/version-2.5.0/security-overview.md b/site2/website/versioned_docs/version-2.5.0/security-overview.md index 91dc68e51d9cf..250db4f755bad 100644 --- a/site2/website/versioned_docs/version-2.5.0/security-overview.md +++ b/site2/website/versioned_docs/version-2.5.0/security-overview.md @@ -17,7 +17,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.5.1/security-overview.md b/site2/website/versioned_docs/version-2.5.1/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.5.1/security-overview.md +++ b/site2/website/versioned_docs/version-2.5.1/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.5.2/security-overview.md b/site2/website/versioned_docs/version-2.5.2/security-overview.md index 3f08cb29ad854..41beb0c87880b 100644 --- a/site2/website/versioned_docs/version-2.5.2/security-overview.md +++ b/site2/website/versioned_docs/version-2.5.2/security-overview.md @@ -20,7 +20,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.6.0/security-overview.md b/site2/website/versioned_docs/version-2.6.0/security-overview.md index 91dc68e51d9cf..250db4f755bad 100644 --- a/site2/website/versioned_docs/version-2.6.0/security-overview.md +++ b/site2/website/versioned_docs/version-2.6.0/security-overview.md @@ -17,7 +17,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.6.1/security-overview.md b/site2/website/versioned_docs/version-2.6.1/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.6.1/security-overview.md +++ b/site2/website/versioned_docs/version-2.6.1/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.6.2/security-overview.md b/site2/website/versioned_docs/version-2.6.2/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.6.2/security-overview.md +++ b/site2/website/versioned_docs/version-2.6.2/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.6.3/security-overview.md b/site2/website/versioned_docs/version-2.6.3/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.6.3/security-overview.md +++ b/site2/website/versioned_docs/version-2.6.3/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.6.4/security-overview.md b/site2/website/versioned_docs/version-2.6.4/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.6.4/security-overview.md +++ b/site2/website/versioned_docs/version-2.6.4/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.0/security-overview.md b/site2/website/versioned_docs/version-2.7.0/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.0/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.0/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.1/security-overview.md b/site2/website/versioned_docs/version-2.7.1/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.1/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.1/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.2/security-overview.md b/site2/website/versioned_docs/version-2.7.2/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.2/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.2/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.3/security-overview.md b/site2/website/versioned_docs/version-2.7.3/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.3/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.3/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.4/security-overview.md b/site2/website/versioned_docs/version-2.7.4/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.4/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.4/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.7.5/security-overview.md b/site2/website/versioned_docs/version-2.7.5/security-overview.md index a1664fcb8b450..6eb1e2d9309f0 100644 --- a/site2/website/versioned_docs/version-2.7.5/security-overview.md +++ b/site2/website/versioned_docs/version-2.7.5/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.8.0-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.8.0-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.8.0-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.8.0-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.8.1-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.8.1-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.8.1-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.8.1-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.8.2-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.8.2-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.8.2-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.8.2-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.8.3-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.8.3-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.8.3-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.8.3-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.8.x/security-overview.md b/site2/website/versioned_docs/version-2.8.x/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.8.x/security-overview.md +++ b/site2/website/versioned_docs/version-2.8.x/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.9.0-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.9.0-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.9.0-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.9.0-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.9.1-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.9.1-deprecated/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.9.1-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.9.1-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.9.2-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.9.2-deprecated/security-overview.md index df630ab519bb4..a8120f984bf82 100644 --- a/site2/website/versioned_docs/version-2.9.2-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.9.2-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.9.3-deprecated/security-overview.md b/site2/website/versioned_docs/version-2.9.3-deprecated/security-overview.md index df630ab519bb4..a8120f984bf82 100644 --- a/site2/website/versioned_docs/version-2.9.3-deprecated/security-overview.md +++ b/site2/website/versioned_docs/version-2.9.3-deprecated/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers diff --git a/site2/website/versioned_docs/version-2.9.x/security-overview.md b/site2/website/versioned_docs/version-2.9.x/security-overview.md index 227dd5b5d4fc3..c6bd9b64e4f76 100644 --- a/site2/website/versioned_docs/version-2.9.x/security-overview.md +++ b/site2/website/versioned_docs/version-2.9.x/security-overview.md @@ -21,7 +21,7 @@ You had better secure the service components in your Apache Pulsar deployment. In Pulsar, a *role* is a string, like `admin` or `app1`, which can represent a single client or multiple clients. You can use roles to control permission for clients to produce or consume from certain topics, administer the configuration for tenants, and so on. -Apache Pulsar uses a [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. +Apache Pulsar uses an [Authentication Provider](#authentication-providers) to establish the identity of a client and then assign a *role token* to that client. This role token is then used for [Authorization and ACLs](security-authorization.md) to determine what the client is authorized to do. ## Authentication providers From 3dc48d132cedc1ba96af0512328b021b9baab13a Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 14 Oct 2022 09:40:12 +0800 Subject: [PATCH 07/32] [fix][io] Remove unused import statement in the MariadbJdbcAutoSchemaSink (#18031) --- .../org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java | 1 - 1 file changed, 1 deletion(-) diff --git a/pulsar-io/jdbc/mariadb/src/main/java/org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java b/pulsar-io/jdbc/mariadb/src/main/java/org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java index d0f44e4c77c63..fcc135bfad74c 100644 --- a/pulsar-io/jdbc/mariadb/src/main/java/org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java +++ b/pulsar-io/jdbc/mariadb/src/main/java/org/apache/pulsar/io/jdbc/MariadbJdbcAutoSchemaSink.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import org.apache.pulsar.io.core.annotations.Connector; import org.apache.pulsar.io.core.annotations.IOType; From 97035ba59106e0341742d1dcd47830f9a9c2b364 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 14 Oct 2022 03:44:05 +0200 Subject: [PATCH 08/32] [fix][connectors] Fix builtin sink transformation on k8s (#18019) --- .../apache/pulsar/functions/worker/rest/api/SinksImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java index dede6307e76f0..80454ee890160 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/SinksImpl.java @@ -433,8 +433,8 @@ private void setTransformFunctionPackageLocation(Function.FunctionMetaData.Build Function.FunctionDetails functionDetails, String transformFunction) { File functionPackageFile = null; try { - String builtin = null; - if (!transformFunction.startsWith(Utils.BUILTIN)) { + String builtin = functionDetails.getBuiltin(); + if (isBlank(builtin)) { functionPackageFile = getPackageFile(transformFunction); } Function.PackageLocationMetaData.Builder functionPackageLocation = From fc61d5256d615b84ea964e549eea4ce82d7c4993 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Fri, 14 Oct 2022 16:23:35 +0800 Subject: [PATCH 09/32] [fix][test] Fix flaky test AdminApi2Test.resetClusters (#18043) --- .../pulsar/broker/admin/AdminApi2Test.java | 3 +- .../auth/MockedPulsarServiceBaseTest.java | 20 +++- .../OwnerShipForCurrentServerTestBase.java | 4 +- .../service/BacklogQuotaManagerTest.java | 21 ++++- .../pulsar/broker/service/BrokerTestBase.java | 94 ++++++++++++++++--- .../service/CanPausedNamespaceService.java | 66 +++++++++++++ .../transaction/TransactionTestBase.java | 25 ++++- .../api/DispatcherBlockConsumerTest.java | 4 +- 8 files changed, 210 insertions(+), 27 deletions(-) create mode 100644 pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanPausedNamespaceService.java diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java index cab1333b5bffb..357b801d265dc 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApi2Test.java @@ -164,7 +164,8 @@ public void resetClusters() throws Exception { pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); for (String tenant : admin.tenants().getTenants()) { for (String namespace : admin.namespaces().getNamespaces(tenant)) { - deleteNamespaceGraceFully(namespace, true); + deleteNamespaceGraceFullyByMultiPulsars(namespace, true, admin, pulsar, + mockPulsarSetup.getPulsar()); } admin.tenants().deleteTenant(tenant, true); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java index ace2e8a18f9d9..1ea11bb1ac4e4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/auth/MockedPulsarServiceBaseTest.java @@ -63,6 +63,7 @@ import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.CanPausedNamespaceService; import org.apache.pulsar.broker.service.PulsarMetadataEventSynchronizer; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; @@ -382,7 +383,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { doReturn(configSynchronizer != null ? createConfigurationMetadataStore(configSynchronizer) : createConfigurationMetadataStore()).when(pulsar).createConfigurationMetadataStore(any()); - Supplier namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(CanPausedNamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); doReturn(sameThreadOrderedSafeExecutor).when(pulsar).getOrderedExecutor(); @@ -667,19 +669,27 @@ public Map, Collection>> register(Object callback, Object... c } /** - * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarService, PulsarAdmin)} + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ protected void deleteNamespaceGraceFully(String ns, boolean force) throws Exception { - BrokerTestBase.deleteNamespaceGraceFully(ns, force, pulsar, admin); + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar); } /** - * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarService, PulsarAdmin)} + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ protected void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin) throws Exception { - BrokerTestBase.deleteNamespaceGraceFully(ns, force, pulsar, admin); + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar); + } + + /** + * see see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} + */ + protected void deleteNamespaceGraceFullyByMultiPulsars(String ns, boolean force, PulsarAdmin admin, + PulsarService...pulsars) throws Exception { + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsars); } private static final Logger log = LoggerFactory.getLogger(MockedPulsarServiceBaseTest.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java index 79183b358b8ad..1712fcc18488a 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/namespace/OwnerShipForCurrentServerTestBase.java @@ -45,6 +45,7 @@ import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.auth.SameThreadOrderedSafeExecutor; import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; +import org.apache.pulsar.broker.service.CanPausedNamespaceService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; @@ -131,7 +132,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { MockZooKeeperSession mockZooKeeperSession = MockZooKeeperSession.newInstance(mockZooKeeper); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createLocalMetadataStore(null); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createConfigurationMetadataStore(null); - Supplier namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); + Supplier namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs( + CanPausedNamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); SameThreadOrderedSafeExecutor executor = new SameThreadOrderedSafeExecutor(); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java index e60486b344182..9eb6281eddc6d 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BacklogQuotaManagerTest.java @@ -26,6 +26,7 @@ import java.net.URL; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Optional; @@ -81,11 +82,27 @@ public class BacklogQuotaManagerTest { private static final int MAX_ENTRIES_PER_LEDGER = 5; /** - * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarService, PulsarAdmin)} + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ protected void deleteNamespaceGraceFully(String ns, boolean force) throws Exception { - BrokerTestBase.deleteNamespaceGraceFully(ns, force, pulsar, admin); + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar); + } + + /** + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} + */ + protected void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin) + throws Exception { + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsar); + } + + /** + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} + */ + protected void deleteNamespaceGraceFullyByMultiPulsars(String ns, boolean force, PulsarAdmin admin, + PulsarService...pulsars) throws Exception { + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsars); } @DataProvider(name = "backlogQuotaSizeGB") diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerTestBase.java index efdfe0fe188c4..a9a40bce1c784 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/BrokerTestBase.java @@ -23,7 +23,10 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import org.apache.commons.collections4.CollectionUtils; @@ -32,8 +35,10 @@ import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.namespace.NamespaceEphemeralData; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.systopic.SystemTopicClient; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminException; +import org.apache.pulsar.common.events.PulsarEvent; import org.apache.pulsar.common.naming.NamespaceBundle; import org.apache.pulsar.common.naming.NamespaceName; import org.apache.pulsar.common.naming.SystemTopicNames; @@ -41,9 +46,11 @@ import org.apache.pulsar.common.partition.PartitionedTopicMetadata; import org.apache.pulsar.common.policies.data.ClusterData; import org.apache.pulsar.common.policies.data.TenantInfoImpl; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.coordination.LockManager; import org.awaitility.Awaitility; +import org.awaitility.reflect.WhiteboxImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -128,42 +135,91 @@ protected String newTopicName() { } /** - * see {@link #deleteNamespaceGraceFully} + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ protected void deleteNamespaceGraceFully(String ns, boolean force) throws Exception { - deleteNamespaceGraceFully(ns, force, pulsar, admin); + deleteNamespaceGraceFully(ns, force, admin, pulsar); } /** - * Wait until system topic "__change_event" and subscription "__compaction" are created, and then delete the namespace. + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ - public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarService pulsar, PulsarAdmin admin) + public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin, PulsarService...pulsars) throws Exception { + deleteNamespaceGraceFully(ns, force, admin, Arrays.asList(pulsars)); + } + + /** + * Wait until system topic "__change_event" and subscription "__compaction" are created, and then delete the namespace. + */ + public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin, + Collection pulsars) throws Exception { // namespace v1 should not wait system topic create. if (ns.split("/").length > 2){ admin.namespaces().deleteNamespace(ns, force); return; } - if (!pulsar.getConfiguration().isSystemTopicEnabled()){ + + // If disabled system-topic, should not wait system topic create. + boolean allBrokerDisabledSystemTopic = true; + for (PulsarService pulsar : pulsars) { + if (!pulsar.getConfiguration().isSystemTopicEnabled()) { + continue; + } + TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); + if (!(topicPoliciesService instanceof SystemTopicBasedTopicPoliciesService)) { + continue; + } + allBrokerDisabledSystemTopic = false; + } + if (allBrokerDisabledSystemTopic){ admin.namespaces().deleteNamespace(ns, force); return; } - // If no bundle has been loaded, then the System Topic will not trigger creation. - LockManager lockManager = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class); - List lockedBundles = (List) lockManager.listLocks("/namespace" + "/" + ns).join(); - if (CollectionUtils.isEmpty(lockedBundles)){ + + // Stop trigger "onNamespaceBundleOwned". + List>> createReaderTasks = new ArrayList<>(); + List lockedBundles = new ArrayList<>(); + for (PulsarService pulsar : pulsars) { + // Prevents new events from triggering system topic creation. + CanPausedNamespaceService canPausedNamespaceService = (CanPausedNamespaceService) pulsar.getNamespaceService(); + canPausedNamespaceService.pause(); + + // If no bundle has been loaded, then the System Topic will not trigger creation. + LockManager lockManager = pulsar.getCoordinationService().getLockManager(NamespaceEphemeralData.class); + lockedBundles.addAll((List) lockManager.listLocks("/namespace" + "/" + ns).join()); + + // Determines whether the creation of System topic is triggered. + // If readerCaches contains namespace, the creation of System topic already triggered. + TopicPoliciesService topicPoliciesService = pulsar.getTopicPoliciesService(); + if (topicPoliciesService instanceof + SystemTopicBasedTopicPoliciesService systemTopicBasedTopicPoliciesService) { + Map>> readerCaches = + WhiteboxImpl.getInternalState(systemTopicBasedTopicPoliciesService, "readerCaches"); + if (readerCaches.containsKey(NamespaceName.get(ns))) { + createReaderTasks.add(readerCaches.get(NamespaceName.get(ns))); + } + } + } + // Wait all reader-create tasks. + FutureUtil.waitForAll(createReaderTasks).join(); + + // If the bundle elect has not yet been triggered, skip wait. + if (CollectionUtils.isEmpty(lockedBundles) && createReaderTasks.isEmpty()){ admin.namespaces().deleteNamespace(ns, force); return; } + // Trigger change event topic create. + PulsarService firstPulsar = pulsars.iterator().next(); NamespaceName namespace = NamespaceName.get(ns); NamespaceBundle namespaceBundle = mock(NamespaceBundle.class); when(namespaceBundle.getNamespaceObject()).thenReturn(namespace); - pulsar.getTopicPoliciesService().addOwnedNamespaceBundleAsync(namespaceBundle); + firstPulsar.getTopicPoliciesService().addOwnedNamespaceBundleAsync(namespaceBundle); // Wait for change event topic and compaction create finish. - String allowAutoTopicCreationType = pulsar.getConfiguration().getAllowAutoTopicCreationType(); - int defaultNumPartitions = pulsar.getConfiguration().getDefaultNumPartitions(); + String allowAutoTopicCreationType = firstPulsar.getConfiguration().getAllowAutoTopicCreationType(); + int defaultNumPartitions = firstPulsar.getConfiguration().getDefaultNumPartitions(); ArrayList expectChangeEventTopics = new ArrayList<>(); if ("non-partitioned".equals(allowAutoTopicCreationType)){ String t = String.format("persistent://%s/%s", ns, NAMESPACE_EVENTS_LOCAL_NAME); @@ -177,7 +233,7 @@ public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarSer Awaitility.await().until(() -> { boolean finished = true; for (String changeEventTopicName : expectChangeEventTopics){ - boolean bundleExists = pulsar.getNamespaceService() + boolean bundleExists = firstPulsar.getNamespaceService() .checkTopicOwnership(TopicName.get(changeEventTopicName)) .exceptionally(ex -> false).join(); if (!bundleExists){ @@ -185,7 +241,7 @@ public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarSer break; } CompletableFuture> completableFuture = - pulsar.getBrokerService().getTopic(changeEventTopicName, false); + firstPulsar.getBrokerService().getTopic(changeEventTopicName, false); if (completableFuture == null){ finished = false; break; @@ -206,6 +262,8 @@ public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarSer } return finished; }); + + // Do delete. int retryTimes = 3; while (true) { try { @@ -223,6 +281,14 @@ public static void deleteNamespaceGraceFully(String ns, boolean force, PulsarSer throw ex; } } + + // Resume trigger "onNamespaceBundleOwned". + for (PulsarService pulsarService : pulsars) { + // Prevents new events from triggering system topic creation. + CanPausedNamespaceService canPausedNamespaceService = + (CanPausedNamespaceService) pulsarService.getNamespaceService(); + canPausedNamespaceService.resume(); + } } private static final Logger LOG = LoggerFactory.getLogger(BrokerTestBase.class); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanPausedNamespaceService.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanPausedNamespaceService.java new file mode 100644 index 0000000000000..c5527a9149897 --- /dev/null +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/CanPausedNamespaceService.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.broker.service; + +import java.util.concurrent.locks.ReentrantLock; +import org.apache.pulsar.broker.PulsarService; +import org.apache.pulsar.broker.namespace.NamespaceService; +import org.apache.pulsar.common.naming.NamespaceBundle; + +public class CanPausedNamespaceService extends NamespaceService { + + private volatile boolean paused = false; + + private ReentrantLock lock = new ReentrantLock(); + + public CanPausedNamespaceService(PulsarService pulsar) { + super(pulsar); + } + + @Override + protected void onNamespaceBundleOwned(NamespaceBundle bundle) { + lock.lock(); + try { + if (paused){ + return; + } + super.onNamespaceBundleOwned(bundle); + } finally { + lock.unlock(); + } + } + + public void pause(){ + lock.lock(); + try { + paused = true; + } finally { + lock.unlock(); + } + } + + public void resume(){ + lock.lock(); + try { + paused = false; + } finally { + lock.unlock(); + } + } +} diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java index dd5c0f29237be..123755fe870d4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTestBase.java @@ -27,6 +27,7 @@ import io.netty.channel.EventLoopGroup; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Optional; @@ -48,6 +49,7 @@ import org.apache.pulsar.broker.intercept.CounterBrokerInterceptor; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.broker.service.CanPausedNamespaceService; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.common.naming.NamespaceName; @@ -195,7 +197,8 @@ protected void setupBrokerMocks(PulsarService pulsar) throws Exception { MockZooKeeperSession mockZooKeeperSession = MockZooKeeperSession.newInstance(mockZooKeeper); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createLocalMetadataStore(null); doReturn(new ZKMetadataStore(mockZooKeeperSession)).when(pulsar).createConfigurationMetadataStore(null); - Supplier namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(CanPausedNamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); SameThreadOrderedSafeExecutor executor = new SameThreadOrderedSafeExecutor(); @@ -329,10 +332,26 @@ protected final void internalCleanup() { } /** - * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarService, PulsarAdmin)} + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} */ protected void deleteNamespaceGraceFully(String ns, boolean force) throws Exception { - BrokerTestBase.deleteNamespaceGraceFully(ns, force, pulsarServiceList.get(0), admin); + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsarServiceList); + } + + /** + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} + */ + protected void deleteNamespaceGraceFully(String ns, boolean force, PulsarAdmin admin) + throws Exception { + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsarServiceList); + } + + /** + * see {@link BrokerTestBase#deleteNamespaceGraceFully(String, boolean, PulsarAdmin, Collection)} + */ + protected void deleteNamespaceGraceFullyByMultiPulsars(String ns, boolean force, PulsarAdmin admin, + Collection pulsars) throws Exception { + BrokerTestBase.deleteNamespaceGraceFully(ns, force, admin, pulsars); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java index 6c5ac0460d57f..93b51ebf871f4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/DispatcherBlockConsumerTest.java @@ -50,6 +50,7 @@ import lombok.Cleanup; import org.apache.pulsar.broker.namespace.NamespaceService; import org.apache.pulsar.broker.service.BrokerService; +import org.apache.pulsar.broker.service.CanPausedNamespaceService; import org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers; import org.apache.pulsar.broker.service.persistent.PersistentTopic; import org.apache.pulsar.client.impl.ConsumerImpl; @@ -633,7 +634,8 @@ public void testBrokerSubscriptionRecovery(boolean unloadBundleGracefully) throw // if broker unload bundle gracefully then cursor metadata recovered from zk else from ledger if (unloadBundleGracefully) { // set clean namespace which will not let broker unload bundle gracefully: stop broker - Supplier namespaceServiceSupplier = () -> spyWithClassAndConstructorArgs(NamespaceService.class, pulsar); + Supplier namespaceServiceSupplier = + () -> spyWithClassAndConstructorArgs(CanPausedNamespaceService.class, pulsar); doReturn(namespaceServiceSupplier).when(pulsar).getNamespaceServiceProvider(); } stopBroker(); From b4518802f0dfad857abf3575758a1f69aa9457f8 Mon Sep 17 00:00:00 2001 From: ZhangJian He Date: Fri, 14 Oct 2022 19:14:56 +0800 Subject: [PATCH 10/32] [fix] [pulsar-client] Fix pendingLookupRequestSemaphore leak when channel inactive (#17856) ### Motivation https://github.com/apache/pulsar/blob/b89c1451551a6bbe681465726906a2e61c9d8a69/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java#L282-L297 The `pendingLookupRequestSemaphore` will leak when channel inactive. There are `LookUpRequestSemaphore` not released when removing it from `pendingRequests` ### Modifications We can't easily release the semaphore in `channelInactive`, because there are not only `LookUpRequest`. So release the semaphore when connectionException ### Verifying this change Add unit test case to cover this change ### Documentation - [ ] `doc-required` (Your PR needs to update docs and you will update later) - [x] `doc-not-needed` bug fixs, no need doc - [ ] `doc` (Your PR contains doc changes) - [ ] `doc-complete` (Docs have been already added) --- .../apache/pulsar/client/impl/ClientCnx.java | 8 +++ .../pulsar/client/impl/ClientCnxTest.java | 66 +++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java index a6b9005611c0f..14a33cd3203e1 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/ClientCnx.java @@ -141,6 +141,9 @@ public class ClientCnx extends PulsarHandler { private final CompletableFuture connectionFuture = new CompletableFuture(); private final ConcurrentLinkedQueue requestTimeoutQueue = new ConcurrentLinkedQueue<>(); + + @VisibleForTesting + @Getter(AccessLevel.PACKAGE) private final Semaphore pendingLookupRequestSemaphore; private final Semaphore maxLookupRequestSemaphore; private final EventLoopGroup eventLoopGroup; @@ -780,6 +783,11 @@ public CompletableFuture newLookup(ByteBuf request, long reque TimedCompletableFuture future = new TimedCompletableFuture<>(); if (pendingLookupRequestSemaphore.tryAcquire()) { + future.whenComplete((lookupDataResult, throwable) -> { + if (throwable instanceof ConnectException) { + pendingLookupRequestSemaphore.release(); + } + }); addPendingLookupRequests(requestId, future); ctx.writeAndFlush(request).addListener(writeFuture -> { if (!writeFuture.isSuccess()) { diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java index c6eba43fb7a16..63aa7b7048be0 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/ClientCnxTest.java @@ -34,6 +34,7 @@ import io.netty.util.concurrent.DefaultThreadFactory; import java.lang.reflect.Field; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ThreadFactory; import java.util.function.Consumer; @@ -50,6 +51,7 @@ import org.apache.pulsar.common.protocol.Commands; import org.apache.pulsar.common.protocol.PulsarHandler; import org.apache.pulsar.common.util.netty.EventLoopUtil; +import org.awaitility.Awaitility; import org.testng.annotations.Test; public class ClientCnxTest { @@ -80,6 +82,70 @@ public void testClientCnxTimeout() throws Exception { eventLoop.shutdownGracefully(); } + @Test + public void testPendingLookupRequestSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + CountDownLatch countDownLatch = new CountDownLatch(1); + CompletableFuture completableFuture = new CompletableFuture<>(); + new Thread(() -> { + try { + Thread.sleep(1_000); + CompletableFuture future = + cnx.newLookup(null, 123); + countDownLatch.countDown(); + future.get(); + } catch (Exception e) { + completableFuture.complete(e); + } + }).start(); + countDownLatch.await(); + cnx.channelInactive(ctx); + assertTrue(completableFuture.get().getCause() instanceof PulsarClientException.ConnectException); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + + @Test + public void testPendingWaitingLookupRequestSemaphore() throws Exception { + EventLoopGroup eventLoop = EventLoopUtil.newEventLoopGroup(1, false, new DefaultThreadFactory("testClientCnxTimeout")); + ClientConfigurationData conf = new ClientConfigurationData(); + conf.setOperationTimeoutMs(10_000); + conf.setKeepAliveIntervalSeconds(0); + ClientCnx cnx = new ClientCnx(conf, eventLoop); + + ChannelHandlerContext ctx = mock(ChannelHandlerContext.class); + Channel channel = mock(Channel.class); + when(ctx.channel()).thenReturn(channel); + ChannelFuture listenerFuture = mock(ChannelFuture.class); + when(listenerFuture.addListener(any())).thenReturn(listenerFuture); + when(ctx.writeAndFlush(any())).thenReturn(listenerFuture); + cnx.channelActive(ctx); + for (int i = 0; i < 5001; i++) { + cnx.newLookup(null, i); + } + cnx.channelInactive(ctx); + // wait for subsequent calls over + Awaitility.await().untilAsserted(() -> { + assertEquals(cnx.getPendingLookupRequestSemaphore().availablePermits(), conf.getConcurrentLookupRequest()); + }); + eventLoop.shutdownGracefully(); + } + @Test public void testReceiveErrorAtSendConnectFrameState() throws Exception { ThreadFactory threadFactory = new DefaultThreadFactory("testReceiveErrorAtSendConnectFrameState"); From 28d8514e883b7962a22c9792988fe79e2ed3ddb6 Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Fri, 14 Oct 2022 19:54:00 +0800 Subject: [PATCH 11/32] [improve][doc] Update workflow of submitting release note (#18047) --- wiki/release/release-note-guide.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/wiki/release/release-note-guide.md b/wiki/release/release-note-guide.md index 17a0acea0c607..2ae6a51be265d 100644 --- a/wiki/release/release-note-guide.md +++ b/wiki/release/release-note-guide.md @@ -41,9 +41,17 @@ For the [Pulsar Release Note page](https://pulsar.apache.org/release-notes/): ## Submit release notes -Follow the steps below to submit release notes for Pulsar and clients. +Follow the steps below to submit release notes for Pulsar and clients (**Java and WebSocket**). -1. On the [Pulsar Releases Page - GitHub](https://github.com/apache/pulsar/releases), add a release note for the new release. +> **Note** +> +> For **C++, Python, Go, Node.js, and C#**, you do not need to take care of them since their release notes are synced from their repos to the [Pulsar Release Note page](https://pulsar.apache.org/release-notes). + +1. Submit a PR to add **separate** release notes for Pulsar and clients (**Java and WebSocket**) to [pulsar-site/site2/website-next/release-notes/versioned/](https://github.com/apache/pulsar-site/tree/main/site2/website-next/release-notes/versioned). + + Get this PR reviewed and merged. + +2. Copy the release note to the [Pulsar Releases Page - GitHub](https://github.com/apache/pulsar/releases). @@ -55,20 +63,16 @@ Follow the steps below to submit release notes for Pulsar and clients. - + - - - - - - +
Pulsar coreAdd a release note for it.Copy the release note content.
Pulsar clients - Java

- WebSocket

- C++

- Python		Add separate release notes for them, that is, create independent sections in the release note.

Example

Java client release note example
- Go

- Node.js

- C#		No action is needed. You do not need to take care of them since their release notes are synced from their repos to the Pulsar Release Note page. - Java

- WebSocket

+ 		Create independent sections for each client and copy release note content.

Example

Java client release note example
After the new release is published, all the information about the release is automatically added to the [Pulsar Release Note page](https://pulsar.apache.org/release-notes/). -2. Check whether the release information is shown on the [Pulsar Release Note page](https://pulsar.apache.org/release-notes/) after the website is updated and built successfully. \ No newline at end of file +3. Check whether the release information is shown on the [Pulsar Release Note page](https://pulsar.apache.org/release-notes/) after the website is updated and built successfully. \ No newline at end of file From 7dc9a5f1b269b05e7d7c1b101a89637951c55026 Mon Sep 17 00:00:00 2001 From: Michael Marshall Date: Fri, 14 Oct 2022 08:05:11 -0700 Subject: [PATCH 12/32] [cleanup][proxy] Use correct address for HAProxyMessage destination (#16045) * [fix][proxy] Use correct channel for HAProxyMessage dest address * Fix implementation after researching HAProxy ### Motivation When reading through the Proxy code, I noticed that the `remoteAddress` was used where it probably makes sense to use the `localAddress`. After researching it a bit with the AWS docs https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/enable-proxy-protocol.html and the protocol spec http://www.haproxy.org/download/1.8/doc/proxy-protocol.txt, I think this implementation is correct. We could look at using the value in the broker logs if we wanted to make it clear which proxy the traffic is ingressing through. For now, I propose we just fix the code in the proxy. ### Modifications * Replace `inboundChannel.localAddress` with `inboundChannel.localAddress`. * Fix type checking - [x] `doc-not-needed` --- .../org/apache/pulsar/proxy/server/DirectProxyHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java index 1179b4db88040..3994cf9034a8a 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/DirectProxyHandler.java @@ -243,11 +243,11 @@ private void writeHAProxyMessage() { .addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); } else { if (inboundChannel.remoteAddress() instanceof InetSocketAddress - && outboundChannel.localAddress() instanceof InetSocketAddress) { + && inboundChannel.localAddress() instanceof InetSocketAddress) { InetSocketAddress clientAddress = (InetSocketAddress) inboundChannel.remoteAddress(); String sourceAddress = clientAddress.getAddress().getHostAddress(); int sourcePort = clientAddress.getPort(); - InetSocketAddress proxyAddress = (InetSocketAddress) inboundChannel.remoteAddress(); + InetSocketAddress proxyAddress = (InetSocketAddress) inboundChannel.localAddress(); String destinationAddress = proxyAddress.getAddress().getHostAddress(); int destinationPort = proxyAddress.getPort(); HAProxyMessage msg = new HAProxyMessage(HAProxyProtocolVersion.V1, HAProxyCommand.PROXY, From 6d86a2fe7486f2cfb3ea635c2c753535f7865cd0 Mon Sep 17 00:00:00 2001 From: Christophe Bornet Date: Fri, 14 Oct 2022 17:17:36 +0200 Subject: [PATCH 13/32] [fix][io] Fix builtin connector/function download filename (#18044) When using KubernetesRuntime, if uploadBuiltinSinksSources is false (non-default), the originalFileName is not set for builtin components. But with k8s, we need a filename since the component will be downloaded. This results in the component currently being downloaded with the download directory as filename. Also when using a transform function, we use the originalFileName to know if there's a builtin function to download. --- .../worker/rest/api/ComponentImpl.java | 30 ++++++++----------- .../api/v3/FunctionApiV3ResourceTest.java | 1 + .../rest/api/v3/SinkApiV3ResourceTest.java | 6 ++-- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java index 7cd35352bd21f..95c9d70058d3b 100644 --- a/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java +++ b/pulsar-functions/worker/src/main/java/org/apache/pulsar/functions/worker/rest/api/ComponentImpl.java @@ -324,26 +324,22 @@ PackageLocationMetaData.Builder getFunctionPackageLocation(final FunctionMetaDat // For externally managed schedulers, the pkgUrl/builtin stuff can be copied to bk // if the function worker image does not include connectors if (!isEmpty(builtin)) { + File component; + switch (componentType) { + case SOURCE: + component = worker().getConnectorsManager().getSourceArchive(builtin).toFile(); + break; + case SINK: + component = worker().getConnectorsManager().getSinkArchive(builtin).toFile(); + break; + default: + component = worker().getFunctionsManager().getFunctionArchive(builtin).toFile(); + break; + } + packageLocationMetaDataBuilder.setOriginalFileName(component.getName()); if (worker().getWorkerConfig().getUploadBuiltinSinksSources()) { - File component; - String archiveName; - switch (componentType) { - case SOURCE: - archiveName = functionDetails.getSource().getBuiltin(); - component = worker().getConnectorsManager().getSourceArchive(archiveName).toFile(); - break; - case SINK: - archiveName = functionDetails.getSink().getBuiltin(); - component = worker().getConnectorsManager().getSinkArchive(archiveName).toFile(); - break; - default: - archiveName = functionDetails.getBuiltin(); - component = worker().getFunctionsManager().getFunctionArchive(archiveName).toFile(); - break; - } packageLocationMetaDataBuilder.setPackagePath(createPackagePath(tenant, namespace, componentName, component.getName())); - packageLocationMetaDataBuilder.setOriginalFileName(component.getName()); if (isPackageManagementEnabled) { packageLocationMetaDataBuilder.setPackagePath(packageName.toString()); worker().getBrokerAdmin().packages().upload(metadata, diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java index fb09a4026a59c..d8dc9a063c4d1 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/FunctionApiV3ResourceTest.java @@ -776,6 +776,7 @@ public void testRegisterFunctionSuccessK8sNoUpload() throws Exception { .classLoader(mockedClassLoader) .build(); when(mockedFunctionsManager.getFunction("exclamation")).thenReturn(functionArchive); + when(mockedFunctionsManager.getFunctionArchive(any())).thenReturn(getPulsarApiExamplesNar().toPath()); when(mockedWorkerService.getFunctionsManager()).thenReturn(mockedFunctionsManager); diff --git a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java index 7fd0fb1030fc9..05073b9732e2f 100644 --- a/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java +++ b/pulsar-functions/worker/src/test/java/org/apache/pulsar/functions/worker/rest/api/v3/SinkApiV3ResourceTest.java @@ -1754,14 +1754,14 @@ public void testRegisterSinkSuccessK8sNoUpload() throws Exception { ctx.when(() -> FunctionCommon.getClassLoaderFromPackage(any(), any(), any(), any())).thenCallRealMethod(); ctx.when(() -> FunctionCommon.isFunctionCodeBuiltin(any())).thenReturn(true); ctx.when(() -> FunctionCommon.extractNarClassLoader(any(), any())).thenReturn(mockedClassLoader); - }); ConnectorsManager mockedConnManager = mock(ConnectorsManager.class); Connector connector = Connector.builder() - .classLoader(mockedClassLoader) - .build(); + .classLoader(mockedClassLoader) + .build(); when(mockedConnManager.getConnector("cassandra")).thenReturn(connector); + when(mockedConnManager.getSinkArchive(any())).thenReturn(getPulsarIOCassandraNar().toPath()); when(mockedWorkerService.getConnectorsManager()).thenReturn(mockedConnManager); when(mockedRuntimeFactory.externallyManaged()).thenReturn(true); From 71fb9ebc3de192d247d3bb083d88fc8c5959a736 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 14 Oct 2022 23:22:16 +0800 Subject: [PATCH 14/32] Bump commons-text to 1.10.0 (#18053) --- distribution/server/src/assemble/LICENSE.bin.txt | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index b7ac2c7327dc6..19c2a014e94e3 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -284,7 +284,7 @@ The Apache Software License, Version 2.0 - org.apache.commons-commons-collections4-4.4.jar - org.apache.commons-commons-compress-1.21.jar - org.apache.commons-commons-lang3-3.11.jar - - org.apache.commons-commons-text-1.9.jar + - org.apache.commons-commons-text-1.10.0.jar * Netty - io.netty-netty-buffer-4.1.77.Final.jar - io.netty-netty-codec-4.1.77.Final.jar diff --git a/pom.xml b/pom.xml index 8fbc816475620..2dea7306703a4 100644 --- a/pom.xml +++ b/pom.xml @@ -121,7 +121,7 @@ flexible messaging model and an intuitive client API. 4.15.1 3.8.0 1.5.0 - 1.9 + 1.10.0 1.1.8.4 4.1.12.1 5.1.0 From fc7262ac8f5f59c2d47e4e23f9d4652abd4f0fce Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Sat, 15 Oct 2022 00:24:28 +0800 Subject: [PATCH 15/32] Add link to CPP/Python client to README.md (#18054) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f858e1065723e..4b2486e50b7e1 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ components in the Pulsar ecosystem, including connectors, adapters, and other la ### Clients +- [C++ Client](https://github.com/apache/pulsar-client-cpp) +- [Python Client](https://github.com/apache/pulsar-client-python) - [.NET/C# Client](https://github.com/apache/pulsar-dotpulsar) - [Go Client](https://github.com/apache/pulsar-client-go) - [NodeJS Client](https://github.com/apache/pulsar-client-node) From da0db33c3e003adc922a71b61563a4553d33bc52 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Fri, 14 Oct 2022 11:28:51 -0500 Subject: [PATCH 16/32] [fix][ci] Fix deprecation warnings about set-output (#18048) Error message is: The `set-output` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/ --- .github/workflows/ci-go-functions.yaml | 2 +- .github/workflows/pulsar-ci-flaky.yaml | 2 +- .github/workflows/pulsar-ci.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-go-functions.yaml b/.github/workflows/ci-go-functions.yaml index 862742569b458..cb21a5028c63e 100644 --- a/.github/workflows/ci-go-functions.yaml +++ b/.github/workflows/ci-go-functions.yaml @@ -54,7 +54,7 @@ jobs: - name: Check changed files id: check_changes run: | - echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} diff --git a/.github/workflows/pulsar-ci-flaky.yaml b/.github/workflows/pulsar-ci-flaky.yaml index e4c0eeb66c077..b5fa8e4a7540c 100644 --- a/.github/workflows/pulsar-ci-flaky.yaml +++ b/.github/workflows/pulsar-ci-flaky.yaml @@ -56,7 +56,7 @@ jobs: - name: Check changed files id: check_changes run: | - echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT - name: Check if the PR has been approved for testing if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index d095178cfaae3..a58fef08c15a5 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -56,7 +56,7 @@ jobs: - name: Check changed files id: check_changes run: | - echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" + echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT - name: Check if the PR has been approved for testing From 09f5eeb0c946ee890483e087f802e30a2a2b60ab Mon Sep 17 00:00:00 2001 From: Enrico Olivelli Date: Sat, 15 Oct 2022 18:24:20 +0200 Subject: [PATCH 17/32] Make BookieId work with PulsarRegistrationDriver (second take) (#17922) * Make BookieId work with PulsarRegistrationDriver (#17762) * Make BookieId work with PulsarRegistrationDriver * Switch to MetadataCache * checkstyle * Do not execute lookup on MetadataCache in the getBookieServiceInfo caller thread --- .../bookkeeper/BookieServiceInfoSerde.java | 55 +++++++- .../bookkeeper/PulsarRegistrationClient.java | 119 +++++++++++++++++- .../PulsarRegistrationClientTest.java | 62 +++++++++ 3 files changed, 230 insertions(+), 6 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BookieServiceInfoSerde.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BookieServiceInfoSerde.java index 78a33179e76b6..b7e3024b637fb 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BookieServiceInfoSerde.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/BookieServiceInfoSerde.java @@ -24,6 +24,7 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.bookkeeper.discover.BookieServiceInfo; +import org.apache.bookkeeper.discover.BookieServiceInfoUtils; import org.apache.bookkeeper.proto.DataFormats.BookieServiceInfoFormat; import org.apache.pulsar.metadata.api.MetadataSerde; import org.apache.pulsar.metadata.api.Stat; @@ -63,7 +64,57 @@ public byte[] serialize(String path, BookieServiceInfo bookieServiceInfo) throws } @Override - public BookieServiceInfo deserialize(String path, byte[] content, Stat stat) throws IOException { - return null; + public BookieServiceInfo deserialize(String path, byte[] bookieServiceInfo, Stat stat) throws IOException { + // see https://github.com/apache/bookkeeper/blob/ + // 034ef8566ad037937a4d58a28f70631175744f53/bookkeeper-server/ + // src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java#L311 + String bookieId = extractBookiedIdFromPath(path); + if (bookieServiceInfo == null || bookieServiceInfo.length == 0) { + return BookieServiceInfoUtils.buildLegacyBookieServiceInfo(bookieId); + } + + BookieServiceInfoFormat builder = BookieServiceInfoFormat.parseFrom(bookieServiceInfo); + BookieServiceInfo bsi = new BookieServiceInfo(); + List endpoints = builder.getEndpointsList().stream() + .map(e -> { + BookieServiceInfo.Endpoint endpoint = new BookieServiceInfo.Endpoint(); + endpoint.setId(e.getId()); + endpoint.setPort(e.getPort()); + endpoint.setHost(e.getHost()); + endpoint.setProtocol(e.getProtocol()); + endpoint.setAuth(e.getAuthList()); + endpoint.setExtensions(e.getExtensionsList()); + return endpoint; + }) + .collect(Collectors.toList()); + + bsi.setEndpoints(endpoints); + bsi.setProperties(builder.getPropertiesMap()); + + return bsi; + + } + + /** + * Extract the BookieId + * The path should look like /ledgers/available/bookieId + * or /ledgers/available/readonly/bookieId. + * But the prefix depends on the configuration. + * @param path + * @return the bookieId + */ + private static String extractBookiedIdFromPath(String path) throws IOException { + // https://github.com/apache/bookkeeper/blob/ + // 034ef8566ad037937a4d58a28f70631175744f53/bookkeeper-server/ + // src/main/java/org/apache/bookkeeper/discover/ZKRegistrationClient.java#L258 + if (path == null) { + path = ""; + } + int last = path.lastIndexOf("/"); + if (last >= 0) { + return path.substring(last + 1); + } else { + throw new IOException("The path " + path + " doesn't look like a valid path for a BookieServiceInfo node"); + } } } diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java index 52b50e3ea4b08..f314c0efaf058 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClient.java @@ -22,22 +22,33 @@ import static org.apache.bookkeeper.util.BookKeeperConstants.COOKIE_NODE; import static org.apache.bookkeeper.util.BookKeeperConstants.READONLY; import io.netty.util.concurrent.DefaultThreadFactory; +import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.client.BKException; +import org.apache.bookkeeper.common.concurrent.FutureUtils; +import org.apache.bookkeeper.discover.BookieServiceInfo; import org.apache.bookkeeper.discover.RegistrationClient; import org.apache.bookkeeper.net.BookieId; +import org.apache.bookkeeper.versioning.LongVersion; import org.apache.bookkeeper.versioning.Version; import org.apache.bookkeeper.versioning.Versioned; +import org.apache.pulsar.common.util.FutureUtil; +import org.apache.pulsar.metadata.api.MetadataCache; import org.apache.pulsar.metadata.api.MetadataStore; import org.apache.pulsar.metadata.api.Notification; import org.apache.pulsar.metadata.api.NotificationType; +@Slf4j public class PulsarRegistrationClient implements RegistrationClient { private final MetadataStore store; @@ -47,14 +58,18 @@ public class PulsarRegistrationClient implements RegistrationClient { private final String bookieAllRegistrationPath; private final String bookieReadonlyRegistrationPath; + private final ConcurrentHashMap> bookieServiceInfoCache = + new ConcurrentHashMap(); private final Map writableBookiesWatchers = new ConcurrentHashMap<>(); private final Map readOnlyBookiesWatchers = new ConcurrentHashMap<>(); + private final MetadataCache bookieServiceInfoMetadataCache; private final ScheduledExecutorService executor; public PulsarRegistrationClient(MetadataStore store, String ledgersRootPath) { this.store = store; this.ledgersRootPath = ledgersRootPath; + this.bookieServiceInfoMetadataCache = store.getMetadataCache(BookieServiceInfoSerde.INSTANCE); // Following Bookie Network Address Changes is an expensive operation // as it requires additional ZooKeeper watches @@ -99,7 +114,25 @@ public CompletableFuture>> getReadOnlyBookies() { private CompletableFuture>> getChildren(String path) { return store.getChildren(path) - .thenApply(PulsarRegistrationClient::convertToBookieAddresses) + .thenComposeAsync(children -> { + Set bookieIds = PulsarRegistrationClient.convertToBookieAddresses(children); + Set bookies = convertToBookieAddresses(children); + List>> bookieInfoUpdated = + new ArrayList<>(bookies.size()); + for (BookieId id : bookies) { + // update the cache for new bookies + if (!bookieServiceInfoCache.containsKey(id)) { + bookieInfoUpdated.add(readBookieServiceInfoAsync(id)); + } + } + if (bookieInfoUpdated.isEmpty()) { + return CompletableFuture.completedFuture(bookieIds); + } else { + return FutureUtil + .waitForAll(bookieInfoUpdated) + .thenApply(___ -> bookieIds); + } + }) .thenApply(s -> new Versioned<>(s, Version.NEW)); } @@ -129,10 +162,20 @@ public void unwatchReadOnlyBookies(RegistrationListener registrationListener) { private void updatedBookies(Notification n) { if (n.getType() == NotificationType.Created || n.getType() == NotificationType.Deleted) { + + if (n.getType() == NotificationType.Deleted) { + BookieId bookieId = stripBookieIdFromPath(n.getPath()); + log.info("Bookie {} disappeared", bookieId); + if (bookieId != null) { + bookieServiceInfoCache.remove(bookieId); + } + } + if (n.getPath().startsWith(bookieReadonlyRegistrationPath)) { - getReadOnlyBookies().thenAccept(bookies -> - readOnlyBookiesWatchers.keySet() - .forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies)))); + getReadOnlyBookies().thenAccept(bookies -> { + readOnlyBookiesWatchers.keySet() + .forEach(w -> executor.execute(() -> w.onBookiesChanged(bookies))); + }); } else if (n.getPath().startsWith(bookieRegistrationPath)) { getWritableBookies().thenAccept(bookies -> writableBookiesWatchers.keySet() @@ -141,6 +184,22 @@ private void updatedBookies(Notification n) { } } + private static BookieId stripBookieIdFromPath(String path) { + if (path == null) { + return null; + } + final int slash = path.lastIndexOf('/'); + if (slash >= 0) { + try { + return BookieId.parse(path.substring(slash + 1)); + } catch (IllegalArgumentException e) { + log.warn("Cannot decode bookieId from {}", path, e); + } + } + return null; + } + + private static Set convertToBookieAddresses(List children) { // Read the bookie addresses into a set for efficient lookup HashSet newBookieAddrs = new HashSet<>(); @@ -153,4 +212,56 @@ private static Set convertToBookieAddresses(List children) { } return newBookieAddrs; } + + @Override + public CompletableFuture> getBookieServiceInfo(BookieId bookieId) { + // this method cannot perform blocking calls to the MetadataStore + // or return a CompletableFuture that is completed on the MetadataStore main thread + // this is because there are a few cases in which some operations on the main thread + // wait for the result. This is due to the fact that resolving the address of a bookie + // is needed in many code paths. + Versioned resultFromCache = bookieServiceInfoCache.get(bookieId); + if (log.isDebugEnabled()) { + log.debug("getBookieServiceInfo {} -> {}", bookieId, resultFromCache); + } + if (resultFromCache != null) { + return CompletableFuture.completedFuture(resultFromCache); + } else { + return FutureUtils.exception(new BKException.BKBookieHandleNotAvailableException()); + } + } + + public CompletableFuture> readBookieServiceInfoAsync(BookieId bookieId) { + String asWritable = bookieRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.get(asWritable) + .thenCompose((Optional getResult) -> { + if (getResult.isPresent()) { + Versioned res = + new Versioned<>(getResult.get(), new LongVersion(-1)); + log.info("Update BookieInfoCache (writable bookie) {} -> {}", bookieId, getResult.get()); + bookieServiceInfoCache.put(bookieId, res); + return CompletableFuture.completedFuture(res); + } else { + return readBookieInfoAsReadonlyBookie(bookieId); + } + } + ); + } + + final CompletableFuture> readBookieInfoAsReadonlyBookie(BookieId bookieId) { + String asReadonly = bookieReadonlyRegistrationPath + "/" + bookieId; + return bookieServiceInfoMetadataCache.get(asReadonly) + .thenApply((Optional getResultAsReadOnly) -> { + if (getResultAsReadOnly.isPresent()) { + Versioned res = + new Versioned<>(getResultAsReadOnly.get(), new LongVersion(-1)); + log.info("Update BookieInfoCache (readonly bookie) {} -> {}", bookieId, + getResultAsReadOnly.get()); + bookieServiceInfoCache.put(bookieId, res); + return res; + } else { + throw new CompletionException(new BKException.BKBookieHandleNotAvailableException()); + } + }); + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java index 496cfebea512f..35db268461731 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/bookkeeper/PulsarRegistrationClientTest.java @@ -23,6 +23,8 @@ import static org.testng.Assert.assertFalse; import static org.mockito.Mockito.mock; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -114,6 +116,66 @@ public void testGetReadonlyBookies(String provider, Supplier urlSupplier assertEquals(result.getValue().size(), addresses.size()); } + @Test(dataProvider = "impl") + public void testGetBookieServiceInfo(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStoreExtended store = + MetadataStoreExtended.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + + String ledgersRoot = "/test/ledgers-" + UUID.randomUUID(); + + @Cleanup + RegistrationManager rm = new PulsarRegistrationManager(store, ledgersRoot, mock(AbstractConfiguration.class)); + + @Cleanup + RegistrationClient rc = new PulsarRegistrationClient(store, ledgersRoot); + + List addresses = new ArrayList<>(prepareNBookies(10)); + List bookieServiceInfos = new ArrayList<>(); + int port = 223; + for (BookieId address : addresses) { + BookieServiceInfo info = new BookieServiceInfo(); + BookieServiceInfo.Endpoint endpoint = new BookieServiceInfo.Endpoint(); + endpoint.setAuth(Collections.emptyList()); + endpoint.setExtensions(Collections.emptyList()); + endpoint.setId("id"); + endpoint.setHost("localhost"); + endpoint.setPort(port++); + endpoint.setProtocol("bookie-rpc"); + info.setEndpoints(Arrays.asList(endpoint)); + bookieServiceInfos.add(info); + // some readonly, some writable + boolean readOnly = port % 2 == 0; + rm.registerBookie(address, readOnly, info); + } + + // trigger loading the BookieServiceInfo in the local cache + rc.getAllBookies().join(); + + int i = 0; + for (BookieId address : addresses) { + BookieServiceInfo bookieServiceInfo = rc.getBookieServiceInfo(address).get().getValue(); + compareBookieServiceInfo(bookieServiceInfo, bookieServiceInfos.get(i++)); + } + + } + + private void compareBookieServiceInfo(BookieServiceInfo a, BookieServiceInfo b) { + assertEquals(a.getProperties(), b.getProperties()); + assertEquals(a.getEndpoints().size(), b.getEndpoints().size()); + for (int i = 0; i < a.getEndpoints().size(); i++) { + BookieServiceInfo.Endpoint e1 = a.getEndpoints().get(i); + BookieServiceInfo.Endpoint e2 = b.getEndpoints().get(i); + assertEquals(e1.getHost(), e2.getHost()); + assertEquals(e1.getPort(), e2.getPort()); + assertEquals(e1.getId(), e2.getId()); + assertEquals(e1.getProtocol(), e2.getProtocol()); + assertEquals(e1.getExtensions(), e2.getExtensions()); + assertEquals(e1.getAuth(), e2.getAuth()); + } + + } + @Test(dataProvider = "impl") public void testGetAllBookies(String provider, Supplier urlSupplier) throws Exception { @Cleanup From 9c8a6ad642bfe42c9c236b01543a0ed5d018fd8a Mon Sep 17 00:00:00 2001 From: AlvaroStream <102966649+AlvaroStream@users.noreply.github.com> Date: Mon, 17 Oct 2022 03:17:15 +0200 Subject: [PATCH 18/32] [Improve][doc] Insist about default behaviour for retention. (#17958) --- site2/docs/cookbooks-retention-expiry.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site2/docs/cookbooks-retention-expiry.md b/site2/docs/cookbooks-retention-expiry.md index 1e552fbd4e287..e50beb0fb39d0 100644 --- a/site2/docs/cookbooks-retention-expiry.md +++ b/site2/docs/cookbooks-retention-expiry.md @@ -58,7 +58,7 @@ When a retention limit on a topic is exceeded, the oldest message is marked for ### Defaults -You can set message retention at instance level with the following two parameters: `defaultRetentionTimeInMinutes` and `defaultRetentionSizeInMB`. Both parameters are set to `0` by default. +You can set message retention at instance level with the following two parameters: `defaultRetentionTimeInMinutes` and `defaultRetentionSizeInMB`. By default, both parameters are set to `0`, which means disabling message retention. For more information on the two parameters, refer to the [`broker.conf`](reference-configuration.md#broker) configuration file. @@ -472,4 +472,4 @@ The entry log rollover period is configurable but is purely based on the entry l The diagram below illustrates one of the cases that the consumed storage size is larger than the given limits for backlog and retention. Messages over the retention limit are kept because other messages in the same segment are still within retention period. ![](/assets/retention-storage-size.svg) -If you do not have any retention period and you never have much of a backlog, the upper limit for retained messages, which are acknowledged, equals the Pulsar segment rollover period + entry log rollover period + (garbage collection interval * garbage collection ratios). \ No newline at end of file +If you do not have any retention period and you never have much of a backlog, the upper limit for retained messages, which are acknowledged, equals the Pulsar segment rollover period + entry log rollover period + (garbage collection interval * garbage collection ratios). From a99082255482e8f23546c442b5e67667f81fdc6a Mon Sep 17 00:00:00 2001 From: Xiangying Meng <55571188+liangyepianzhou@users.noreply.github.com> Date: Mon, 17 Oct 2022 09:43:27 +0800 Subject: [PATCH 19/32] [fix][test] Fix TransactionTest failure due to clean up after class (#17746) --- .../broker/transaction/TransactionTest.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java index b50d4e6a00d80..952cd9b2c45a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/transaction/TransactionTest.java @@ -19,6 +19,7 @@ package org.apache.pulsar.broker.transaction; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.pulsar.common.naming.SystemTopicNames.PENDING_ACK_STORE_CURSOR_NAME; import static org.apache.pulsar.common.naming.SystemTopicNames.PENDING_ACK_STORE_SUFFIX; import static org.apache.pulsar.transaction.coordinator.impl.MLTransactionLogImpl.TRANSACTION_LOG_PREFIX; import static org.apache.pulsar.transaction.coordinator.impl.DisabledTxnLogBufferedWriterMetricsStats.DISABLED_BUFFERED_WRITER_METRICS; @@ -406,20 +407,19 @@ public void testGetTxnID() throws Exception { Transaction transaction = pulsarClient.newTransaction() .build().get(); TxnID txnID = transaction.getTxnID(); - Assert.assertEquals(txnID.getLeastSigBits(), 0); - Assert.assertEquals(txnID.getMostSigBits(), 0); + transaction.abort(); transaction = pulsarClient.newTransaction() .build().get(); - txnID = transaction.getTxnID(); - Assert.assertEquals(txnID.getLeastSigBits(), 1); - Assert.assertEquals(txnID.getMostSigBits(), 0); + TxnID txnID1 = transaction.getTxnID(); + Assert.assertEquals(txnID1.getLeastSigBits(), txnID.getLeastSigBits() + 1); + Assert.assertEquals(txnID1.getMostSigBits(), 0); } @Test public void testSubscriptionRecreateTopic() throws PulsarAdminException, NoSuchFieldException, IllegalAccessException, PulsarClientException { - String topic = "persistent://pulsar/system/testReCreateTopic"; + String topic = "persistent://pulsar/system/testSubscriptionRecreateTopic"; String subName = "sub_testReCreateTopic"; int retentionSizeInMbSetTo = 5; int retentionSizeInMbSetTopic = 6; @@ -481,11 +481,7 @@ public void testSubscriptionRecreateTopic() } ); }); - - }); - - } @Test @@ -782,6 +778,8 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ Class pulsarServiceClass = PulsarService.class; Field field = pulsarServiceClass.getDeclaredField("transactionPendingAckStoreProvider"); field.setAccessible(true); + TransactionPendingAckStoreProvider originPendingAckStoreProvider = + (TransactionPendingAckStoreProvider) field.get(getPulsarServiceList().get(0)); field.set(getPulsarServiceList().get(0), pendingAckStoreProvider); PendingAckHandleImpl pendingAckHandle1 = new PendingAckHandleImpl(persistentSubscription); @@ -811,6 +809,7 @@ public void testEndTPRecoveringWhenManagerLedgerDisReadable() throws Exception{ // cleanup transactionTimer.stop(); + field.set(getPulsarServiceList().get(0), originPendingAckStoreProvider); } @Test @@ -1075,7 +1074,7 @@ public void testAutoCreateSchemaForTransactionSnapshot() throws Exception { public void testPendingAckMarkDeletePosition() throws Exception { getPulsarServiceList().get(0).getConfig().setTransactionPendingAckLogIndexMinLag(1); getPulsarServiceList().get(0).getConfiguration().setManagedLedgerDefaultMarkDeleteRateLimit(5); - String topic = NAMESPACE1 + "/test1"; + String topic = NAMESPACE1 + "/testPendingAckMarkDeletePosition"; @Cleanup Producer producer = pulsarClient @@ -1136,7 +1135,7 @@ public void testPendingAckMarkDeletePosition() throws Exception { .getPendingAckInternalStats(topic, "sub", false) .pendingAckLogStats .managedLedgerInternalStats; - String [] markDeletePosition = managedLedgerInternalStats.cursors.get("__pending_ack_state") + String [] markDeletePosition = managedLedgerInternalStats.cursors.get(PENDING_ACK_STORE_CURSOR_NAME) .markDeletePosition.split(":"); String [] lastConfirmedEntry = managedLedgerInternalStats.lastConfirmedEntry.split(":"); Assert.assertEquals(markDeletePosition[0], lastConfirmedEntry[0]); From b530184464ce0afe8b002c14ca62fbd96ea4dfcf Mon Sep 17 00:00:00 2001 From: congbo <39078850+congbobo184@users.noreply.github.com> Date: Mon, 17 Oct 2022 09:44:11 +0800 Subject: [PATCH 20/32] [fix][test] AdvertisedListenersTest.setup (#17869) --- .../mledger/impl/ManagedLedgerBkTest.java | 4 +- .../test/BookKeeperClusterTestCase.java | 9 ++- .../pulsar/broker/MultiBrokerBaseTest.java | 5 ++ .../loadbalance/AdvertisedListenersTest.java | 8 +-- .../SimpleProtocolHandlerTestsBase.java | 13 +++- .../worker/PulsarFunctionTlsTest.java | 11 ++- .../pulsar/common/util/PortManager.java | 69 +++++++++++++++++++ .../pulsar/common/util/PortManagerTest.java | 37 ++++++++++ .../test/BookKeeperClusterTestCase.java | 8 ++- .../SimpleProxyExtensionTestBase.java | 13 +++- 10 files changed, 158 insertions(+), 19 deletions(-) create mode 100644 pulsar-common/src/main/java/org/apache/pulsar/common/util/PortManager.java create mode 100644 pulsar-common/src/test/java/org/apache/pulsar/common/util/PortManagerTest.java diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java index 3ad521eeb79ec..5c42e7f46bda5 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java @@ -18,6 +18,7 @@ */ package org.apache.bookkeeper.mledger.impl; +import static org.apache.pulsar.common.util.PortManager.releaseLockedPort; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; import static org.testng.Assert.assertTrue; @@ -117,7 +118,7 @@ public void testBookieFailure() throws Exception { metadataStore.unsetAlwaysFail(); bkc = new BookKeeperTestClient(baseClientConf); - startNewBookie(); + int port = startNewBookie(); // Reconnect a new bk client factory.shutdown(); @@ -147,6 +148,7 @@ public void testBookieFailure() throws Exception { assertEquals("entry-2", new String(entries.get(0).getData())); entries.forEach(Entry::release); factory.shutdown(); + releaseLockedPort(port); } @Test diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java index e4f1d470bf6de..5518838be37a9 100644 --- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java @@ -24,6 +24,7 @@ package org.apache.bookkeeper.test; import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertFalse; import com.google.common.base.Stopwatch; @@ -62,7 +63,7 @@ import org.apache.bookkeeper.proto.BookieServer; import org.apache.bookkeeper.replication.Auditor; import org.apache.bookkeeper.replication.ReplicationWorker; -import org.apache.bookkeeper.util.PortManager; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.api.MetadataStoreConfig; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.FaultInjectionMetadataStore; @@ -113,6 +114,7 @@ public void handleTestMethodName(Method method) { private boolean isAutoRecoveryEnabled; protected ExecutorService executor; + private final List bookiePorts = new ArrayList<>(); SynchronousQueue asyncExceptions = new SynchronousQueue<>(); protected void captureThrowable(Runnable c) { @@ -264,7 +266,7 @@ protected void startBKCluster(String metadataServiceUri) throws Exception { // Create Bookie Servers (B1, B2, B3) for (int i = 0; i < numBookies; i++) { - startNewBookie(); + bookiePorts.add(startNewBookie()); } } @@ -283,6 +285,7 @@ protected void stopBKCluster() throws Exception { t.shutdown(); } servers.clear(); + bookiePorts.removeIf(PortManager::releaseLockedPort); } protected ServerConfiguration newServerConfiguration() throws Exception { @@ -290,7 +293,7 @@ protected ServerConfiguration newServerConfiguration() throws Exception { int port; if (baseConf.isEnableLocalTransport() || !baseConf.getAllowEphemeralPorts()) { - port = PortManager.nextFreePort(); + port = nextLockedFreePort(); } else { port = 0; } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java index c00ae8cd0d39d..5b78a32dc37e5 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/MultiBrokerBaseTest.java @@ -21,12 +21,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; + import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.client.admin.PulsarAdmin; import org.apache.pulsar.client.admin.PulsarAdminBuilder; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.PulsarClient; import org.apache.pulsar.client.api.PulsarClientException; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.api.MetadataStoreException; import org.apache.pulsar.metadata.api.extended.MetadataStoreExtended; import org.apache.pulsar.metadata.impl.ZKMetadataStore; @@ -124,6 +126,9 @@ protected void additionalBrokersCleanup() { try { pulsarService.getConfiguration().setBrokerShutdownTimeoutMs(0L); pulsarService.close(); + pulsarService.getConfiguration().getBrokerServicePort().ifPresent(PortManager::releaseLockedPort); + pulsarService.getConfiguration().getWebServicePort().ifPresent(PortManager::releaseLockedPort); + pulsarService.getConfiguration().getWebServicePortTls().ifPresent(PortManager::releaseLockedPort); } catch (PulsarServerException e) { // ignore } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java index 489efa5755ba8..9ca4510a209b4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/loadbalance/AdvertisedListenersTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.broker.loadbalance; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -25,7 +26,6 @@ import java.util.Optional; import lombok.Cleanup; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.util.PortManager; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; import org.apache.http.client.methods.CloseableHttpResponse; @@ -66,9 +66,9 @@ protected ServiceConfiguration createConfForAdditionalBroker(int additionalBroke } private void updateConfig(ServiceConfiguration conf, String advertisedAddress) { - int pulsarPort = PortManager.nextFreePort(); - int httpPort = PortManager.nextFreePort(); - int httpsPort = PortManager.nextFreePort(); + int pulsarPort = nextLockedFreePort(); + int httpPort = nextLockedFreePort(); + int httpsPort = nextLockedFreePort(); // Use invalid domain name as identifier and instead make sure the advertised listeners work as intended conf.setAdvertisedAddress(advertisedAddress); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java index 639ccf7ecd01b..f08771934f666 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/protocol/SimpleProtocolHandlerTestsBase.java @@ -22,12 +22,12 @@ import io.netty.channel.*; import io.netty.channel.socket.SocketChannel; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.util.PortManager; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.BrokerTestBase; +import org.apache.pulsar.common.util.PortManager; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; @@ -39,11 +39,14 @@ import java.net.SocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; @Slf4j @@ -54,6 +57,8 @@ public static final class MyProtocolHandler implements ProtocolHandler { private ServiceConfiguration conf; + private final List ports = new ArrayList<>(); + @Override public String protocolName() { return "test"; @@ -81,7 +86,9 @@ public void start(BrokerService service) { @Override public Map> newChannelInitializers() { - return Collections.singletonMap(new InetSocketAddress(conf.getBindAddress(), PortManager.nextFreePort()), + int port = nextLockedFreePort(); + this.ports.add(port); + return Collections.singletonMap(new InetSocketAddress(conf.getBindAddress(), port), new ChannelInitializer() { @Override protected void initChannel(SocketChannel socketChannel) throws Exception { @@ -106,7 +113,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { @Override public void close() { - + ports.removeIf(PortManager::releaseLockedPort); } } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java index 3aaf914b9eb54..bfc9928a7dec7 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/functions/worker/PulsarFunctionTlsTest.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.worker; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertEquals; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -33,7 +34,6 @@ import java.util.Optional; import java.util.Set; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.util.PortManager; import org.apache.pulsar.broker.PulsarService; import org.apache.pulsar.broker.ServiceConfiguration; import org.apache.pulsar.broker.authentication.AuthenticationProviderTls; @@ -44,6 +44,7 @@ import org.apache.pulsar.common.policies.data.TenantInfo; import org.apache.pulsar.common.util.ClassLoaderUtils; import org.apache.pulsar.common.util.ObjectMapperFactory; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.functions.api.utils.IdentityFunction; import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactory; import org.apache.pulsar.functions.runtime.thread.ThreadRuntimeFactoryConfig; @@ -86,8 +87,8 @@ void setup() throws Exception { // start brokers for (int i = 0; i < BROKER_COUNT; i++) { - int brokerPort = PortManager.nextFreePort(); - int webPort = PortManager.nextFreePort(); + int brokerPort = nextLockedFreePort(); + int webPort = nextLockedFreePort(); ServiceConfiguration config = new ServiceConfiguration(); config.setBrokerShutdownTimeoutMs(0L); @@ -196,6 +197,10 @@ void tearDown() throws Exception { for (int i = 0; i < BROKER_COUNT; i++) { if (pulsarServices[i] != null) { pulsarServices[i].close(); + pulsarServices[i].getConfiguration(). + getBrokerServicePort().ifPresent(PortManager::releaseLockedPort); + pulsarServices[i].getConfiguration() + .getWebServicePort().ifPresent(PortManager::releaseLockedPort); } } bkEnsemble.stop(); diff --git a/pulsar-common/src/main/java/org/apache/pulsar/common/util/PortManager.java b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PortManager.java new file mode 100644 index 0000000000000..b9df071fdbc7e --- /dev/null +++ b/pulsar-common/src/main/java/org/apache/pulsar/common/util/PortManager.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import java.net.ServerSocket; +import java.util.HashSet; +import java.util.Set; + +public class PortManager { + + private static final Set PORTS = new HashSet<>(); + + /** + * Return a locked available port. + * + * @return locked available port. + */ + public static synchronized int nextLockedFreePort() { + int exceptionCount = 0; + while (true) { + try (ServerSocket ss = new ServerSocket(0)) { + int port = ss.getLocalPort(); + if (!checkPortIfLocked(port)) { + PORTS.add(port); + return port; + } + } catch (Exception e) { + exceptionCount++; + if (exceptionCount > 100) { + throw new RuntimeException("Unable to allocate socket port", e); + } + } + } + } + + /** + * Returns whether the port was released successfully. + * + * @return whether the release is successful. + */ + public static synchronized boolean releaseLockedPort(int lockedPort) { + return PORTS.remove(lockedPort); + } + + /** + * Check port if locked. + * + * @return whether the port is locked. + */ + public static synchronized boolean checkPortIfLocked(int lockedPort) { + return PORTS.contains(lockedPort); + } +} diff --git a/pulsar-common/src/test/java/org/apache/pulsar/common/util/PortManagerTest.java b/pulsar-common/src/test/java/org/apache/pulsar/common/util/PortManagerTest.java new file mode 100644 index 0000000000000..88057ba943ab2 --- /dev/null +++ b/pulsar-common/src/test/java/org/apache/pulsar/common/util/PortManagerTest.java @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pulsar.common.util; + +import org.testng.annotations.Test; + +import static org.apache.pulsar.common.util.PortManager.checkPortIfLocked; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; +import static org.apache.pulsar.common.util.PortManager.releaseLockedPort; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; + +public class PortManagerTest { + @Test + public void testCheckPortIfLockedAndRemove() { + int port = nextLockedFreePort(); + assertTrue(checkPortIfLocked(port)); + assertTrue(releaseLockedPort(port)); + assertFalse(checkPortIfLocked(port)); + } +} diff --git a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java index 7921e784f60e7..327562c77de4c 100644 --- a/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java +++ b/pulsar-package-management/bookkeeper-storage/src/test/java/org/apache/pulsar/packages/management/storage/bookkeeper/bookkeeper/test/BookKeeperClusterTestCase.java @@ -27,6 +27,7 @@ import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_INDEX_SCOPE; import static org.apache.bookkeeper.bookie.BookKeeperServerStats.LD_LEDGER_SCOPE; import static org.apache.bookkeeper.util.BookKeeperConstants.AVAILABLE_NODE; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.testng.Assert.assertFalse; import com.google.common.base.Stopwatch; import java.io.File; @@ -83,7 +84,7 @@ import org.apache.bookkeeper.test.ZooKeeperCluster; import org.apache.bookkeeper.test.ZooKeeperClusterUtil; import org.apache.bookkeeper.util.DiskChecker; -import org.apache.bookkeeper.util.PortManager; +import org.apache.pulsar.common.util.PortManager; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; import org.slf4j.Logger; @@ -129,6 +130,8 @@ public void handleTestMethodName(Method method) { private boolean isAutoRecoveryEnabled; + private final List bookiePorts = new ArrayList<>(); + SynchronousQueue asyncExceptions = new SynchronousQueue<>(); protected void captureThrowable(Runnable c) { try { @@ -282,6 +285,7 @@ protected void stopBKCluster() throws Exception { t.shutdown(); } servers.clear(); + bookiePorts.removeIf(PortManager::releaseLockedPort); } protected ServerConfiguration newServerConfiguration() throws Exception { @@ -289,7 +293,7 @@ protected ServerConfiguration newServerConfiguration() throws Exception { int port; if (baseConf.isEnableLocalTransport() || !baseConf.getAllowEphemeralPorts()) { - port = PortManager.nextFreePort(); + port = nextLockedFreePort(); } else { port = 0; } diff --git a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java index 0cc4bbb6bb500..c779acb6ebe90 100644 --- a/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java +++ b/pulsar-proxy/src/test/java/org/apache/pulsar/proxy/extensions/SimpleProxyExtensionTestBase.java @@ -22,12 +22,12 @@ import io.netty.channel.*; import io.netty.channel.socket.SocketChannel; import lombok.extern.slf4j.Slf4j; -import org.apache.bookkeeper.util.PortManager; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.authentication.AuthenticationService; import org.apache.pulsar.common.configuration.PulsarConfigurationLoader; +import org.apache.pulsar.common.util.PortManager; import org.apache.pulsar.metadata.impl.ZKMetadataStore; import org.apache.pulsar.proxy.server.ProxyConfiguration; import org.apache.pulsar.proxy.server.ProxyService; @@ -43,12 +43,15 @@ import java.net.SocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import static org.apache.pulsar.common.util.PortManager.nextLockedFreePort; import static org.mockito.Mockito.doReturn; import static org.testng.Assert.assertEquals; @@ -60,6 +63,8 @@ public static final class MyProxyExtension implements ProxyExtension { private ProxyConfiguration conf; + private final List ports = new ArrayList<>(); + @Override public String extensionName() { return "test"; @@ -81,7 +86,9 @@ public void start(ProxyService service) { @Override public Map> newChannelInitializers() { - return Collections.singletonMap(new InetSocketAddress(conf.getBindAddress(), PortManager.nextFreePort()), + int port = nextLockedFreePort(); + this.ports.add(port); + return Collections.singletonMap(new InetSocketAddress(conf.getBindAddress(), port), new ChannelInitializer() { @Override protected void initChannel(SocketChannel socketChannel) throws Exception { @@ -106,7 +113,7 @@ public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { @Override public void close() { - + ports.removeIf(PortManager::releaseLockedPort); } } From 96cc2403c8797f6fbe022c71c166337069907412 Mon Sep 17 00:00:00 2001 From: Shen Liu Date: Mon, 17 Oct 2022 10:59:03 +0800 Subject: [PATCH 21/32] [improve][test] Add test case for system topic schema not compatible bug. (#17992) Co-authored-by: druidliu --- .../NamespaceEventsSystemTopicServiceTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java index 58eb70fb4c794..91db232d2aa4e 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/systopic/NamespaceEventsSystemTopicServiceTest.java @@ -18,12 +18,15 @@ */ package org.apache.pulsar.broker.systopic; +import static org.mockito.Mockito.mock; import com.google.common.collect.Sets; import java.util.HashSet; import lombok.Cleanup; +import org.apache.bookkeeper.mledger.ManagedLedger; import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest; import org.apache.pulsar.broker.service.BrokerService; import org.apache.pulsar.broker.service.persistent.PersistentTopic; +import org.apache.pulsar.broker.service.persistent.SystemTopic; import org.apache.pulsar.client.admin.PulsarAdminException; import org.apache.pulsar.client.api.Message; import org.apache.pulsar.client.api.MessageId; @@ -90,6 +93,16 @@ public void testSchemaCompatibility() throws Exception { Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); } + @Test + public void testSystemTopicSchemaCompatibility() throws Exception { + TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory + .createTopicPoliciesSystemTopicClient(NamespaceName.get(NAMESPACE1)); + String topicName = systemTopicClientForNamespace1.getTopicName().toString(); + SystemTopic topic = new SystemTopic(topicName, mock(ManagedLedger.class), pulsar.getBrokerService()); + + Assert.assertEquals(SchemaCompatibilityStrategy.ALWAYS_COMPATIBLE, topic.getSchemaCompatibilityStrategy()); + } + @Test public void testSendAndReceiveNamespaceEvents() throws Exception { TopicPoliciesSystemTopicClient systemTopicClientForNamespace1 = systemTopicFactory From 796afff2684b069228ad7d45b990af4838ee259e Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Mon, 17 Oct 2022 11:45:37 +0800 Subject: [PATCH 22/32] [improve][test] Improve SubscriptionMessageDispatchThrottlingTest to reduce the execution time (#18039) --- .../api/MessageDispatchThrottlingTest.java | 41 ++++++++++++++++--- ...criptionMessageDispatchThrottlingTest.java | 22 ++++++---- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java index d024d43eb73db..d3e5067ed3123 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/MessageDispatchThrottlingTest.java @@ -53,8 +53,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; +import org.testng.annotations.AfterClass; import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -62,7 +63,7 @@ public class MessageDispatchThrottlingTest extends ProducerConsumerBase { private static final Logger log = LoggerFactory.getLogger(MessageDispatchThrottlingTest.class); - @BeforeMethod + @BeforeClass @Override protected void setup() throws Exception { this.conf.setClusterName("test"); @@ -70,12 +71,35 @@ protected void setup() throws Exception { super.producerBaseSetup(); } - @AfterMethod(alwaysRun = true) + @AfterClass(alwaysRun = true) @Override protected void cleanup() throws Exception { super.internalCleanup(); } + @AfterMethod(alwaysRun = true) + protected void reset() throws Exception { + pulsar.getConfiguration().setForceDeleteTenantAllowed(true); + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(true); + + for (String tenant : admin.tenants().getTenants()) { + for (String namespace : admin.namespaces().getNamespaces(tenant)) { + admin.namespaces().deleteNamespace(namespace, true); + } + admin.tenants().deleteTenant(tenant, true); + } + + for (String cluster : admin.clusters().getClusters()) { + admin.clusters().deleteCluster(cluster); + } + + pulsar.getConfiguration().setForceDeleteTenantAllowed(false); + pulsar.getConfiguration().setForceDeleteNamespaceAllowed(false); + + super.producerBaseSetup(); + } + + @DataProvider(name = "subscriptions") public Object[][] subscriptionsProvider() { return new Object[][] { new Object[] { SubscriptionType.Shared }, { SubscriptionType.Exclusive } }; @@ -280,6 +304,7 @@ public void testClusterMsgByteRateLimitingClusterConfig() throws Exception { final long byteRate = 1024 * 1024;// 1MB rate enough to let all msg to be delivered int initValue = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInMsg(); + long initBytes = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInByte(); // (1) Update message-dispatch-rate limit admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", Integer.toString(messageRate)); @@ -325,7 +350,9 @@ public void testClusterMsgByteRateLimitingClusterConfig() throws Exception { consumer.close(); producer.close(); - pulsar.getConfiguration().setDispatchThrottlingRatePerTopicInMsg(initValue); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", + Integer.toString(initValue)); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInByte", Long.toString(initBytes)); log.info("-- Exiting {} test --", methodName); } @@ -675,7 +702,8 @@ public void testClusterRateLimitingConfiguration(SubscriptionType subscription) consumer.close(); producer.close(); - pulsar.getConfiguration().setDispatchThrottlingRatePerTopicInMsg(initValue); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", + Integer.toString(initValue)); log.info("-- Exiting {} test --", methodName); } @@ -981,7 +1009,8 @@ public void testClusterPolicyOverrideConfiguration() throws Exception { producer.close(); producer2.close(); - + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerTopicInMsg", + Integer.toString(initValue)); log.info("-- Exiting {} test --", methodName); } diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java index 7881038cc7aaf..642bec7321edd 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/api/SubscriptionMessageDispatchThrottlingTest.java @@ -18,12 +18,11 @@ */ package org.apache.pulsar.client.api; +import static org.awaitility.Awaitility.await; import com.google.common.collect.Sets; - import java.time.Duration; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; - import org.apache.pulsar.broker.BrokerTestUtil; import org.apache.pulsar.broker.service.Dispatcher; import org.apache.pulsar.broker.service.persistent.DispatchRateLimiter; @@ -38,8 +37,6 @@ import org.testng.Assert; import org.testng.annotations.Test; -import static org.awaitility.Awaitility.await; - @Test(groups = "flaky") public class SubscriptionMessageDispatchThrottlingTest extends MessageDispatchThrottlingTest { private static final Logger log = LoggerFactory.getLogger(SubscriptionMessageDispatchThrottlingTest.class); @@ -243,6 +240,7 @@ private void testMessageNotDuplicated(SubscriptionType subscription) throws Exce admin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); admin.namespaces().setSubscriptionDispatchRate(namespace, subscriptionDispatchRate); admin.namespaces().setDispatchRate(namespace, topicDispatchRate); + long initBytes = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInByte(); admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", "" + brokerRate); final int numProducedMessages = 30; @@ -302,6 +300,9 @@ private void testMessageNotDuplicated(SubscriptionType subscription) throws Exce consumer.close(); producer.close(); + + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", Long.toString(initBytes)); + admin.topics().delete(topicName, true); admin.namespaces().deleteNamespace(namespace); } @@ -417,6 +418,7 @@ private void testDispatchRate(SubscriptionType subscription, admin.namespaces().createNamespace(namespace, Sets.newHashSet("test")); admin.namespaces().setSubscriptionDispatchRate(namespace, subscriptionDispatchRate); admin.namespaces().setDispatchRate(namespace, topicDispatchRate); + long initBytes = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInByte(); admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", "" + brokerRate); final int numProducedMessages = 30; @@ -480,6 +482,7 @@ private void testDispatchRate(SubscriptionType subscription, consumer.close(); producer.close(); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", Long.toString(initBytes)); admin.topics().delete(topicName, true); admin.namespaces().deleteNamespace(namespace); } @@ -532,6 +535,7 @@ public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(Subscri final String topicName2 = BrokerTestUtil.newUniqueName("persistent://" + namespace2 + "/throttlingAll"); final String subName = "my-subscriber-name-" + subscription; + long initBytes = pulsar.getConfiguration().getDispatchThrottlingRatePerTopicInByte(); final int byteRate = 1000; admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", "" + byteRate); admin.namespaces().createNamespace(namespace1, Sets.newHashSet("test")); @@ -591,6 +595,7 @@ public void testBrokerBytesRateLimitingReceiveAllMessagesAfterThrottling(Subscri consumer2.close(); producer1.close(); producer2.close(); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRateInByte", Long.toString(initBytes)); log.info("-- Exiting {} test --", methodName); } @@ -739,7 +744,9 @@ public void testClusterRateLimitingConfiguration(SubscriptionType subscription) consumer.close(); producer.close(); - pulsar.getConfiguration().setDispatchThrottlingRatePerSubscriptionInMsg(initValue); + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerSubscriptionInMsg", + Integer.toString(initValue)); + conf.setDispatchThrottlingOnNonBacklogConsumerEnabled(false); log.info("-- Exiting {} test --", methodName); } @@ -855,11 +862,12 @@ public void testClusterPolicyOverrideConfiguration() throws Exception { producer.close(); producer2.close(); - + admin.brokers().updateDynamicConfiguration("dispatchThrottlingRatePerSubscriptionInMsg", + Integer.toString(initValue)); log.info("-- Exiting {} test --", methodName); } - @Test(dataProvider = "subscriptions", timeOut = 10000) + @Test(dataProvider = "subscriptions", timeOut = 11000) public void testClosingRateLimiter(SubscriptionType subscription) throws Exception { log.info("-- Starting {} test --", methodName); From 44215012ae74671674aee8b83c0a4522c309f421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Oct 2022 09:02:56 +0200 Subject: [PATCH 23/32] [fix][sec] Upgrade JacksonXML to 2.13.4 (#18020) --- .../server/src/assemble/LICENSE.bin.txt | 16 +++++------ pom.xml | 2 +- pulsar-sql/presto-distribution/LICENSE | 28 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index 19c2a014e94e3..2920c15c8d142 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -246,14 +246,14 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.82.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.9.1.jar * Jackson - - com.fasterxml.jackson.core-jackson-annotations-2.13.3.jar - - com.fasterxml.jackson.core-jackson-core-2.13.3.jar - - com.fasterxml.jackson.core-jackson-databind-2.13.3.jar - - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.3.jar - - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.3.jar - - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.3.jar - - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.3.jar + - com.fasterxml.jackson.core-jackson-annotations-2.13.4.jar + - com.fasterxml.jackson.core-jackson-core-2.13.4.jar + - com.fasterxml.jackson.core-jackson-databind-2.13.4.jar + - com.fasterxml.jackson.dataformat-jackson-dataformat-yaml-2.13.4.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-base-2.13.4.jar + - com.fasterxml.jackson.jaxrs-jackson-jaxrs-json-provider-2.13.4.jar + - com.fasterxml.jackson.module-jackson-module-jaxb-annotations-2.13.4.jar + - com.fasterxml.jackson.module-jackson-module-jsonSchema-2.13.4.jar * Caffeine -- com.github.ben-manes.caffeine-caffeine-2.9.1.jar * Conscrypt -- org.conscrypt-conscrypt-openjdk-uber-2.5.2.jar * Proto Google Common Protos -- com.google.api.grpc-proto-google-common-protos-2.0.1.jar diff --git a/pom.xml b/pom.xml index 2dea7306703a4..d0c9cbf7b0add 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ flexible messaging model and an intuitive client API. 2.18.0 1.69 1.0.2 - 2.13.3 + 2.13.4 0.9.11 1.6.2 8.37 diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 13d50e07c5ebd..9ec3679f87e00 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -207,19 +207,19 @@ This projects includes binary packages with the following licenses: The Apache Software License, Version 2.0 * Jackson - - jackson-annotations-2.13.3.jar - - jackson-core-2.13.3.jar - - jackson-databind-2.13.3.jar - - jackson-dataformat-smile-2.13.3.jar - - jackson-datatype-guava-2.13.3.jar - - jackson-datatype-jdk8-2.13.3.jar - - jackson-datatype-joda-2.13.3.jar - - jackson-datatype-jsr310-2.13.3.jar - - jackson-dataformat-yaml-2.13.3.jar - - jackson-jaxrs-base-2.13.3.jar - - jackson-jaxrs-json-provider-2.13.3.jar - - jackson-module-jaxb-annotations-2.13.3.jar - - jackson-module-jsonSchema-2.13.3.jar + - jackson-annotations-2.13.4.jar + - jackson-core-2.13.4.jar + - jackson-databind-2.13.4.jar + - jackson-dataformat-smile-2.13.4.jar + - jackson-datatype-guava-2.13.4.jar + - jackson-datatype-jdk8-2.13.4.jar + - jackson-datatype-joda-2.13.4.jar + - jackson-datatype-jsr310-2.13.4.jar + - jackson-dataformat-yaml-2.13.4.jar + - jackson-jaxrs-base-2.13.4.jar + - jackson-jaxrs-json-provider-2.13.4.jar + - jackson-module-jaxb-annotations-2.13.4.jar + - jackson-module-jsonSchema-2.13.4.jar * Guava - guava-31.0.1-jre.jar - listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar @@ -445,7 +445,7 @@ The Apache Software License, Version 2.0 * Snappy - snappy-java-1.1.8.4.jar * Jackson - - jackson-module-parameter-names-2.13.3.jar + - jackson-module-parameter-names-2.13.4.jar * Java Assist - javassist-3.25.0-GA.jar * Java Native Access From ffbcdc0dfbfded63301e92e1c37ae9c6d46741cf Mon Sep 17 00:00:00 2001 From: AlvaroStream <102966649+AlvaroStream@users.noreply.github.com> Date: Mon, 17 Oct 2022 09:09:05 +0200 Subject: [PATCH 24/32] [improve][doc] Add options to stats and partitioned-stats (#17734) --- site2/docs/reference-pulsar-admin.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/site2/docs/reference-pulsar-admin.md b/site2/docs/reference-pulsar-admin.md index a2d017a1697a9..cbd4005eaa30a 100644 --- a/site2/docs/reference-pulsar-admin.md +++ b/site2/docs/reference-pulsar-admin.md @@ -2429,6 +2429,15 @@ Usage pulsar-admin topics stats topic ``` +Options + +|Flag|Description|Default| +|---|---|---| +|`-etb`,`--get-earliest-time-in-backlog` | Set to `true` to get the earliest time in backlog | false | +|`-gpb`,`--get-precise-backlog`| Set to `true` to get precise backlog | false | +|`-sbs`,` --get-subscription-backlog-size`| Set to `true` to get backlog size for each subscription, locking required | false | + + :::note The unit of `storageSize` and `averageMsgSize` is Byte. @@ -2466,8 +2475,11 @@ Options |Flag|Description|Default| |---|---|---| +|`-gpb`,`--get-precise-backlog`| Set to `true` to get precise backlog | false | +|`-sbs`,` --get-subscription-backlog-size`| Set to `true` to get backlog size for each subscription, locking required | false | |`--per-partition`|Get per-partition stats|false| + ### `partitioned-stats-internal` Get the internal stats for the partitioned topic and its connected producers and consumers. All the rates are computed over a 1 minute window and are relative the last completed 1 minute period. @@ -2725,6 +2737,7 @@ pulsar-admin topics remove-message-ttl tenant/namespace/topic ``` Options + |Flag|Description|Default| |---|---|---| |`--enable`, `-e`|Enable message deduplication on the specified topic.|false| From 4b5de98610fe491b62c47081bfe82a7a8fddc5fb Mon Sep 17 00:00:00 2001 From: Lan Date: Mon, 17 Oct 2022 15:11:40 +0800 Subject: [PATCH 25/32] [cleanup][broker] Delete unuse metrics of zk_write_latency and zk_read_latency (#17505) --- .../stats/BrokerOperabilityMetrics.java | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java index ac131121d0b79..3eb52c59a8b7a 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/BrokerOperabilityMetrics.java @@ -32,8 +32,6 @@ public class BrokerOperabilityMetrics { private final List metricsList; private final String localCluster; private final DimensionStats topicLoadStats; - private final DimensionStats zkWriteLatencyStats; - private final DimensionStats zkReadLatencyStats; private final String brokerName; private final LongAdder connectionTotalCreatedCount; private final LongAdder connectionCreateSuccessCount; @@ -45,8 +43,6 @@ public BrokerOperabilityMetrics(String localCluster, String brokerName) { this.metricsList = new ArrayList<>(); this.localCluster = localCluster; this.topicLoadStats = new DimensionStats("topic_load_times", 60); - this.zkWriteLatencyStats = new DimensionStats("zk_write_latency", 60); - this.zkReadLatencyStats = new DimensionStats("zk_read_latency", 60); this.brokerName = brokerName; this.connectionTotalCreatedCount = new LongAdder(); this.connectionCreateSuccessCount = new LongAdder(); @@ -62,8 +58,6 @@ public List getMetrics() { private void generate() { metricsList.add(getTopicLoadMetrics()); - metricsList.add(getZkWriteLatencyMetrics()); - metricsList.add(getZkReadLatencyMetrics()); metricsList.add(getConnectionMetrics()); } @@ -93,14 +87,6 @@ Metrics getTopicLoadMetrics() { return getDimensionMetrics("topic_load_times", "topic_load", topicLoadStats); } - Metrics getZkWriteLatencyMetrics() { - return getDimensionMetrics("zk_write_latency", "zk_write", zkWriteLatencyStats); - } - - Metrics getZkReadLatencyMetrics() { - return getDimensionMetrics("zk_read_latency", "zk_read", zkReadLatencyStats); - } - Metrics getDimensionMetrics(String metricsName, String dimensionName, DimensionStats stats) { Metrics dMetrics = Metrics.create(getDimensionMap(metricsName)); @@ -120,22 +106,12 @@ Metrics getDimensionMetrics(String metricsName, String dimensionName, DimensionS public void reset() { metricsList.clear(); topicLoadStats.reset(); - zkWriteLatencyStats.reset(); - zkReadLatencyStats.reset(); } public void recordTopicLoadTimeValue(long topicLoadLatencyMs) { topicLoadStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS); } - public void recordZkWriteLatencyTimeValue(long topicLoadLatencyMs) { - zkWriteLatencyStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS); - } - - public void recordZkReadLatencyTimeValue(long topicLoadLatencyMs) { - zkReadLatencyStats.recordDimensionTimeValue(topicLoadLatencyMs, TimeUnit.MILLISECONDS); - } - public void recordConnectionCreate() { this.connectionTotalCreatedCount.increment(); this.connectionActive.increment(); From 574f78419b0366560c69d8b7a07825ff3f5c6f81 Mon Sep 17 00:00:00 2001 From: Ayman Khalil Date: Mon, 17 Oct 2022 00:46:51 -0700 Subject: [PATCH 26/32] [improve][fn] Run connectors extraction in parallel (#17902) --- .../functions/utils/io/ConnectorUtils.java | 104 ++++++++++++------ 1 file changed, 72 insertions(+), 32 deletions(-) diff --git a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java index a4c74d0cdfdd0..a1835856f0b37 100644 --- a/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java +++ b/pulsar-functions/utils/src/main/java/org/apache/pulsar/functions/utils/io/ConnectorUtils.java @@ -18,6 +18,7 @@ */ package org.apache.pulsar.functions.utils.io; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; import java.lang.annotation.Annotation; @@ -27,11 +28,17 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -39,6 +46,7 @@ import org.apache.pulsar.common.io.ConnectorDefinition; import org.apache.pulsar.common.nar.NarClassLoader; import org.apache.pulsar.common.nar.NarClassLoaderBuilder; +import org.apache.pulsar.common.util.FutureUtil; import org.apache.pulsar.common.util.ObjectMapperFactory; import org.apache.pulsar.common.util.Reflections; import org.apache.pulsar.functions.utils.Exceptions; @@ -142,49 +150,81 @@ public static TreeMap searchForConnectors(String connectorsDi Path path = Paths.get(connectorsDirectory).toAbsolutePath(); log.info("Searching for connectors in {}", path); - TreeMap connectors = new TreeMap<>(); if (!path.toFile().exists()) { log.warn("Connectors archive directory not found"); - return connectors; + return new TreeMap<>(); } + List archives = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(path, "*.nar")) { for (Path archive : stream) { - try { - - NarClassLoader ncl = NarClassLoaderBuilder.builder() - .narFile(new File(archive.toString())) - .extractionDirectory(narExtractionDirectory) - .build(); - - Connector.ConnectorBuilder connectorBuilder = Connector.builder(); - ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl); - log.info("Found connector {} from {}", cntDef, archive); - - connectorBuilder.archivePath(archive); - if (!StringUtils.isEmpty(cntDef.getSourceClass())) { - if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) { - connectorBuilder.sourceConfigFieldDefinitions(ConnectorUtils - .getConnectorConfigDefinition(ncl, cntDef.getSourceConfigClass())); - } - } + archives.add(archive); + } + } + if (archives.isEmpty()) { + return new TreeMap<>(); + } - if (!StringUtils.isEmpty(cntDef.getSinkClass())) { - if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) { - connectorBuilder.sinkConfigFieldDefinitions( - ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass())); - } - } + ExecutorService oneTimeExecutor = null; + try { + int nThreads = Math.min(Runtime.getRuntime().availableProcessors(), archives.size()); + log.info("Loading {} connector definitions with a thread pool of size {}", archives.size(), nThreads); + oneTimeExecutor = Executors.newFixedThreadPool(nThreads, + new ThreadFactoryBuilder().setNameFormat("connector-extraction-executor-%d").build()); + List>> futures = new ArrayList<>(); + for (Path archive : archives) { + CompletableFuture> future = CompletableFuture.supplyAsync(() -> + getConnectorDefinitionEntry(archive, narExtractionDirectory), oneTimeExecutor); + futures.add(future); + } - connectorBuilder.classLoader(ncl); - connectorBuilder.connectorDefinition(cntDef); - connectors.put(cntDef.getName(), connectorBuilder.build()); - } catch (Throwable t) { - log.warn("Failed to load connector from {}", archive, t); + FutureUtil.waitForAll(futures).join(); + return futures.stream() + .map(CompletableFuture::join) + .filter(entry -> entry != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> a, TreeMap::new)); + } finally { + if (oneTimeExecutor != null) { + oneTimeExecutor.shutdown(); + } + } + } + + private static Map.Entry getConnectorDefinitionEntry(Path archive, + String narExtractionDirectory) { + try { + + NarClassLoader ncl = NarClassLoaderBuilder.builder() + .narFile(new File(archive.toString())) + .extractionDirectory(narExtractionDirectory) + .build(); + + Connector.ConnectorBuilder connectorBuilder = Connector.builder(); + ConnectorDefinition cntDef = ConnectorUtils.getConnectorDefinition(ncl); + log.info("Found connector {} from {}", cntDef, archive); + + connectorBuilder.archivePath(archive); + if (!StringUtils.isEmpty(cntDef.getSourceClass())) { + if (!StringUtils.isEmpty(cntDef.getSourceConfigClass())) { + connectorBuilder.sourceConfigFieldDefinitions( + ConnectorUtils.getConnectorConfigDefinition(ncl, + cntDef.getSourceConfigClass())); } } - return connectors; + if (!StringUtils.isEmpty(cntDef.getSinkClass())) { + if (!StringUtils.isEmpty(cntDef.getSinkConfigClass())) { + connectorBuilder.sinkConfigFieldDefinitions( + ConnectorUtils.getConnectorConfigDefinition(ncl, cntDef.getSinkConfigClass())); + } + } + + connectorBuilder.classLoader(ncl); + connectorBuilder.connectorDefinition(cntDef); + return new AbstractMap.SimpleEntry(cntDef.getName(), connectorBuilder.build()); + } catch (Throwable t) { + log.warn("Failed to load connector from {}", archive, t); + return null; } } } From 8f44c1a4bae8442b8e25ab7fbccbfbb2917d9322 Mon Sep 17 00:00:00 2001 From: Jiwei Guo Date: Mon, 17 Oct 2022 17:45:35 +0800 Subject: [PATCH 27/32] [fix][admin] Fix NPE when get OffloadThreshold on namespace (#18061) --- .../java/org/apache/pulsar/broker/admin/v2/Namespaces.java | 3 ++- .../java/org/apache/pulsar/broker/admin/NamespacesTest.java | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java index dace261ceb6ae..9b07427c72d49 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/v2/Namespaces.java @@ -2054,7 +2054,8 @@ public void getOffloadThreshold( validateNamespacePolicyOperationAsync(namespaceName, PolicyName.OFFLOAD, PolicyOperation.READ) .thenCompose(__ -> getNamespacePoliciesAsync(namespaceName)) .thenAccept(policies -> { - if (policies.offload_policies == null) { + if (policies.offload_policies == null + || policies.offload_policies.getManagedLedgerOffloadThresholdInBytes() == null) { asyncResponse.resume(policies.offload_threshold); } else { asyncResponse.resume(policies.offload_policies.getManagedLedgerOffloadThresholdInBytes()); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java index 0d2b8808ed76b..a4f9fc3f52f19 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java @@ -1401,6 +1401,9 @@ public void testSetOffloadThreshold() throws Exception { admin.namespaces().createNamespace(namespace, Set.of(testLocalCluster)); admin.topics().createNonPartitionedTopic(topicName.toString()); + admin.namespaces().setOffloadDeleteLag(namespace, 10000, TimeUnit.SECONDS); + assertEquals(-1, admin.namespaces().getOffloadThreshold(namespace)); + // assert we get the default which indicates it will fall back to default assertEquals(-1, admin.namespaces().getOffloadThreshold(namespace)); // the ledger config should have the expected value From 63cdb5472ddd353cdba2b23076c58dc54920c0e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Oct 2022 11:59:08 +0200 Subject: [PATCH 28/32] [fix][ci] Upload to codecov only upstream pull requests (#18018) --- .github/workflows/pulsar-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index a58fef08c15a5..c7198b04917ec 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -218,6 +218,7 @@ jobs: uses: ./.github/actions/copy-test-reports - name: Upload to Codecov + if: ${{ github.repository == 'apache/pulsar' }} uses: codecov/codecov-action@v3 continue-on-error: true with: From 0c7a0d144d9e527fbd25a1d738715934fbcf38ab Mon Sep 17 00:00:00 2001 From: Anonymitaet <50226895+Anonymitaet@users.noreply.github.com> Date: Mon, 17 Oct 2022 19:00:27 +0800 Subject: [PATCH 29/32] [feat][doc] Separate CLI docs for doc gen automation (#18051) --- site2/docs/reference-cli-bookkeeper.md | 106 ++ site2/docs/reference-cli-broker-tool.md | 36 + site2/docs/reference-cli-pulsar-daemon.md | 48 + site2/docs/reference-cli-pulsar-shell.md | 155 +++ site2/docs/reference-cli-tools.md | 1069 +-------------------- 5 files changed, 357 insertions(+), 1057 deletions(-) create mode 100644 site2/docs/reference-cli-bookkeeper.md create mode 100644 site2/docs/reference-cli-broker-tool.md create mode 100644 site2/docs/reference-cli-pulsar-daemon.md create mode 100644 site2/docs/reference-cli-pulsar-shell.md diff --git a/site2/docs/reference-cli-bookkeeper.md b/site2/docs/reference-cli-bookkeeper.md new file mode 100644 index 0000000000000..2f6cf6ac7e85c --- /dev/null +++ b/site2/docs/reference-cli-bookkeeper.md @@ -0,0 +1,106 @@ +--- +id: reference-cli-bookkeeper +title: Pulsar CLI tools - bookkeeper +sidebar_label: "cli-bookkeeper" +--- + +## `bookkeeper` +A tool for managing BookKeeper. + +Usage + +```bash +bookkeeper command +``` + +Environment variables + +The table below lists the environment variables that you can use to configure the bookkeeper tool. + +|Variable|Description|Default| +|---|---|---| +|BOOKIE_LOG_CONF|Log4j configuration file|conf/log4j2.yaml| +|BOOKIE_CONF|BookKeeper configuration file|conf/bk_server.conf| +|BOOKIE_EXTRA_OPTS|Extra options to be passed to the JVM|| +|BOOKIE_EXTRA_CLASSPATH|Extra paths for BookKeeper's classpath|| +|ENTRY_FORMATTER_CLASS|The Java class used to format entries|| +|BOOKIE_PID_DIR|Folder where the BookKeeper server PID file should be stored|| +|BOOKIE_STOP_TIMEOUT|Wait time before forcefully killing the Bookie server instance if attempts to stop it are not successful|| +|BOOKIE_GC_LOG|Gc options to be passed to the jvm|| + + +### `autorecovery` +Runs an auto-recovery service + +Usage + +```bash +bookkeeper autorecovery options +``` + +Options + +|Flag|Description|Default| +|---|---|---| +|`-c`, `--conf`|Configuration for the auto-recovery|| + + +### `bookie` +Starts up a BookKeeper server (aka bookie) + +Usage + +```bash +bookkeeper bookie options +``` + +Options + +|Flag|Description|Default| +|---|---|---| +|`-c`, `--conf`|Configuration for the auto-recovery|| +|-readOnly|Force start a read-only bookie server|false| +|-withAutoRecovery|Start auto-recovery service bookie server|false| + + +### `localbookie` +Runs a test ensemble of N bookies locally + +Usage + +```bash +bookkeeper localbookie N +``` + +### `upgrade` +Upgrade the bookie’s filesystem + +Usage + +```bash +bookkeeper upgrade options +``` + +Options + +|Flag|Description|Default| +|---|---|---| +|`-c`, `--conf`|Configuration for the auto-recovery|| +|`-u`, `--upgrade`|Upgrade the bookie’s directories|| + + +### `shell` +Run shell for admin commands. To see a full listing of those commands, run bookkeeper shell without an argument. + +Usage + +```bash +bookkeeper shell +``` + +Example + +```bash +bookkeeper shell bookiesanity +``` + diff --git a/site2/docs/reference-cli-broker-tool.md b/site2/docs/reference-cli-broker-tool.md new file mode 100644 index 0000000000000..56e2de9f5a288 --- /dev/null +++ b/site2/docs/reference-cli-broker-tool.md @@ -0,0 +1,36 @@ +--- +id: reference-cli-broker-tool +title: Pulsar CLI tools - broker-tool +sidebar_label: "cli-broker-tool" +--- + +## `broker-tool` + +The `broker- tool` is used for operations on a specific broker. + +Usage + +```bash +broker-tool command +``` + +Example + +Two ways to get more information about a command as below: + +```bash +broker-tool help command +broker-tool command --help +``` + +### `load-report` + +Collect the load report of a specific broker. +The command is run on a broker, and used for troubleshooting why broker can’t collect right load report. + +Options + +|Flag|Description|Default| +|---|---|---| +|`-i`, `--interval`| Interval to collect load report, in milliseconds || +|`-h`, `--help`| Display help information || diff --git a/site2/docs/reference-cli-pulsar-daemon.md b/site2/docs/reference-cli-pulsar-daemon.md new file mode 100644 index 0000000000000..b210f1793da3f --- /dev/null +++ b/site2/docs/reference-cli-pulsar-daemon.md @@ -0,0 +1,48 @@ +--- +id: reference-cli-pulsar-daemon +title: Pulsar CLI tools - pulsar-daemon +sidebar_label: "cli-pulsar-daemon" +--- + +## `pulsar-daemon` + +A wrapper around the pulsar tool that’s used to start and stop processes, such as ZooKeeper, bookies, and Pulsar brokers, in the background using nohup. + +pulsar-daemon has a similar interface to the pulsar command but adds start and stop commands for various services. For a listing of those services, run pulsar-daemon to see the help output or see the documentation for the pulsar command. + +Usage + +```bash +pulsar-daemon command +``` + +### `start` +Start a service in the background using nohup. + +Usage + +```bash +pulsar-daemon start service +``` + +### `stop` +Stop a service that’s already been started using start. + +Usage + +```bash +pulsar-daemon stop service options +``` + +Options + +|Flag|Description|Default| +|---|---|---| +|-force|Stop the service forcefully if not stopped by normal shutdown.|false| + +### `restart` +Restart a service that has already been started. + +```bash +pulsar-daemon restart service +``` \ No newline at end of file diff --git a/site2/docs/reference-cli-pulsar-shell.md b/site2/docs/reference-cli-pulsar-shell.md new file mode 100644 index 0000000000000..fc22af59034a2 --- /dev/null +++ b/site2/docs/reference-cli-pulsar-shell.md @@ -0,0 +1,155 @@ +--- +id: reference-cli-pulsar-shell +title: Pulsar CLI tools - pulsar-shell +sidebar_label: "cli-pulsar-shell" +--- + +## `pulsar-shell` + +[Pulsar shell](administration-pulsar-shell.md) tool. + +### Interactive mode + +Usage + +```bash +pulsar-shell +``` + +Options + +| Flag | Description | Default | +|--------------------|---------------------------------------------------------------------------|------------------| +| `-c`, `--config` | Client configuration file. It is used as a `default` config. | conf/client.conf | +| `--fail-on-error` | If true, the shell is interrupted when a command throws an exception. | false | +| `-h`, `--help` | Show this help. | | + + +### Non interactive mode + +Usage + +```bash +pulsar-shell -f [FILE] +pulsar-shell -e [COMMAND] +echo "[COMMAND]" | pulsar-shell - +``` + +Options + +| Flag | Description | Default | +|---------------------------|-----------------------------------------------------------------------------------------------------|-----------------| +| `-c`, `--config` | Client configuration file. It is used as a `default` config. | conf/client.conf | +| `--fail-on-error` | If true, the shell is interrupted when a command throws an exception. | false | +| `-np`, `--no-progress` | Display raw output of the commands without the fancy progress visualization. | false | +| `-f`, `--filename` | Input filename with a list of commands to be executed. Each command must be separated by a newline. | | +| `-e`, `--execute-command` | Execute this command and exit. | | +| `-` | Read commands from the standard input. | | +| `-h`, `--help` | Show this help. | | + + +Commands +* `admin` - See [Admin API](admin-api-overview.md) +* `client` - See [pulsar-client](#pulsar-client) +* `config` + + +### `config` + +Manage shell configurations. + +#### `use` + +Use a specific configuration for next commands. + +```bash +default(localhost)> config use mycluster +``` + +#### `create` + +Create a new configuration. + +```bash +default(localhost)> config create --file ./conf/client.conf mycluster +``` + +Options + +| Flag | Description | Default | +|----------|--------------------------|-----------------| +| `--file` | File path of the config. | | +| `--url` | URL of the config. | | +| `--value` | Inline value of the config. Base64-encoded value is supported with the prefix `base64:`. | | + +#### `update` + +Update an existing configuration. + +```bash +default(localhost)> config update --file ./conf/client.conf mycluster +``` + +Options + +| Flag | Description | Default | +|----------|--------------------------|-----------------| +| `--file` | File path of the config. | | +| `--url` | URL of the config. | | +| `--value` | Inline value of the config. Base64-encoded value is supported with the prefix `base64:`. | | + +#### `set-property` + +Set a value for a specified configuration property. + +```bash +default(localhost)> config set-property -p webServiceUrl -v http:// mycluster +``` + +Options + +| Flag | Description | Default | +|--------------------|-----------------------------|-----------------| +| `-p`, `--property` | Property name to update. | | +| `-v`, `--value` | New value for the property. | | + + +#### `get-property` + +Get the value for a specified configuration property. + +```bash +default(localhost)> config get-property -p webServiceUrl mycluster +``` + +Options + +| Flag | Description | Default | +|--------------------|-----------------------------|-----------------| +| `-p`, `--property` | Property name to update. | | + + +#### `view` + +View details of a config. + +```bash +default(localhost)> config view mycluster +``` + +#### `delete` + +Delete a config. You can't delete a config if it's currently used. + +```bash +default(localhost)> config delete mycluster +``` + + +#### `list` + +List all the configuration names. + +```bash +default(localhost)> config list +``` \ No newline at end of file diff --git a/site2/docs/reference-cli-tools.md b/site2/docs/reference-cli-tools.md index 8d172c8a3b467..cde841484a4f4 100644 --- a/site2/docs/reference-cli-tools.md +++ b/site2/docs/reference-cli-tools.md @@ -6,1066 +6,21 @@ sidebar_label: "Pulsar CLI tools" Pulsar offers several command-line tools that you can use for managing Pulsar installations, performance testing, using command-line producers and consumers, and more. -All Pulsar command-line tools can be run from the `bin` directory of your [installed Pulsar package](getting-started-standalone.md). The following tools are currently documented: +* [`pulsar-admin`](https://pulsar.apache.org/reference/#/latest/pulsar-admin/) +* [`pulsar`](https://pulsar.apache.org/reference/#/latest/pulsar/) +* [`pulsar-client`](https://pulsar.apache.org/reference/#/latest/pulsar-client/) +* [`pulsar-perf`](https://pulsar.apache.org/reference/#/latest/pulsar-perf/) +* [`pulsar-daemon`](reference-cli-pulsar-daemon.md) +* [`pulsar-shell`](reference-cli-pulsar-shell.md) +* [`bookkeeper`](reference-cli-bookkeeper.md) +* [`broker-tool`](reference-cli-broker-tool.md) -* [`pulsar`](#pulsar) -* [`pulsar-client`](#pulsar-client) -* [`pulsar-daemon`](#pulsar-daemon) -* [`pulsar-perf`](#pulsar-perf) -* [`bookkeeper`](#bookkeeper) -* [`broker-tool`](#broker-tool) -* [`pulsar-shell`](#pulsar-shell) +All Pulsar command-line tools can be run from the `bin` directory of your [installed Pulsar package](getting-started-standalone.md). -:::tip +You can get help for any CLI tool, command, or subcommand using the `--help` flag, or `-h` for short. Here's an example: - - This page only shows **some frequently used commands**. For the latest information about `pulsar`, `pulsar-client`, and `pulsar-perf`, including commands, flags, descriptions, and more information, see [Pulsar tools](/tools/). - - - You can get help for any CLI tool, command, or subcommand using the `--help` flag, or `-h` for short. Here's an example: - -::: - -> ```shell -> bin/pulsar broker --help -> ``` - - -## `pulsar` - -The pulsar tool is used to start Pulsar components, such as bookies and ZooKeeper, in the foreground. - -These processes can also be started in the background, using nohup, using the pulsar-daemon tool, which has the same command interface as pulsar. - -Usage: - -```bash -pulsar command -``` - -Commands: -* `bookie` -* `broker` -* `compact-topic` -* `configuration-store` -* `initialize-cluster-metadata` -* `proxy` -* `standalone` -* `websocket` -* `zookeeper` -* `zookeeper-shell` -* `autorecovery` - -Example: - -```bash -PULSAR_BROKER_CONF=/path/to/broker.conf pulsar broker +```shell +bin/pulsar broker --help ``` -Environment variables - -The table below lists the environment variables that you can use to configure the `pulsar` tool. - -|Variable|Description|Default| -|---|---|---| -|`PULSAR_LOG_CONF`|Log4j configuration file|`conf/log4j2.yaml`| -|`PULSAR_BROKER_CONF`|Configuration file for broker|`conf/broker.conf`| -|`PULSAR_BOOKKEEPER_CONF`|description: Configuration file for bookie|`conf/bookkeeper.conf`| -|`PULSAR_ZK_CONF`|Configuration file for zookeeper|`conf/zookeeper.conf`| -|`PULSAR_CONFIGURATION_STORE_CONF`|Configuration file for the configuration store|`conf/global_zookeeper.conf`| -|`PULSAR_WEBSOCKET_CONF`|Configuration file for websocket proxy|`conf/websocket.conf`| -|`PULSAR_STANDALONE_CONF`|Configuration file for standalone|`conf/standalone.conf`| -|`PULSAR_EXTRA_OPTS`|Extra options to be passed to the jvm|| -|`PULSAR_EXTRA_CLASSPATH`|Extra paths for Pulsar's classpath|| -|`PULSAR_PID_DIR`|Folder where the pulsar server PID file should be stored|| -|`PULSAR_STOP_TIMEOUT`|Wait time before forcefully killing the Bookie server instance if attempts to stop it are not successful|| -|`PULSAR_GC_LOG`|Gc options to be passed to the jvm|| - - -### `bookie` - -Starts up a bookie server - -Usage: - -```bash -pulsar bookie options -``` - -Options - -|Option|Description|Default| -|---|---|---| -|`-readOnly`|Force start a read-only bookie server|false| -|`-withAutoRecovery`|Start auto-recover service bookie server|false| - - -Example - -```bash -PULSAR_BOOKKEEPER_CONF=/path/to/bookkeeper.conf pulsar bookie \ --readOnly \ --withAutoRecovery -``` - -### `broker` - -Starts up a Pulsar broker - -Usage - -```bash -pulsar broker options -``` - -Options - -|Option|Description|Default| -|---|---|---| -|`-bc` , `--bookie-conf`|Configuration file for BookKeeper|| -|`-rb` , `--run-bookie`|Run a BookKeeper bookie on the same host as the Pulsar broker|false| -|`-ra` , `--run-bookie-autorecovery`|Run a BookKeeper autorecovery daemon on the same host as the Pulsar broker|false| - -Example - -```bash -PULSAR_BROKER_CONF=/path/to/broker.conf pulsar broker -``` - -### `compact-topic` - -Run compaction against a Pulsar topic (in a new process) - -Usage - -```bash -pulsar compact-topic options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-t` , `--topic`|The Pulsar topic that you would like to compact|| - -Example - -```bash -pulsar compact-topic --topic topic-to-compact -``` - -### `configuration-store` - -Starts up the Pulsar configuration store - -Usage - -```bash -pulsar configuration-store -``` - -Example - -```bash -PULSAR_CONFIGURATION_STORE_CONF=/path/to/configuration_store.conf pulsar configuration-store -``` - -### `initialize-cluster-metadata` - -One-time cluster metadata initialization - -Usage - -```bash -pulsar initialize-cluster-metadata options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-ub` , `--broker-service-url`|The broker service URL for the new cluster|| -|`-tb` , `--broker-service-url-tls`|The broker service URL for the new cluster with TLS encryption|| -|`-c` , `--cluster`|Cluster name|| -|`-cms` , `--configuration-metadata-store`|The configuration metadata store quorum connection string|| -|`--existing-bk-metadata-service-uri`|The metadata service URI of the existing BookKeeper cluster that you want to use|| -|`-h` , `--help`|Help message|false| -|`--initial-num-stream-storage-containers`|The number of storage containers of BookKeeper stream storage|16| -|`--initial-num-transaction-coordinators`|The number of transaction coordinators assigned in a cluster|16| -|`-uw` , `--web-service-url`|The web service URL for the new cluster|| -|`-tw` , `--web-service-url-tls`|The web service URL for the new cluster with TLS encryption|| -|`-md` , `--metadata-store`|The metadata store service url|| -|`--zookeeper-session-timeout-ms`|The local ZooKeeper session timeout. The time unit is in millisecond(ms)|30000| - - -### `proxy` - -Manages the Pulsar proxy - -Usage - -```bash -pulsar proxy options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-cms`, `--configuration-metadata-store`|Configuration metadata store connection string|| -|`-md` , `--metadata-store`|Metadata Store service url|| - -Example - -```bash -PULSAR_PROXY_CONF=/path/to/proxy.conf pulsar proxy \ ---metadata-store zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 \ ---configuration-metadata-store zk:my-zk-1:2181,my-zk-2:2181,my-zk-3:2181 -``` - -### `standalone` - -Run a broker service with local bookies and local ZooKeeper - -Usage - -```bash -pulsar standalone options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-a` , `--advertised-address`|The standalone broker advertised address|| -|`--bookkeeper-dir`|Local bookies’ base data directory|data/standalone/bookkeeper| -|`--bookkeeper-port`|Local bookies’ base port|3181| -|`--no-broker`|Only start ZooKeeper and BookKeeper services, not the broker|false| -|`--num-bookies`|The number of local bookies|1| -|`--only-broker`|Only start the Pulsar broker service (not ZooKeeper or BookKeeper)|| -|`--wipe-data`|Clean up previous ZooKeeper/BookKeeper data|| -|`--zookeeper-dir`|Local ZooKeeper’s data directory|data/standalone/zookeeper| -|`--zookeeper-port` |Local ZooKeeper’s port|2181| - -Example - -```bash -PULSAR_STANDALONE_CONF=/path/to/standalone.conf pulsar standalone -``` - -### `websocket` - -Usage - -```bash -pulsar websocket -``` - -Example - -```bash -PULSAR_WEBSOCKET_CONF=/path/to/websocket.conf pulsar websocket -``` - -### `zookeeper` - -Starts up a ZooKeeper cluster - -Usage - -```bash -pulsar zookeeper -``` - -Example - -```bash -PULSAR_ZK_CONF=/path/to/zookeeper.conf pulsar zookeeper -``` - -### `zookeeper-shell` - -Connects to a running ZooKeeper cluster using the ZooKeeper shell - -Usage - -```bash -pulsar zookeeper-shell options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-c`, `--conf`|Configuration file for ZooKeeper|| -|`-server`|Configuration zk address, eg: `127.0.0.1:2181`|| - -### `autorecovery` - -Runs an auto-recovery service. - -Usage - -```bash -pulsar autorecovery options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-c`, `--conf`|Configuration for the autorecovery|N/A| - - -## `pulsar-client` - -The pulsar-client tool - -Usage - -```bash -pulsar-client command -``` - -Commands -* `produce` -* `consume` - - -Options - -|Flag|Description|Default| -|---|---|---| -|`--auth-params`|Authentication parameters, whose format is determined by the implementation of method `configure` in authentication plugin class, for example "key1:val1,key2:val2" or "{\"key1\":\"val1\",\"key2\":\"val2\"}"|{"saslJaasClientSectionName":"PulsarClient", "serverType":"broker"}| -|`--auth-plugin`|Authentication plugin class name|org.apache.pulsar.client.impl.auth.AuthenticationSasl| -|`--listener-name`|Listener name for the broker|| -|`--proxy-protocol`|Proxy protocol to select type of routing at proxy|| -|`--proxy-url`|Proxy-server URL to which to connect|| -|`--url`|Broker URL to which to connect|pulsar://localhost:6650/
ws://localhost:8080 | -| `-v`, `--version` | Get the version of the Pulsar client -|`-h`, `--help`|Show this help - - -### `produce` -Send a message or messages to a specific broker and topic - -Usage - -```bash -pulsar-client produce topic options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-f`, `--files`|Comma-separated file paths to send; either -m or -f must be specified|[]| -|`-m`, `--messages`|Comma-separated string of messages to send; either -m or -f must be specified|[]| -|`-n`, `--num-produce`|The number of times to send the message(s); the count of messages/files * num-produce should be below 1000|1| -|`-r`, `--rate`|Rate (in messages per second) at which to produce; a value 0 means to produce messages as fast as possible|0.0| -|`-db`, `--disable-batching`|Disable batch sending of messages|false| -|`-c`, `--chunking`|Split the message and publish in chunks if the message size is larger than the allowed max size|false| -|`-s`, `--separator`|Character to split messages string with.|","| -|`-k`, `--key`|Message key to add|key=value string, like k1=v1,k2=v2.| -|`-p`, `--properties`|Properties to add. If you want to add multiple properties, use the comma as the separator, e.g. `k1=v1,k2=v2`.| | -|`-ekn`, `--encryption-key-name`|The public key name to encrypt payload.| | -|`-ekv`, `--encryption-key-value`|The URI of public key to encrypt payload. For example, `file:///path/to/public.key` or `data:application/x-pem-file;base64,*****`.| | - - -### `consume` -Consume messages from a specific broker and topic - -Usage - -```bash -pulsar-client consume topic options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`--hex`|Display binary messages in hexadecimal format.|false| -|`-n`, `--num-messages`|Number of messages to consume, 0 means to consume forever.|1| -|`-r`, `--rate`|Rate (in messages per second) at which to consume; a value 0 means to consume messages as fast as possible|0.0| -|`--regex`|Indicate the topic name is a regex pattern|false| -|`-s`, `--subscription-name`|Subscription name|| -|`-t`, `--subscription-type`|The type of the subscription. Possible values: Exclusive, Shared, Failover, Key_Shared.|Exclusive| -|`-p`, `--subscription-position`|The position of the subscription. Possible values: Latest, Earliest.|Latest| -|`-m`, `--subscription-mode`|Subscription mode. Possible values: Durable, NonDurable.|Durable| -|`-q`, `--queue-size`|The size of consumer's receiver queue.|0| -|`-mc`, `--max_chunked_msg`|Max pending chunk messages.|0| -|`-ac`, `--auto_ack_chunk_q_full`|Auto ack for the oldest message in consumer's receiver queue if the queue full.|false| -|`--hide-content`|Do not print the message to the console.|false| -|`-st`, `--schema-type`|Set the schema type. Use `auto_consume` to dump AVRO and other structured data types. Possible values: bytes, auto_consume.|bytes| -|`-ekv`, `--encryption-key-value`|The URI of public key to encrypt payload. For example, `file:///path/to/public.key` or `data:application/x-pem-file;base64,*****`.| | -|`-pm`, `--pool-messages`|Use the pooled message.|true| - -## `pulsar-daemon` -A wrapper around the pulsar tool that’s used to start and stop processes, such as ZooKeeper, bookies, and Pulsar brokers, in the background using nohup. - -pulsar-daemon has a similar interface to the pulsar command but adds start and stop commands for various services. For a listing of those services, run pulsar-daemon to see the help output or see the documentation for the pulsar command. - -Usage - -```bash -pulsar-daemon command -``` - -Commands -* `start` -* `stop` -* `restart` - - -### `start` -Start a service in the background using nohup. - -Usage - -```bash -pulsar-daemon start service -``` - -### `stop` -Stop a service that’s already been started using start. - -Usage - -```bash -pulsar-daemon stop service options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|-force|Stop the service forcefully if not stopped by normal shutdown.|false| - -### `restart` -Restart a service that has already been started. - -```bash -pulsar-daemon restart service -``` - -## `pulsar-perf` -A tool for performance testing a Pulsar broker. - -Usage - -```bash -pulsar-perf command -``` - -Commands -* `consume` -* `produce` -* `read` -* `websocket-producer` -* `managed-ledger` -* `monitor-brokers` -* `simulation-client` -* `simulation-controller` -* `transaction` -* `help` - -Environment variables - -The table below lists the environment variables that you can use to configure the pulsar-perf tool. - -|Variable|Description|Default| -|---|---|---| -|`PULSAR_LOG_CONF`|Log4j configuration file|conf/log4j2.yaml| -|`PULSAR_CLIENT_CONF`|Configuration file for the client|conf/client.conf| -|`PULSAR_EXTRA_OPTS`|Extra options to be passed to the JVM|| -|`PULSAR_EXTRA_CLASSPATH`|Extra paths for Pulsar's classpath|| -|`PULSAR_GC_LOG`|Gc options to be passed to the jvm|| - -Commands `consume`, `produce`, `read` and `transaction` share the following client options: - -|Flag|Description|Default| -|---|---|---| -|`--auth-params`|Authentication parameters, whose format is determined by the implementation of method `configure` in authentication plugin class. For example, `key1:val1,key2:val2` or `{"key1":"val1","key2":"val2"}`.|| -|`--auth-plugin`|Authentication plugin class name|| -|`-bw`, `--busy-wait`|Enable or disable Busy-Wait on the Pulsar client|false| -|`-c`, `--max-connections`|Max number of TCP connections to a single broker|100| -|`-cf`, `--conf-file`|Configuration file|| -|`-i`, `--stats-interval-seconds`|Statistics interval seconds. If 0, statistics will be disabled|0| -|`-ioThreads`, `--num-io-threads`|Set the number of threads to be used for handling connections to brokers|1| -|`--listener-name`|Listener name for the broker|| -|`-lt`, `--num-listener-threads`|Set the number of threads to be used for message listeners|1| -|`--tls-allow-insecure`|Allow insecure TLS connection|| -|`--tls-enable-hostname-verification`|Enable TLS hostname verification|| -|`--trust-cert-file`|Path for the trusted TLS certificate file|| -|`-u`, `--service-url`|Pulsar service URL|| - - -### `consume` -Run a consumer - -Usage - -``` -pulsar-perf consume options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-ac`, `--auto_ack_chunk_q_full`|Auto ack for the oldest message in consumer's receiver queue if the queue full|false| -|`--acks-delay-millis`|Acknowledgments grouping delay in millis|100| -|`--batch-index-ack`|Enable or disable the batch index acknowledgment|false| -|`-v`, `--encryption-key-value-file`|The file which contains the private key to decrypt payload|| -|`-m`, `--num-messages`|Number of messages to consume in total. If the value is equal to or smaller than 0, it keeps consuming messages.|0| -|`-e`, `--expire_time_incomplete_chunked_messages`|The expiration time for incomplete chunk messages (in milliseconds)|0| -|`-mc`, `--max_chunked_msg`|Max pending chunk messages|0| -|`-n`, `--num-consumers`|Number of consumers (per topic)|1| -|`-ns`, `--num-subscriptions`|Number of subscriptions (per topic)|1| -|`-t`, `--num-topics`|The number of topics|1| -|`-pm`, `--pool-messages`|Use the pooled message|true| -|`-r`, `--rate`|Simulate a slow message consumer (rate in msg/s)|0| -|`-q`, `--receiver-queue-size`|Size of the receiver queue|1000| -|`-p`, `--receiver-queue-size-across-partitions`|Max total size of the receiver queue across partitions|50000| -|`--replicated`|Whether the subscription status should be replicated|false| -|`-s`, `--subscriber-name`|Subscriber name prefix|| -|`-ss`, `--subscriptions`|A list of subscriptions to consume on (e.g. sub1,sub2)|sub| -|`-st`, `--subscription-type`|Subscriber type. Possible values are Exclusive, Shared, Failover, Key_Shared.|Exclusive| -|`-sp`, `--subscription-position`|Subscriber position. Possible values are Latest, Earliest.|Latest| -|`-time`, `--test-duration`|Test duration (in seconds). If this value is less than or equal to 0, it keeps consuming messages.|0| - - -Below are **transaction** related options. - -If you want `--txn-timeout`, `--numMessage-perTransaction`, `-nmt`, `-ntxn`, or `-abort` take effect, set `--txn-enable` to true. - -|Flag|Description|Default| -|---|---|---| -`-tto`, `--txn-timeout`|Set the time of transaction timeout (in second). |10 -`-nmt`, `--numMessage-perTransaction`|The number of messages acknowledged by a transaction. |50 -`-txn`, `--txn-enable`|Enable or disable a transaction.|false -`-ntxn`|The number of opened transactions. 0 means the number of transactions is unlimited. |0 -`-abort`|Abort a transaction. |true - -### `produce` -Run a producer - -Usage - -```bash -pulsar-perf produce options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-am`, `--access-mode`|Producer access mode. Valid values are `Shared`, `Exclusive` and `WaitForExclusive`|Shared| -|`-au`, `--admin-url`|Pulsar admin URL|| -|`-b`, `--batch-time-window`|Batch messages in a window of the specified number of milliseconds|1| -|`-bb`, `--batch-max-bytes`|Maximum number of bytes per batch|4194304| -|`-bm`, `--batch-max-messages`|Maximum number of messages per batch|1000| -|`-ch`, `--chunking`|Split the message and publish in chunks if the message size is larger than allowed max size|false| -|`-d`, `--delay`|Mark messages with a given delay in seconds|0s| -|`-z`, `--compression`|Compress messages’ payload. Possible values are NONE, LZ4, ZLIB, ZSTD or SNAPPY.|| -|`-k`, `--encryption-key-name`|The public key name to encrypt payload|| -|`-v`, `--encryption-key-value-file`|The file which contains the public key to encrypt payload|| -|`-ef`, `--exit-on-failure`|Exit from the process on publish failure|false| -|`-fc`, `--format-class`|Custom Formatter class name|org.apache.pulsar.testclient.DefaultMessageFormatter| -|`-fp`, `--format-payload`|Format %i as a message index in the stream from producer and/or %t as the timestamp nanoseconds|false| -|`-h`, `--help`|Help message|false| -|`-o`, `--max-outstanding`|Max number of outstanding messages|1000| -|`-p`, `--max-outstanding-across-partitions`|Max number of outstanding messages across partitions|50000| -|`-m`, `--num-messages`|Number of messages to publish in total. If this value is less than or equal to 0, it keeps publishing messages.|0| -|`-mk`, `--message-key-generation-mode`|The generation mode of message key. Valid options are `autoIncrement`, `random`|| -|`-n`, `--num-producers`|The number of producers (per topic)|1| -|`-threads`, `--num-test-threads`|Number of test threads|1| -|`-t`, `--num-topic`|The number of topics|1| -|`-np`, `--partitions`|Create partitioned topics with the given number of partitions. Setting this value to 0 means not trying to create a topic|| -|`-f`, `--payload-file`|Use payload from an UTF-8 encoded text file and a payload will be randomly selected when publishing messages|| -|`-e`, `--payload-delimiter`|The delimiter used to split lines when using payload from a file|\n| -|`-pn`, `--producer-name`|Producer Name|| -|`-r`, `--rate`|Publish rate msg/s across topics|100| -|`--send-timeout`|Set the sendTimeout|0| -|`--separator`|Separator between the topic and topic number|-| -|`-s`, `--size`|Message size (in bytes)|1024| -|`-time`, `--test-duration`|Test duration (in seconds). If this value is less than or equal to 0, it keeps publishing messages.|0| -|`--warmup-time`|Warm-up time in seconds|1| - -Below are **transaction** related options. - -If you want `--txn-timeout`, `--numMessage-perTransaction`, or `-abort` take effect, set `--txn-enable` to true. - -|Flag|Description|Default| -|---|---|---| -`-tto`, `--txn-timeout`|Set the time of transaction timeout (in second). |5 -`-nmt`, `--numMessage-perTransaction`|The number of messages acknowledged by a transaction. |50 -`-txn`, `--txn-enable`|Enable or disable a transaction.|true -`-abort`|Abort a transaction. |true - -### `read` -Run a topic reader - -Usage - -```bash -pulsar-perf read options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-h`, `--help`|Help message|false| -|`-n`, `--num-messages`|Number of messages to consume in total. If the value is equal to or smaller than 0, it keeps consuming messages.|0| -|`-t`, `--num-topics`|The number of topics|1| -|`-r`, `--rate`|Simulate a slow message reader (rate in msg/s)|0| -|`-q`, `--receiver-queue-size`|Size of the receiver queue|1000| -|`-m`, `--start-message-id`|Start message id. This can be either 'earliest', 'latest' or a specific message id by using 'lid:eid'|earliest| -|`-time`, `--test-duration`|Test duration (in seconds). If this value is less than or equal to 0, it keeps consuming messages.|0| -|`--use-tls`|Use TLS encryption on the connection|false| - -### `websocket-producer` -Run a websocket producer - -Usage - -```bash -pulsar-perf websocket-producer options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`--auth-params`|Authentication parameters, whose format is determined by the implementation of method `configure` in authentication plugin class. For example, `key1:val1,key2:val2` or `{"key1":"val1","key2":"val2"}`.|| -|`--auth-plugin`|Authentication plugin class name|| -|`-cf`, `--conf-file`|Configuration file|| -|`-h`, `--help`|Help message|false| -|`-m`, `--num-messages`|Number of messages to publish in total. If this value is less than or equal to 0, it keeps publishing messages.|0| -|`-t`, `--num-topic`|The number of topics|1| -|`-f`, `--payload-file`|Use payload from a file instead of empty buffer|| -|`-e`, `--payload-delimiter`|The delimiter used to split lines when using payload from a file|\n| -|`-fp`, `--format-payload`|Format %i as a message index in the stream from producer and/or %t as the timestamp nanoseconds|false| -|`-fc`, `--format-class`|Custom formatter class name|`org.apache.pulsar.testclient.DefaultMessageFormatter`| -|`-u`, `--proxy-url`|Pulsar Proxy URL, e.g., "ws://localhost:8080/"|| -|`-r`, `--rate`|Publish rate msg/s across topics|100| -|`-s`, `--size`|Message size in byte|1024| -|`-time`, `--test-duration`|Test duration (in seconds). If this value is less than or equal to 0, it keeps publishing messages.|0| - - -### `managed-ledger` -Write directly on managed-ledgers - -Usage - -```bash -pulsar-perf managed-ledger options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-a`, `--ack-quorum`|Ledger ack quorum|1| -|`-dt`, `--digest-type`|BookKeeper digest type. Possible Values: [CRC32, MAC, CRC32C, DUMMY]|CRC32C| -|`-e`, `--ensemble-size`|Ledger ensemble size|1| -|`-h`, `--help`|Help message|false| -|`-c`, `--max-connections`|Max number of TCP connections to a single bookie|1| -|`-o`, `--max-outstanding`|Max number of outstanding requests|1000| -|`-m`, `--num-messages`|Number of messages to publish in total. If this value is less than or equal to 0, it keeps publishing messages.|0| -|`-t`, `--num-topic`|Number of managed ledgers|1| -|`-r`, `--rate`|Write rate msg/s across managed ledgers|100| -|`-s`, `--size`|Message size in byte|1024| -|`-time`, `--test-duration`|Test duration (in seconds). If this value is less than or equal to 0, it keeps publishing messages.|0| -|`--threads`|Number of threads writing|1| -|`-w`, `--write-quorum`|Ledger write quorum|1| -|`-md`, `--metadata-store`|Metadata store service URL. For example: zk:my-zk:2181|| - - -### `monitor-brokers` -Continuously receive broker data and/or load reports - -Usage - -```bash -pulsar-perf monitor-brokers options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`--connect-string`|A connection string for one or more ZooKeeper servers|| -|`-h`, `--help`|Help message|false| - - -### `simulation-client` -Run a simulation server acting as a Pulsar client. Uses the client configuration specified in `conf/client.conf`. - -Usage - -```bash -pulsar-perf simulation-client options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`--port`|Port to listen on for controller|0| -|`--service-url`|Pulsar Service URL|| -|`-h`, `--help`|Help message|false| - -### `simulation-controller` -Run a simulation controller to give commands to servers - -Usage - -```bash -pulsar-perf simulation-controller options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`--client-port`|The port that the clients are listening on|0| -|`--clients`|Comma-separated list of client hostnames|| -|`--cluster`|The cluster to test on|| -|`-h`, `--help`|Help message|false| - -### `transaction` - -Run a transaction. For more information, see [Pulsar transactions](txn-why.md). - -**Usage** - -```bash -pulsar-perf transaction options -``` - -**Options** - -|Flag|Description|Default| -|---|---|---| -`-au`, `--admin-url`|Pulsar admin URL.|N/A -`-h`, `--help`|Help messages.|N/A -`-ns`, `--num-subscriptions`|Number of subscriptions per topic.|1 -`-threads`, `--num-test-threads`|Number of test threads.

This thread is for a new transaction to ack messages from consumer topics, produce messages to producer topics, and commit or abort this transaction.

Increasing the number of threads increases the parallelism of the performance test, consequently, it increases the intensity of the stress test.|1 -`-nmc`, `--numMessage-perTransaction-consume`|Set the number of messages consumed in a transaction.

If transaction is disabled, it means the number of messages consumed in a task instead of in a transaction.|1 -`-nmp`, `--numMessage-perTransaction-produce`|Set the number of messages produced in a transaction.

If transaction is disabled, it means the number of messages produced in a task instead of in a transaction.|1 -`-ntxn`, `--number-txn`|Set the number of transactions.

0 means the number of transactions is unlimited.

If transaction is disabled, it means the number of tasks instead of transactions. |0 -`-np`, `--partitions`|Create partitioned topics with a given number of partitions.

0 means not trying to create a topic. -`-q`, `--receiver-queue-size`|Size of the receiver queue.|1000 -`-sp`, `--subscription-position`|Subscription position.|Earliest -`-st`, `--subscription-type`|Subscription type.|Shared -`-ss`, `--subscriptions`|A list of subscriptions to consume.

For example, sub1,sub2.|[sub] -`-time`, `--test-duration`|Test duration (in second).

0 means keeping publishing messages.|0 -`--topics-c`|All topics assigned to consumers.|[test-consume] -`--topics-p`|All topics assigned to producers . |[test-produce] -`--txn-disEnable`|Disable transaction.|true -`-tto`, `--txn-timeout`|Set the time of transaction timeout (in second).

If you want `--txn-timeout` takes effect, set `--txn-enable` to true.|5 -`-abort`|Abort the transaction.

If you want `-abort` takes effect, set `--txn-disEnable` to false.|true -`-txnRate`|Set the rate of opened transactions or tasks.

0 means no limit.|0 - -### `help` -This help message - -Usage - -```bash -pulsar-perf help -``` - -## `bookkeeper` -A tool for managing BookKeeper. - -Usage - -```bash -bookkeeper command -``` - -Commands -* `autorecovery` -* `bookie` -* `localbookie` -* `upgrade` -* `shell` - - -Environment variables - -The table below lists the environment variables that you can use to configure the bookkeeper tool. - -|Variable|Description|Default| -|---|---|---| -|BOOKIE_LOG_CONF|Log4j configuration file|conf/log4j2.yaml| -|BOOKIE_CONF|BookKeeper configuration file|conf/bk_server.conf| -|BOOKIE_EXTRA_OPTS|Extra options to be passed to the JVM|| -|BOOKIE_EXTRA_CLASSPATH|Extra paths for BookKeeper's classpath|| -|ENTRY_FORMATTER_CLASS|The Java class used to format entries|| -|BOOKIE_PID_DIR|Folder where the BookKeeper server PID file should be stored|| -|BOOKIE_STOP_TIMEOUT|Wait time before forcefully killing the Bookie server instance if attempts to stop it are not successful|| -|BOOKIE_GC_LOG|Gc options to be passed to the jvm|| - - -### `autorecovery` -Runs an auto-recovery service - -Usage - -```bash -bookkeeper autorecovery options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-c`, `--conf`|Configuration for the auto-recovery|| - - -### `bookie` -Starts up a BookKeeper server (aka bookie) - -Usage - -```bash -bookkeeper bookie options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-c`, `--conf`|Configuration for the auto-recovery|| -|-readOnly|Force start a read-only bookie server|false| -|-withAutoRecovery|Start auto-recovery service bookie server|false| - - -### `localbookie` -Runs a test ensemble of N bookies locally - -Usage - -```bash -bookkeeper localbookie N -``` - -### `upgrade` -Upgrade the bookie’s filesystem - -Usage - -```bash -bookkeeper upgrade options -``` - -Options - -|Flag|Description|Default| -|---|---|---| -|`-c`, `--conf`|Configuration for the auto-recovery|| -|`-u`, `--upgrade`|Upgrade the bookie’s directories|| - - -### `shell` -Run shell for admin commands. To see a full listing of those commands, run bookkeeper shell without an argument. - -Usage - -```bash -bookkeeper shell -``` - -Example - -```bash -bookkeeper shell bookiesanity -``` - -## `broker-tool` - -The `broker- tool` is used for operations on a specific broker. - -Usage - -```bash -broker-tool command -``` - -Commands -* `load-report` -* `help` - -Example -Two ways to get more information about a command as below: - -```bash -broker-tool help command -broker-tool command --help -``` - -### `load-report` - -Collect the load report of a specific broker. -The command is run on a broker, and used for troubleshooting why broker can’t collect right load report. - -Options - -|Flag|Description|Default| -|---|---|---| -|`-i`, `--interval`| Interval to collect load report, in milliseconds || -|`-h`, `--help`| Display help information || - - -## `pulsar-shell` - -[Pulsar shell](administration-pulsar-shell.md) tool. - -### Interactive mode - -Usage - -```bash -pulsar-shell -``` - -Options - -| Flag | Description | Default | -|--------------------|---------------------------------------------------------------------------|------------------| -| `-c`, `--config` | Client configuration file. It is used as a `default` config. | conf/client.conf | -| `--fail-on-error` | If true, the shell is interrupted when a command throws an exception. | false | -| `-h`, `--help` | Show this help. | | - - -### Non interactive mode - -Usage - -```bash -pulsar-shell -f [FILE] -pulsar-shell -e [COMMAND] -echo "[COMMAND]" | pulsar-shell - -``` - -Options - -| Flag | Description | Default | -|---------------------------|-----------------------------------------------------------------------------------------------------|-----------------| -| `-c`, `--config` | Client configuration file. It is used as a `default` config. | conf/client.conf | -| `--fail-on-error` | If true, the shell is interrupted when a command throws an exception. | false | -| `-np`, `--no-progress` | Display raw output of the commands without the fancy progress visualization. | false | -| `-f`, `--filename` | Input filename with a list of commands to be executed. Each command must be separated by a newline. | | -| `-e`, `--execute-command` | Execute this command and exit. | | -| `-` | Read commands from the standard input. | | -| `-h`, `--help` | Show this help. | | - - -Commands -* `admin` - See [Admin API](admin-api-overview.md) -* `client` - See [pulsar-client](#pulsar-client) -* `config` - - -### `config` - -Manage shell configurations. - -#### `use` - -Use a specific configuration for next commands. - -```bash -default(localhost)> config use mycluster -``` - -#### `create` - -Create a new configuration. - -```bash -default(localhost)> config create --file ./conf/client.conf mycluster -``` - -Options - -| Flag | Description | Default | -|----------|--------------------------|-----------------| -| `--file` | File path of the config. | | -| `--url` | URL of the config. | | -| `--value` | Inline value of the config. Base64-encoded value is supported with the prefix `base64:`. | | - -#### `update` - -Update an existing configuration. - -```bash -default(localhost)> config update --file ./conf/client.conf mycluster -``` - -Options - -| Flag | Description | Default | -|----------|--------------------------|-----------------| -| `--file` | File path of the config. | | -| `--url` | URL of the config. | | -| `--value` | Inline value of the config. Base64-encoded value is supported with the prefix `base64:`. | | - -#### `set-property` - -Set a value for a specified configuration property. - -```bash -default(localhost)> config set-property -p webServiceUrl -v http:// mycluster -``` - -Options - -| Flag | Description | Default | -|--------------------|-----------------------------|-----------------| -| `-p`, `--property` | Property name to update. | | -| `-v`, `--value` | New value for the property. | | - - -#### `get-property` - -Get the value for a specified configuration property. - -```bash -default(localhost)> config get-property -p webServiceUrl mycluster -``` - -Options - -| Flag | Description | Default | -|--------------------|-----------------------------|-----------------| -| `-p`, `--property` | Property name to update. | | - - -#### `view` - -View details of a config. - -```bash -default(localhost)> config view mycluster -``` - -#### `delete` - -Delete a config. You can't delete a config if it's currently used. - -```bash -default(localhost)> config delete mycluster -``` - - -#### `list` - -List all the configuration names. -```bash -default(localhost)> config list -``` \ No newline at end of file From 9acafc9b9084cbb127ce5669236bdd5dd8e85a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Boschi?= Date: Mon, 17 Oct 2022 23:09:50 +0200 Subject: [PATCH 30/32] [fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 (#18022) * [fix][sec] File tiered storage: upgrade jettison to get rid of CVE-2022-40149 * fix --- pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pom.xml b/pom.xml index d0c9cbf7b0add..e663de7794274 100644 --- a/pom.xml +++ b/pom.xml @@ -247,6 +247,7 @@ flexible messaging model and an intuitive client API. 3.1 4.2.0 1.2.22 + 1.5.1 0.6.1 @@ -798,6 +799,13 @@ flexible messaging model and an intuitive client API. import + + org.codehaus.jettison + jettison + ${jettison.version} + + + org.hdrhistogram HdrHistogram From 83e830e6deb9b5c7c3ef01b9120e1ba0858db688 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 18 Oct 2022 09:18:42 +0800 Subject: [PATCH 31/32] [improve][doc] Redirect dangling standlaone to getting-started-standalone (#18063) --- site2/docs/standalone.md | 240 +--------------- .../version-2.10.x/standalone.md | 264 +----------------- 2 files changed, 8 insertions(+), 496 deletions(-) diff --git a/site2/docs/standalone.md b/site2/docs/standalone.md index 70122be8a461c..d1658b84cc27e 100644 --- a/site2/docs/standalone.md +++ b/site2/docs/standalone.md @@ -4,240 +4,8 @@ title: Set up a standalone Pulsar locally sidebar_label: "Run Pulsar locally" --- -For local development and testing, you can run Pulsar in standalone mode on your machine. The standalone mode includes a Pulsar broker, the necessary [RocksDB](http://rocksdb.org/) and BookKeeper components running inside of a single Java Virtual Machine (JVM) process. - -> **Pulsar in production?** -> If you're looking to run a full production Pulsar installation, see the [Deploying a Pulsar instance](deploy-bare-metal.md) guide. - -## Install Pulsar standalone - -This tutorial guides you through every step of installing Pulsar locally. - -### System requirements - -Currently, Pulsar is available for 64-bit **macOS**, **Linux**, and **Windows**. To use Pulsar, you need to install 64-bit JRE/JDK 8 or later versions - -:::tip - -By default, Pulsar allocates 2G JVM heap memory to start. It can be changed in `conf/pulsar_env.sh` file under `PULSAR_MEM`. This is an extra option passed into JVM. - -::: - -:::note - -Broker is only supported on 64-bit JVM. - -::: - -### Install Pulsar using binary release - -To get started with Pulsar, download a binary tarball release in one of the following ways: - -* download from the Apache mirror (Pulsar @pulsar:version@ binary release) - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - wget pulsar:binary_release_url - ``` - -After you download the tarball, untar it and use the `cd` command to navigate to the resulting directory: - -```bash -tar xvfz apache-pulsar-@pulsar:version@-bin.tar.gz -cd apache-pulsar-@pulsar:version@ -``` - -#### What your package contains - -The Pulsar binary package initially contains the following directories: - -Directory | Contains -:---------|:-------- -`bin` | Pulsar's command-line tools, such as [`pulsar`](reference-cli-tools.md#pulsar) and [`pulsar-admin`](/tools/pulsar-admin/). -`conf` | Configuration files for Pulsar, including [broker configuration](reference-configuration.md#broker) and more.
**Note:** Pulsar standalone uses RocksDB as the local metadata store and its configuration file path [`metadataStoreConfigPath`](reference-configuration.md) is configurable in the `standalone.conf` file. For more information about the configurations of RocksDB, see [here](https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini) and related [documentation](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide). -`examples` | A Java JAR file containing [Pulsar Functions](functions-overview.md) example. -`instances` | Artifacts created for [Pulsar Functions](functions-overview.md). -`lib` | The [JAR](https://en.wikipedia.org/wiki/JAR_(file_format)) files used by Pulsar. -`licenses` | License files, in the`.txt` form, for various components of the Pulsar [codebase](https://github.com/apache/pulsar). - -These directories are created once you begin running Pulsar. - -Directory | Contains -:---------|:-------- -`data` | The data storage directory used by RocksDB and BookKeeper. -`logs` | Logs created by the installation. - -:::tip - -If you want to use built-in connectors and tiered storage offloaders, you can install them according to the following instructions： -* [Install built-in connectors (optional)](#install-built-in-connectors-optional) -* [Install tiered storage offloaders (optional)](#install-tiered-storage-offloaders-optional) -Otherwise, skip this step and perform the next step [Start Pulsar standalone](#start-pulsar-standalone). Pulsar can be successfully installed without installing built-in connectors and tiered storage offloaders. - -::: - -### Install built-in connectors (optional) - -Since `2.1.0-incubating` release, Pulsar releases a separate binary distribution, containing all the built-in connectors. -To enable those built-in connectors, you can download the connectors tarball release in one of the following ways: - -* download from the Apache mirror Pulsar IO Connectors @pulsar:version@ release - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - wget pulsar:connector_release_url/{connector}-@pulsar:version@.nar - ``` - -After you download the NAR file, copy the file to the `connectors` directory in the pulsar directory. -For example, if you download the `pulsar-io-aerospike-@pulsar:version@.nar` connector file, enter the following commands: - -```bash -mkdir connectors -mv pulsar-io-aerospike-@pulsar:version@.nar connectors - -ls connectors -pulsar-io-aerospike-@pulsar:version@.nar -... -``` - -:::note - -* If you are running Pulsar in a bare metal cluster, make sure `connectors` tarball is unzipped in every pulsar directory of the broker (or in every pulsar directory of function-worker if you are running a separate worker cluster for Pulsar Functions). -* If you are [running Pulsar in Docker](getting-started-docker.md) or deploying Pulsar using a docker image (e.g. [K8S](deploy-kubernetes.md) or [DC/OS](https://dcos.io/), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled [all built-in connectors](io-overview.md#working-with-connectors). - -::: - -### Install tiered storage offloaders (optional) - -:::tip - -- Since `2.2.0` release, Pulsar releases a separate binary distribution, containing the tiered storage offloaders. -To enable the tiered storage feature, follow the instructions below; otherwise skip this section. - -::: - -To get started with [tiered storage offloaders](concepts-tiered-storage.md), you need to download the offloaders tarball release on every broker node in one of the following ways: - -* download from the Apache mirror Pulsar Tiered Storage Offloaders @pulsar:version@ release - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - wget pulsar:offloader_release_url - ``` - -After you download the tarball, untar the offloaders package and copy the offloaders as `offloaders` -in the pulsar directory: - -```bash -tar xvfz apache-pulsar-offloaders-@pulsar:version@-bin.tar.gz - -// you will find a directory named `apache-pulsar-offloaders-@pulsar:version@` in the pulsar directory -// then copy the offloaders - -mv apache-pulsar-offloaders-@pulsar:version@/offloaders offloaders - -ls offloaders -tiered-storage-jcloud-@pulsar:version@.nar -``` - -For more information on how to configure tiered storage, see [Tiered storage cookbook](cookbooks-tiered-storage.md). - -:::note - -* If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's pulsar directory. -* If you are [running Pulsar in Docker](getting-started-docker.md) or deploying Pulsar using a docker image (e.g. [K8S](deploy-kubernetes.md) or DC/OS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. - -::: - -## Start Pulsar standalone - -Once you have an up-to-date local copy of the release, you can start a local cluster using the [`pulsar`](reference-cli-tools.md#pulsar) command, which is stored in the `bin` directory, and specifying that you want to start Pulsar in standalone mode. - -```bash -bin/pulsar standalone -``` - -If you have started Pulsar successfully, you will see `INFO`-level log messages like this: - -```bash -21:59:29.327 [DLM-/stream/storage-OrderedScheduler-3-0] INFO org.apache.bookkeeper.stream.storage.impl.sc.StorageContainerImpl - Successfully started storage container (0). -21:59:34.576 [main] INFO org.apache.pulsar.broker.authentication.AuthenticationService - Authentication is disabled -21:59:34.576 [main] INFO org.apache.pulsar.websocket.WebSocketService - Pulsar WebSocket Service started -``` - -:::tip - -* The service is running on your terminal, which is under your direct control. If you need to run other commands, open a new terminal window. - -::: - -You can also run the service as a background process using the `bin/pulsar-daemon start standalone` command. For more information, see [pulsar-daemon](reference-cli-tools.md#pulsar-daemon). - -> * By default, there is no encryption, authentication, or authorization configured. Apache Pulsar can be accessed from remote server without any authorization. Please do check [Security Overview](security-overview.md) document to secure your deployment. -> -> * When you start a local standalone cluster, a `public/default` [namespace](concepts-messaging.md#namespaces) is created automatically. The namespace is used for development purposes. All Pulsar topics are managed within namespaces. For more information, see [Topics](concepts-messaging.md#topics). - -## Use Pulsar standalone - -Pulsar provides a CLI tool called [`pulsar-client`](reference-cli-tools.md#pulsar-client). The pulsar-client tool enables you to consume and produce messages to a Pulsar topic in a running cluster. - -### Consume a message - -The following command consumes a message with the subscription name `first-subscription` to the `my-topic` topic: - -```bash -bin/pulsar-client consume my-topic -s "first-subscription" -``` - -If the message has been successfully consumed, you will see a confirmation like the following in the `pulsar-client` logs: - -``` -22:17:16.781 [main] INFO org.apache.pulsar.client.cli.PulsarClientTool - 1 messages successfully consumed -``` - -:::tip - -As you have noticed that we do not explicitly create the `my-topic` topic, from which we consume the message. When you consume a message from a topic that does not yet exist, Pulsar creates that topic for you automatically. Producing a message to a topic that does not exist will automatically create that topic for you as well. - -::: - -### Produce a message - -The following command produces a message saying `hello-pulsar` to the `my-topic` topic: - -```bash -bin/pulsar-client produce my-topic --messages "hello-pulsar" -``` - -If the message has been successfully published to the topic, you will see a confirmation like the following in the `pulsar-client` logs: - -``` -22:21:08.693 [main] INFO org.apache.pulsar.client.cli.PulsarClientTool - 1 messages successfully produced -``` - -## Stop Pulsar standalone - -Press `Ctrl+C` to stop a local standalone Pulsar. - -:::tip - -If the service runs as a background process using the `bin/pulsar-daemon start standalone` command, then use the `bin/pulsar-daemon stop standalone` command to stop the service. -For more information, see [pulsar-daemon](reference-cli-tools.md#pulsar-daemon). - -::: +````mdx-code-block +import {Redirect} from '@docusaurus/router'; + +```` diff --git a/site2/website/versioned_docs/version-2.10.x/standalone.md b/site2/website/versioned_docs/version-2.10.x/standalone.md index 3d463d635558b..e279db20486ff 100644 --- a/site2/website/versioned_docs/version-2.10.x/standalone.md +++ b/site2/website/versioned_docs/version-2.10.x/standalone.md @@ -5,264 +5,8 @@ sidebar_label: "Run Pulsar locally" original_id: standalone --- -For local development and testing, you can run Pulsar in standalone mode on your machine. The standalone mode includes a Pulsar broker, the necessary [RocksDB](http://rocksdb.org/) and BookKeeper components running inside of a single Java Virtual Machine (JVM) process. - -> **Pulsar in production?** -> If you're looking to run a full production Pulsar installation, see the [Deploying a Pulsar instance](deploy-bare-metal.md) guide. - -## Install Pulsar standalone - -This tutorial guides you through every step of installing Pulsar locally. - -### System requirements - -Currently, Pulsar is available for 64-bit **macOS**, **Linux**, and **Windows**. To use Pulsar, you need to install 64-bit JRE/JDK 8 or later versions - -:::tip - -By default, Pulsar allocates 2G JVM heap memory to start. It can be changed in `conf/pulsar_env.sh` file under `PULSAR_MEM`. This is extra options passed into JVM. - -::: - -:::note - -Broker is only supported on 64-bit JVM. - -::: - -### Install Pulsar using binary release - -To get started with Pulsar, download a binary tarball release in one of the following ways: - -* download from the Apache mirror (Pulsar @pulsar:version@ binary release) - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - - $ wget pulsar:binary_release_url - - ``` - -After you download the tarball, untar it and use the `cd` command to navigate to the resulting directory: - -```bash - -$ tar xvfz apache-pulsar-@pulsar:version@-bin.tar.gz -$ cd apache-pulsar-@pulsar:version@ - -``` - -#### What your package contains - -The Pulsar binary package initially contains the following directories: - -Directory | Contains -:---------|:-------- -`bin` | Pulsar's command-line tools, such as [`pulsar`](reference-cli-tools.md#pulsar) and [`pulsar-admin`](/tools/pulsar-admin/). -`conf` | Configuration files for Pulsar, including [broker configuration](reference-configuration.md#broker) and more.
**Note:** Pulsar standalone uses RocksDB as the local metadata store and its configuration file path [`metadataStoreConfigPath`](reference-configuration.md) is configurable in the `standalone.conf` file. For more information about the configurations of RocksDB, see [here](https://github.com/facebook/rocksdb/blob/main/examples/rocksdb_option_file_example.ini) and related [documentation](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide). -`examples` | A Java JAR file containing [Pulsar Functions](functions-overview.md) example. -`instances` | Artifacts created for [Pulsar Functions](functions-overview.md). -`lib` | The [JAR](https://en.wikipedia.org/wiki/JAR_(file_format)) files used by Pulsar. -`licenses` | License files, in the`.txt` form, for various components of the Pulsar [codebase](https://github.com/apache/pulsar). - -These directories are created once you begin running Pulsar. - -Directory | Contains -:---------|:-------- -`data` | The data storage directory used by RocksDB and BookKeeper. -`logs` | Logs created by the installation. - -:::tip - -If you want to use builtin connectors and tiered storage offloaders, you can install them according to the following instructions： -* [Install builtin connectors (optional)](#install-builtin-connectors-optional) -* [Install tiered storage offloaders (optional)](#install-tiered-storage-offloaders-optional) -Otherwise, skip this step and perform the next step [Start Pulsar standalone](#start-pulsar-standalone). Pulsar can be successfully installed without installing bulitin connectors and tiered storage offloaders. - -::: - -### Install builtin connectors (optional) - -Since `2.1.0-incubating` release, Pulsar releases a separate binary distribution, containing all the `builtin` connectors. -To enable those `builtin` connectors, you can download the connectors tarball release in one of the following ways: - -* download from the Apache mirror Pulsar IO Connectors @pulsar:version@ release - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - - $ wget pulsar:connector_release_url/{connector}-@pulsar:version@.nar - - ``` - -After you download the nar file, copy the file to the `connectors` directory in the pulsar directory. -For example, if you download the `pulsar-io-aerospike-@pulsar:version@.nar` connector file, enter the following commands: - -```bash - -$ mkdir connectors -$ mv pulsar-io-aerospike-@pulsar:version@.nar connectors - -$ ls connectors -pulsar-io-aerospike-@pulsar:version@.nar -... - -``` - -:::note - -* If you are running Pulsar in a bare metal cluster, make sure `connectors` tarball is unzipped in every pulsar directory of the broker (or in every pulsar directory of function-worker if you are running a separate worker cluster for Pulsar Functions). -* If you are [running Pulsar in Docker](getting-started-docker.md) or deploying Pulsar using a docker image (e.g. [K8S](deploy-kubernetes.md) or [DC/OS](https://dcos.io/), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled [all builtin connectors](io-overview.md#working-with-connectors). - -::: - -### Install tiered storage offloaders (optional) - -:::tip - -- Since `2.2.0` release, Pulsar releases a separate binary distribution, containing the tiered storage offloaders. -- To enable tiered storage feature, follow the instructions below; otherwise skip this section. - -::: - -To get started with [tiered storage offloaders](concepts-tiered-storage.md), you need to download the offloaders tarball release on every broker node in one of the following ways: - -* download from the Apache mirror Pulsar Tiered Storage Offloaders @pulsar:version@ release - -* download from the Pulsar [downloads page](pulsar:download_page_url) - -* download from the Pulsar [releases page](https://github.com/apache/pulsar/releases/latest) - -* use [wget](https://www.gnu.org/software/wget): - - ```shell - - $ wget pulsar:offloader_release_url - - ``` - -After you download the tarball, untar the offloaders package and copy the offloaders as `offloaders` -in the pulsar directory: - -```bash - -$ tar xvfz apache-pulsar-offloaders-@pulsar:version@-bin.tar.gz - -// you will find a directory named `apache-pulsar-offloaders-@pulsar:version@` in the pulsar directory -// then copy the offloaders - -$ mv apache-pulsar-offloaders-@pulsar:version@/offloaders offloaders - -$ ls offloaders -tiered-storage-jcloud-@pulsar:version@.nar - -``` - -For more information on how to configure tiered storage, see [Tiered storage cookbook](cookbooks-tiered-storage.md). - -:::note - -* If you are running Pulsar in a bare metal cluster, make sure that `offloaders` tarball is unzipped in every broker's pulsar directory. -* If you are [running Pulsar in Docker](getting-started-docker.md) or deploying Pulsar using a docker image (e.g. [K8S](deploy-kubernetes.md) or DC/OS), you can use the `apachepulsar/pulsar-all` image instead of the `apachepulsar/pulsar` image. `apachepulsar/pulsar-all` image has already bundled tiered storage offloaders. - -::: - -## Start Pulsar standalone - -Once you have an up-to-date local copy of the release, you can start a local cluster using the [`pulsar`](reference-cli-tools.md#pulsar) command, which is stored in the `bin` directory, and specifying that you want to start Pulsar in standalone mode. - -```bash - -$ bin/pulsar standalone - -``` - -If you have started Pulsar successfully, you will see `INFO`-level log messages like this: - -```bash - -21:59:29.327 [DLM-/stream/storage-OrderedScheduler-3-0] INFO org.apache.bookkeeper.stream.storage.impl.sc.StorageContainerImpl - Successfully started storage container (0). -21:59:34.576 [main] INFO org.apache.pulsar.broker.authentication.AuthenticationService - Authentication is disabled -21:59:34.576 [main] INFO org.apache.pulsar.websocket.WebSocketService - Pulsar WebSocket Service started - -``` - -:::tip - -* The service is running on your terminal, which is under your direct control. If you need to run other commands, open a new terminal window. - -::: - -You can also run the service as a background process using the `bin/pulsar-daemon start standalone` command. For more information, see [pulsar-daemon](reference-cli-tools.md#pulsar-daemon). -> -> * By default, there is no encryption, authentication, or authorization configured. Apache Pulsar can be accessed from remote server without any authorization. Please do check [Security Overview](security-overview.md) document to secure your deployment. -> -> * When you start a local standalone cluster, a `public/default` [namespace](concepts-messaging.md#namespaces) is created automatically. The namespace is used for development purposes. All Pulsar topics are managed within namespaces. For more information, see [Topics](concepts-messaging.md#topics). - -## Use Pulsar standalone - -Pulsar provides a CLI tool called [`pulsar-client`](reference-cli-tools.md#pulsar-client). The pulsar-client tool enables you to consume and produce messages to a Pulsar topic in a running cluster. - -### Consume a message - -The following command consumes a message with the subscription name `first-subscription` to the `my-topic` topic: - -```bash - -$ bin/pulsar-client consume my-topic -s "first-subscription" - -``` - -If the message has been successfully consumed, you will see a confirmation like the following in the `pulsar-client` logs: - -``` - -22:17:16.781 [main] INFO org.apache.pulsar.client.cli.PulsarClientTool - 1 messages successfully consumed - -``` - -:::tip - -As you have noticed that we do not explicitly create the `my-topic` topic, from which we consume the message. When you consume a message from a topic that does not yet exist, Pulsar creates that topic for you automatically. Producing a message to a topic that does not exist will automatically create that topic for you as well. - -::: - -### Produce a message - -The following command produces a message saying `hello-pulsar` to the `my-topic` topic: - -```bash - -$ bin/pulsar-client produce my-topic --messages "hello-pulsar" - -``` - -If the message has been successfully published to the topic, you will see a confirmation like the following in the `pulsar-client` logs: - -``` - -22:21:08.693 [main] INFO org.apache.pulsar.client.cli.PulsarClientTool - 1 messages successfully produced - -``` - -## Stop Pulsar standalone - -Press `Ctrl+C` to stop a local standalone Pulsar. - -:::tip - -If the service runs as a background process using the `bin/pulsar-daemon start standalone` command, then use the `bin/pulsar-daemon stop standalone` command to stop the service. -For more information, see [pulsar-daemon](reference-cli-tools.md#pulsar-daemon). - -::: +````mdx-code-block +import {Redirect} from '@docusaurus/router'; + +```` From 40d4456fa2c2060dae73952eda442b218e6256fb Mon Sep 17 00:00:00 2001 From: coderzc Date: Tue, 18 Oct 2022 11:11:54 +0800 Subject: [PATCH 32/32] fix getChildren in memoryMetadata and etcdMetadata --- .../metadata/impl/EtcdMetadataStore.java | 5 ++-- .../impl/LocalMemoryMetadataStore.java | 2 +- .../pulsar/metadata/MetadataStoreTest.java | 29 +++++++++++++++++++ .../impl/LocalMemoryMetadataStoreTest.java | 4 +-- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java index bbe7034d99093..c0dc75f130b5f 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/EtcdMetadataStore.java @@ -385,11 +385,12 @@ private void handleBatchOperationResult(TxnResponse txnResponse, case GET_CHILDREN: { OpGetChildren getChildren = op.asGetChildren(); GetResponse gr = txnResponse.getGetResponses().get(getIdx++); - String basePath = getChildren.getPath() + "/"; + String basePath = + getChildren.getPath().equals("/") ? "/" : getChildren.getPath() + "/"; Set children = gr.getKvs().stream() .map(kv -> kv.getKey().toString(StandardCharsets.UTF_8)) - .map(p -> p.replace(basePath, "")) + .map(p -> p.replaceFirst(basePath, "")) // Only return first-level children .map(k -> k.split("/", 2)[0]) .collect(Collectors.toCollection(TreeSet::new)); diff --git a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java index b520745a40659..edd0dedf45ad1 100644 --- a/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java +++ b/pulsar-metadata/src/main/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStore.java @@ -129,7 +129,7 @@ public CompletableFuture> getChildrenFromStore(String path) { Set children = new TreeSet<>(); map.subMap(firstKey, false, lastKey, false).forEach((key, value) -> { - String relativePath = key.replace(firstKey, ""); + String relativePath = key.replaceFirst(firstKey, ""); // Only return first-level children String child = relativePath.split("/", 2)[0]; diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java index b8a091fef2549..9e0c6887d94f9 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/MetadataStoreTest.java @@ -28,6 +28,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; @@ -500,4 +501,32 @@ public void testConcurrentDelete(String provider, Supplier urlSupplier) assertTrue(f1.isCompletedExceptionally() && !f2.isCompletedExceptionally() || ! f1.isCompletedExceptionally() && f2.isCompletedExceptionally()); } + + @Test(dataProvider = "impl") + public void testGetChildren(String provider, Supplier urlSupplier) throws Exception { + @Cleanup + MetadataStore store = MetadataStoreFactory.create(urlSupplier.get(), MetadataStoreConfig.builder().build()); + + store.put("/a/a-1", "value1".getBytes(StandardCharsets.UTF_8), Optional.empty()).join(); + store.put("/a/a-2", "value1".getBytes(StandardCharsets.UTF_8), Optional.empty()).join(); + store.put("/b/c/b/1", "value1".getBytes(StandardCharsets.UTF_8), Optional.empty()).join(); + + List subPaths = store.getChildren("/").get(); + Set expectedSet = "ZooKeeper".equals(provider) ? Set.of("a", "b", "zookeeper") : Set.of("a", "b"); + for (String subPath : subPaths) { + assertTrue(expectedSet.contains(subPath)); + } + + List subPaths2 = store.getChildren("/a").get(); + Set expectedSet2 = Set.of("a-1", "a-2"); + for (String subPath : subPaths2) { + assertTrue(expectedSet2.contains(subPath)); + } + + List subPaths3 = store.getChildren("/b").get(); + Set expectedSet3 = Set.of("c"); + for (String subPath : subPaths3) { + assertTrue(expectedSet3.contains(subPath)); + } + } } diff --git a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java index 1df9507603355..013b7d7f4270d 100644 --- a/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java +++ b/pulsar-metadata/src/test/java/org/apache/pulsar/metadata/impl/LocalMemoryMetadataStoreTest.java @@ -86,7 +86,7 @@ public void testNotifyEvent() throws Exception { assertEquals((long) event2.getExpectedVersion(), exptectedVersion); assertEquals(event2.getType(), NotificationType.Modified); } - + // (3) delete node sync.notifiedEvents.remove(path); store1.delete(path, Optional.of(exptectedVersion)).join(); @@ -176,7 +176,7 @@ public void testSyncListener() throws Exception { store1.put(path, value1, Optional.empty()).join(); assertTrue(store1.exists(path).join()); - + Stat stats = store1.get(path).get().get().getStat(); MetadataEvent event = new MetadataEvent(path, value2, EMPTY_SET, stats.getVersion(), stats.getModificationTimestamp() + 1, sync.clusterName, NotificationType.Modified);