From 697ff7023d3b99a1c0fb1c871e6bb939cf9db14f Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 16:37:47 +0800 Subject: [PATCH 01/13] CI Tests --- .../workflows/ci-unit-broker-long-time.yml | 109 ++++++++++++++++++ .github/workflows/ci-unit-broker.yml | 2 +- 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-unit-broker-long-time.yml diff --git a/.github/workflows/ci-unit-broker-long-time.yml b/.github/workflows/ci-unit-broker-long-time.yml new file mode 100644 index 0000000000000..8567ff55aa665 --- /dev/null +++ b/.github/workflows/ci-unit-broker-long-time.yml @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Long - Time +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests + + - name: run unit tests pulsar broker PulsarFunctionE2ESecurityTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=PulsarFunctionE2ESecurityTest' -pl pulsar-broker + + - name: run unit tests pulsar broker SchemaCompatibilityCheckTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=SchemaCompatibilityCheckTest' -pl pulsar-broker + + - name: run unit tests pulsar broker CompactedTopicTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=CompactedTopicTest' -pl pulsar-broker + + - name: run unit tests pulsar broker LoadBalancerTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=LoadBalancerTest' -pl pulsar-broker + + - name: run unit tests pulsar broker ConsumerBatchReceiveTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerBatchReceiveTest' -pl pulsar-broker + + - name: run unit tests pulsar broker KeySharedSubscriptionTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=KeySharedSubscriptionTest' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index 45eebaca2428d..a51bd987e8823 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -73,7 +73,7 @@ jobs: run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() From beec3e61cccdf76a59c7462b6841400296004fd0 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 16:40:59 +0800 Subject: [PATCH 02/13] CI Tests --- .../java/org/apache/pulsar/broker/admin/impl/BrokersBase.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java index 3488fd8a88b69..072e91ca3c6a2 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/admin/impl/BrokersBase.java @@ -67,7 +67,9 @@ import io.swagger.annotations.ApiResponse; import io.swagger.annotations.ApiResponses; - +/** + * Broker admin base. + */ public class BrokersBase extends AdminResource { private static final Logger LOG = LoggerFactory.getLogger(BrokersBase.class); private int serviceConfigZkVersion = -1; From 843a2a046829bc0e0bad8ef350f91ca2830767b2 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 17:26:52 +0800 Subject: [PATCH 03/13] CI Tests --- .github/workflows/ci-unit-broker-flaky.yml | 105 ++++++++++++++++++ .../ci-unit-broker-publish-throttle.yml | 101 +++++++++++++++++ .github/workflows/ci-unit-broker.yml | 2 +- .../pulsar/broker/service/BrokerService.java | 4 +- .../MessagePublishBufferThrottleTest.java | 67 ++++++----- 5 files changed, 250 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/ci-unit-broker-flaky.yml create mode 100644 .github/workflows/ci-unit-broker-publish-throttle.yml diff --git a/.github/workflows/ci-unit-broker-flaky.yml b/.github/workflows/ci-unit-broker-flaky.yml new file mode 100644 index 0000000000000..14fe3bd828281 --- /dev/null +++ b/.github/workflows/ci-unit-broker-flaky.yml @@ -0,0 +1,105 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Publish - Flaky +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests + + - name: run unit tests pulsar broker MessageIdTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest' -pl pulsar-broker + + - name: run unit tests pulsar broker UnAcknowledgedMessagesTimeoutTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=UnAcknowledgedMessagesTimeoutTest' -pl pulsar-broker + + - name: run unit tests pulsar broker ConsumerPreciseDispatcherFlowControl + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerPreciseDispatcherFlowControl' -pl pulsar-broker + + - name: run unit tests pulsar broker SimpleSchemaTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=SimpleSchemaTest' -pl pulsar-broker + + - name: run unit tests pulsar broker ConsumerDedupPermitsUpdate + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerDedupPermitsUpdate' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-publish-throttle.yml b/.github/workflows/ci-unit-broker-publish-throttle.yml new file mode 100644 index 0000000000000..8ada9ece69bcb --- /dev/null +++ b/.github/workflows/ci-unit-broker-publish-throttle.yml @@ -0,0 +1,101 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Publish - Throttle +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests + + - name: run unit tests pulsar broker MessagePublishBufferThrottleTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest' -pl pulsar-broker + + - name: run unit tests pulsar broker PrecisTopicPublishRateThrottleTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=PrecisTopicPublishRateThrottleTest' -pl pulsar-broker + + - name: run unit tests pulsar broker PublishRateLimiterTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=PublishRateLimiterTest' -pl pulsar-broker + + - name: run unit tests pulsar broker MessagePublishThrottlingTest + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishThrottlingTest' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index a51bd987e8823..25241e11713b0 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -73,7 +73,7 @@ jobs: run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java index 0790e7399a7f3..efde4b8807b57 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/BrokerService.java @@ -2140,7 +2140,9 @@ public Optional getListenPortTls() { return Optional.empty(); } } - private void checkMessagePublishBuffer() { + + @VisibleForTesting + void checkMessagePublishBuffer() { AtomicLong currentMessagePublishBufferBytes = new AtomicLong(); foreachProducer(producer -> currentMessagePublishBufferBytes.addAndGet(producer.getCnx().getMessagePublishBufferSize())); if (currentMessagePublishBufferBytes.get() >= maxMessagePublishBufferBytes diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java index 5397725a99c67..bce64a4cb6029 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java @@ -51,9 +51,9 @@ public void testMessagePublishBufferThrottleDisabled() throws Exception { super.baseSetup(); final String topic = "persistent://prop/ns-abc/testMessagePublishBufferThrottleDisabled"; Producer producer = pulsarClient.newProducer() - .topic(topic) - .producerName("producer-name") - .create(); + .topic(topic) + .producerName("producer-name") + .create(); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(Long.MAX_VALUE / 2); @@ -76,22 +76,22 @@ public void testMessagePublishBufferThrottleDisabled() throws Exception { @Test public void testMessagePublishBufferThrottleEnable() throws Exception { conf.setMaxMessagePublishBufferSizeInMB(1); - conf.setMessagePublishBufferCheckIntervalInMillis(2); + conf.setMessagePublishBufferCheckIntervalInMillis(Integer.MAX_VALUE); super.baseSetup(); - Thread.sleep(4); Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); final String topic = "persistent://prop/ns-abc/testMessagePublishBufferThrottleEnable"; Producer producer = pulsarClient.newProducer() - .topic(topic) - .producerName("producer-name") - .create(); + .topic(topic) + .producerName("producer-name") + .create(); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(Long.MAX_VALUE / 2); - Thread.sleep(4); - Assert.assertTrue(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); + Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); // The first message can publish success, but the second message should be blocked producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); + getPulsar().getBrokerService().checkMessagePublishBuffer(); + Assert.assertTrue(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); MessageId messageId = null; try { messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); @@ -102,8 +102,8 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { Assert.assertNull(messageId); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(0L); - Thread.sleep(4); - + getPulsar().getBrokerService().checkMessagePublishBuffer(); + Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); List> futures = new ArrayList<>(); // Make sure the producer can publish succeed. for (int i = 0; i < 10; i++) { @@ -113,7 +113,6 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { for (CompletableFuture future : futures) { Assert.assertNotNull(future.get()); } - Thread.sleep(4); Assert.assertEquals(pulsar.getBrokerService().getCurrentMessagePublishBufferSize(), 0L); super.internalCleanup(); } @@ -121,25 +120,23 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { @Test public void testBlockByPublishRateLimiting() throws Exception { conf.setMaxMessagePublishBufferSizeInMB(1); - conf.setMessagePublishBufferCheckIntervalInMillis(2); + conf.setMessagePublishBufferCheckIntervalInMillis(Integer.MAX_VALUE); super.baseSetup(); - Thread.sleep(4); Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); - final String topic = "persistent://prop/ns-abc/testMessagePublishBufferThrottleEnable"; + final String topic = "persistent://prop/ns-abc/testBlockByPublishRateLimiting"; Producer producer = pulsarClient.newProducer() - .topic(topic) - .producerName("producer-name") - .create(); + .topic(topic) + .producerName("producer-name") + .create(); Topic topicRef = pulsar.getBrokerService().getTopicReference(topic).get(); Assert.assertNotNull(topicRef); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(Long.MAX_VALUE / 2); + Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); - Thread.sleep(4); - ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setAutoReadDisabledRateLimiting(true); - ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(0); - Thread.sleep(4); - Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); + // Block by publish buffer. + getPulsar().getBrokerService().checkMessagePublishBuffer(); + Assert.assertTrue(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); MessageId messageId = null; try { messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); @@ -149,9 +146,26 @@ public void testBlockByPublishRateLimiting() throws Exception { } Assert.assertNull(messageId); - ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setAutoReadDisabledRateLimiting(false); + // Block by publish rate. + ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(0L); + getPulsar().getBrokerService().checkMessagePublishBuffer(); + ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setAutoReadDisabledRateLimiting(true); + ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().disableCnxAutoRead(); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().enableCnxAutoRead(); + try { + messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); + Assert.fail("should failed, because producer blocked by publish rate limiting"); + } catch (TimeoutException e) { + // No-op + } + + Assert.assertNull(messageId); + + // Resume message publish. + ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setAutoReadDisabledRateLimiting(false); + ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().enableCnxAutoRead(); + Assert.assertFalse(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); List> futures = new ArrayList<>(); // Make sure the producer can publish succeed. for (int i = 0; i < 10; i++) { @@ -161,8 +175,7 @@ public void testBlockByPublishRateLimiting() throws Exception { for (CompletableFuture future : futures) { Assert.assertNotNull(future.get()); } - Thread.sleep(4); Assert.assertEquals(pulsar.getBrokerService().getCurrentMessagePublishBufferSize(), 0L); super.internalCleanup(); } -} +} \ No newline at end of file From 5f0e7453e58d77bf134d6fd5dc8cf9b3ac0854ae Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 17:47:42 +0800 Subject: [PATCH 04/13] CI Tests --- .github/workflows/ci-unit-broker-flaky.yml | 8 ++++++-- .github/workflows/ci-unit-broker-long-time.yml | 2 +- .github/workflows/ci-unit-broker-others.yml | 2 +- .github/workflows/ci-unit-broker-publish-throttle.yml | 2 +- .github/workflows/ci-unit-broker-sasl.yml | 2 +- .github/workflows/ci-unit-broker.yml | 6 +++--- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-unit-broker-flaky.yml b/.github/workflows/ci-unit-broker-flaky.yml index 14fe3bd828281..fe3fc8b72260e 100644 --- a/.github/workflows/ci-unit-broker-flaky.yml +++ b/.github/workflows/ci-unit-broker-flaky.yml @@ -17,7 +17,7 @@ # under the License. # -name: CI - Unit - Brokers - Publish - Flaky +name: CI - Unit - Brokers - Flaky on: pull_request: branches: @@ -66,7 +66,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker -am - name: run unit tests pulsar broker MessageIdTest if: steps.docs.outputs.changed_only == 'no' @@ -88,6 +88,10 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerDedupPermitsUpdate' -pl pulsar-broker + - name: run unit tests pulsar broker NamespaceOwnershipListenerTests + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=NamespaceOwnershipListenerTests' -pl pulsar-broker + - name: package surefire artifacts if: failure() run: | diff --git a/.github/workflows/ci-unit-broker-long-time.yml b/.github/workflows/ci-unit-broker-long-time.yml index 8567ff55aa665..c061cc2420b29 100644 --- a/.github/workflows/ci-unit-broker-long-time.yml +++ b/.github/workflows/ci-unit-broker-long-time.yml @@ -66,7 +66,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker -am - name: run unit tests pulsar broker PulsarFunctionE2ESecurityTest if: steps.docs.outputs.changed_only == 'no' diff --git a/.github/workflows/ci-unit-broker-others.yml b/.github/workflows/ci-unit-broker-others.yml index e1d3495bfdc1b..4358992e8d658 100644 --- a/.github/workflows/ci-unit-broker-others.yml +++ b/.github/workflows/ci-unit-broker-others.yml @@ -66,7 +66,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker -am - name: run unit tests pulsar broker reader test if: steps.docs.outputs.changed_only == 'no' diff --git a/.github/workflows/ci-unit-broker-publish-throttle.yml b/.github/workflows/ci-unit-broker-publish-throttle.yml index 8ada9ece69bcb..7af2f9eca8ba8 100644 --- a/.github/workflows/ci-unit-broker-publish-throttle.yml +++ b/.github/workflows/ci-unit-broker-publish-throttle.yml @@ -66,7 +66,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker -am - name: run unit tests pulsar broker MessagePublishBufferThrottleTest if: steps.docs.outputs.changed_only == 'no' diff --git a/.github/workflows/ci-unit-broker-sasl.yml b/.github/workflows/ci-unit-broker-sasl.yml index b01e44b0e8015..aa975ab20aef0 100644 --- a/.github/workflows/ci-unit-broker-sasl.yml +++ b/.github/workflows/ci-unit-broker-sasl.yml @@ -57,7 +57,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker-auth-sasl -am - name: run unit tests pulsar auth sasl if: steps.docs.outputs.changed_only == 'no' diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index 25241e11713b0..a35890d4bbbdf 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -17,7 +17,7 @@ # under the License. # -name: CI - Unit - Brokers +name: CI - Unit - Brokers - Default on: pull_request: branches: @@ -66,14 +66,14 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-broker -am - name: run unit test pulsar-broker if: steps.docs.outputs.changed_only == 'no' run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() From 79d1b8e1cb48077a30c8bcde666a4513892f9710 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 18:16:11 +0800 Subject: [PATCH 05/13] CI Tests --- .github/workflows/ci-unit-broker-flaky.yml | 24 +-------- .../workflows/ci-unit-broker-long-time.yml | 24 +-------- .github/workflows/ci-unit-broker-others.yml | 52 +------------------ .../ci-unit-broker-publish-throttle.yml | 16 +----- .github/workflows/ci-unit-proxy.yaml | 2 +- 5 files changed, 9 insertions(+), 109 deletions(-) diff --git a/.github/workflows/ci-unit-broker-flaky.yml b/.github/workflows/ci-unit-broker-flaky.yml index fe3fc8b72260e..4eaf30191145b 100644 --- a/.github/workflows/ci-unit-broker-flaky.yml +++ b/.github/workflows/ci-unit-broker-flaky.yml @@ -68,29 +68,9 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker MessageIdTest + - name: run unit tests pulsar broker flaky tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest' -pl pulsar-broker - - - name: run unit tests pulsar broker UnAcknowledgedMessagesTimeoutTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=UnAcknowledgedMessagesTimeoutTest' -pl pulsar-broker - - - name: run unit tests pulsar broker ConsumerPreciseDispatcherFlowControl - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerPreciseDispatcherFlowControl' -pl pulsar-broker - - - name: run unit tests pulsar broker SimpleSchemaTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=SimpleSchemaTest' -pl pulsar-broker - - - name: run unit tests pulsar broker ConsumerDedupPermitsUpdate - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerDedupPermitsUpdate' -pl pulsar-broker - - - name: run unit tests pulsar broker NamespaceOwnershipListenerTests - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=NamespaceOwnershipListenerTests' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest,UnAcknowledgedMessagesTimeoutTest,ConsumerPreciseDispatcherFlowControl,SimpleSchemaTest,ConsumerDedupPermitsUpdate,NamespaceOwnershipListenerTests' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-long-time.yml b/.github/workflows/ci-unit-broker-long-time.yml index c061cc2420b29..647ee7697726d 100644 --- a/.github/workflows/ci-unit-broker-long-time.yml +++ b/.github/workflows/ci-unit-broker-long-time.yml @@ -68,29 +68,9 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker PulsarFunctionE2ESecurityTest + - name: run unit tests pulsar broker long time tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PulsarFunctionE2ESecurityTest' -pl pulsar-broker - - - name: run unit tests pulsar broker SchemaCompatibilityCheckTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=SchemaCompatibilityCheckTest' -pl pulsar-broker - - - name: run unit tests pulsar broker CompactedTopicTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=CompactedTopicTest' -pl pulsar-broker - - - name: run unit tests pulsar broker LoadBalancerTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=LoadBalancerTest' -pl pulsar-broker - - - name: run unit tests pulsar broker ConsumerBatchReceiveTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=ConsumerBatchReceiveTest' -pl pulsar-broker - - - name: run unit tests pulsar broker KeySharedSubscriptionTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=KeySharedSubscriptionTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=PulsarFunctionE2ESecurityTest,SchemaCompatibilityCheckTest,CompactedTopicTest,LoadBalancerTest,ConsumerBatchReceiveTest,KeySharedSubscriptionTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-others.yml b/.github/workflows/ci-unit-broker-others.yml index 4358992e8d658..7f4d6381ba776 100644 --- a/.github/workflows/ci-unit-broker-others.yml +++ b/.github/workflows/ci-unit-broker-others.yml @@ -68,57 +68,9 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker reader test + - name: run unit tests pulsar broker other tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=ReaderTest' -pl pulsar-broker - - - name: run unit tests pulsar broker rack aware test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=RackAwareTest' -pl pulsar-broker - - - name: run unit tests pulsar broker simple producer consumer test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=SimpleProducerConsumerTest' -pl pulsar-broker - - - name: run unit tests pulsar broker V1 producer consumer test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=V1_ProducerConsumerTest' -pl pulsar-broker - - - name: run unit tests pulsar broker persistent failover end to end test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PersistentFailoverE2ETest' -pl pulsar-broker - - - name: run unit tests pulsar broker client integration test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=BrokerClientIntegrationTest' -pl pulsar-broker - - - name: run unit tests pulsar broker replicatior rate limiter test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=ReplicatorRateLimiterTest' -pl pulsar-broker - - - name: run unit tests pulsar broker persistent dispatcher failover consumer test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PersistentDispatcherFailoverConsumerTest' -pl pulsar-broker - - - name: run unit tests pulsar broker admin api test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=AdminApiTest' -pl pulsar-broker - - - name: run unit tests pulsar broker v1 admin api test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=V1_AdminApiTest' -pl pulsar-broker - - - name: run unit tests pulsar broker compaction test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=CompactionTest' -pl pulsar-broker - - - name: run unit tests pulsar broker batch message test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=BatchMessageTest' -pl pulsar-broker - - - name: run unit tests pulsar broker partitioned topics schema test - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PartitionedTopicsSchemaTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=ReaderTest,RackAwareTest,SimpleProducerConsumerTest,V1_ProducerConsumerTest,PersistentFailoverE2ETest,BrokerClientIntegrationTest,ReplicatorRateLimiterTest,PersistentDispatcherFailoverConsumerTest,AdminApiTest,V1_AdminApiTest,CompactionTest,BatchMessageTest,PartitionedTopicsSchemaTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-publish-throttle.yml b/.github/workflows/ci-unit-broker-publish-throttle.yml index 7af2f9eca8ba8..5e33014072f45 100644 --- a/.github/workflows/ci-unit-broker-publish-throttle.yml +++ b/.github/workflows/ci-unit-broker-publish-throttle.yml @@ -68,21 +68,9 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker MessagePublishBufferThrottleTest + - name: run unit tests pulsar broker publish throttle if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest' -pl pulsar-broker - - - name: run unit tests pulsar broker PrecisTopicPublishRateThrottleTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PrecisTopicPublishRateThrottleTest' -pl pulsar-broker - - - name: run unit tests pulsar broker PublishRateLimiterTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=PublishRateLimiterTest' -pl pulsar-broker - - - name: run unit tests pulsar broker MessagePublishThrottlingTest - if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishThrottlingTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest,PrecisTopicPublishRateThrottleTest,PublishRateLimiterTest,MessagePublishThrottlingTest' -pl pulsar-broker --thread 1 - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml index e7f26fedd0c01..7c623e8751ff1 100644 --- a/.github/workflows/ci-unit-proxy.yaml +++ b/.github/workflows/ci-unit-proxy.yaml @@ -57,7 +57,7 @@ jobs: - name: run unit tests install by skip tests if: steps.docs.outputs.changed_only == 'no' - run: mvn clean install -DskipTests + run: mvn clean install -DskipTests -pl pulsar-proxy -am - name: run unit tests pulsar proxy if: steps.docs.outputs.changed_only == 'no' From b9ec0dc4484c3b372d471812c03ae287df644dca Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 18:52:23 +0800 Subject: [PATCH 06/13] CI Tests --- .github/workflows/ci-unit-broker-publish-throttle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-unit-broker-publish-throttle.yml b/.github/workflows/ci-unit-broker-publish-throttle.yml index 5e33014072f45..016b9f0cb2603 100644 --- a/.github/workflows/ci-unit-broker-publish-throttle.yml +++ b/.github/workflows/ci-unit-broker-publish-throttle.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker publish throttle if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest,PrecisTopicPublishRateThrottleTest,PublishRateLimiterTest,MessagePublishThrottlingTest' -pl pulsar-broker --thread 1 + run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest,PrecisTopicPublishRateThrottleTest,PublishRateLimiterTest,MessagePublishThrottlingTest' -pl pulsar-broker --threads 1 - name: package surefire artifacts if: failure() From a1969738bc26a089ff7ff8b3c3a7051c5b7ee346 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 19:57:52 +0800 Subject: [PATCH 07/13] CI Tests --- .github/workflows/ci-unit-broker-broker.yml | 89 +++++++++++++++++++ .github/workflows/ci-unit-broker-client.yml | 89 +++++++++++++++++++ .github/workflows/ci-unit-broker.yml | 2 +- ...temTopicBasedTopicPoliciesServiceTest.java | 6 ++ 4 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-unit-broker-broker.yml create mode 100644 .github/workflows/ci-unit-broker-client.yml diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml new file mode 100644 index 0000000000000..c9d4b99483769 --- /dev/null +++ b/.github/workflows/ci-unit-broker-broker.yml @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Broker +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests -pl pulsar-broker -am + + - name: run unit tests pulsar broker flaky tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker-client.yml b/.github/workflows/ci-unit-broker-client.yml new file mode 100644 index 0000000000000..da0a922991bef --- /dev/null +++ b/.github/workflows/ci-unit-broker-client.yml @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Client +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests -pl pulsar-broker -am + + - name: run unit tests pulsar broker flaky tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index a35890d4bbbdf..1259089f9ed2b 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -73,7 +73,7 @@ jobs: run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests, !org/apache/pulsar/client/**/*.java, !org/apache/pulsar/broker/**/*.java' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java index 3e51e350e3b0a..a83301c4c66a9 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/SystemTopicBasedTopicPoliciesServiceTest.java @@ -180,6 +180,12 @@ private void prepareData() throws PulsarAdminException { admin.namespaces().createNamespace(NAMESPACE1); admin.namespaces().createNamespace(NAMESPACE2); admin.namespaces().createNamespace(NAMESPACE3); + admin.lookups().lookupTopic(TOPIC1.toString()); + admin.lookups().lookupTopic(TOPIC2.toString()); + admin.lookups().lookupTopic(TOPIC3.toString()); + admin.lookups().lookupTopic(TOPIC4.toString()); + admin.lookups().lookupTopic(TOPIC5.toString()); + admin.lookups().lookupTopic(TOPIC6.toString()); systemTopicFactory = new NamespaceEventsSystemTopicFactory(pulsarClient); systemTopicBasedTopicPoliciesService = (SystemTopicBasedTopicPoliciesService) pulsar.getTopicPoliciesService(); } From 8f33a8a8fc0faee7e1c5246c6bbcfd5da6e7afc6 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 21:28:07 +0800 Subject: [PATCH 08/13] CI Tests --- .github/workflows/ci-unit-broker-broker.yml | 2 +- .github/workflows/ci-unit-broker-client.yml | 2 +- .../ci-unit-broker-publish-throttle.yml | 2 +- .github/workflows/ci-unit-broker.yml | 2 +- .../MessagePublishBufferThrottleTest.java | 25 ------------------- 5 files changed, 4 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml index c9d4b99483769..92ee0bb7b8043 100644 --- a/.github/workflows/ci-unit-broker-broker.yml +++ b/.github/workflows/ci-unit-broker-broker.yml @@ -68,7 +68,7 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker flaky tests + - name: run unit tests pulsar broker tests if: steps.docs.outputs.changed_only == 'no' run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java' -pl pulsar-broker diff --git a/.github/workflows/ci-unit-broker-client.yml b/.github/workflows/ci-unit-broker-client.yml index da0a922991bef..1b2dcfc12659c 100644 --- a/.github/workflows/ci-unit-broker-client.yml +++ b/.github/workflows/ci-unit-broker-client.yml @@ -68,7 +68,7 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker flaky tests + - name: run unit tests pulsar broker client tests if: steps.docs.outputs.changed_only == 'no' run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java' -pl pulsar-broker diff --git a/.github/workflows/ci-unit-broker-publish-throttle.yml b/.github/workflows/ci-unit-broker-publish-throttle.yml index 016b9f0cb2603..3dc0d518a9f4b 100644 --- a/.github/workflows/ci-unit-broker-publish-throttle.yml +++ b/.github/workflows/ci-unit-broker-publish-throttle.yml @@ -68,7 +68,7 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit tests pulsar broker publish throttle + - name: run unit tests pulsar broker publish throttle tests if: steps.docs.outputs.changed_only == 'no' run: mvn test -DfailIfNoTests=false '-Dtest=MessagePublishBufferThrottleTest,PrecisTopicPublishRateThrottleTest,PublishRateLimiterTest,MessagePublishThrottlingTest' -pl pulsar-broker --threads 1 diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index 1259089f9ed2b..d1a6006784183 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -68,7 +68,7 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn clean install -DskipTests -pl pulsar-broker -am - - name: run unit test pulsar-broker + - name: run unit test pulsar-broker default tests if: steps.docs.outputs.changed_only == 'no' run: | df -h diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java index bce64a4cb6029..8aa9c9a1cf387 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/MessagePublishBufferThrottleTest.java @@ -92,14 +92,6 @@ public void testMessagePublishBufferThrottleEnable() throws Exception { producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); getPulsar().getBrokerService().checkMessagePublishBuffer(); Assert.assertTrue(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); - MessageId messageId = null; - try { - messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); - Assert.fail("should failed, because producer blocked by publish buffer limiting"); - } catch (TimeoutException e) { - // No-op - } - Assert.assertNull(messageId); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(0L); getPulsar().getBrokerService().checkMessagePublishBuffer(); @@ -137,14 +129,6 @@ public void testBlockByPublishRateLimiting() throws Exception { // Block by publish buffer. getPulsar().getBrokerService().checkMessagePublishBuffer(); Assert.assertTrue(pulsar.getBrokerService().isReachMessagePublishBufferThreshold()); - MessageId messageId = null; - try { - messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); - Assert.fail("should failed, because producer blocked by publish buffer limiting"); - } catch (TimeoutException e) { - // No-op - } - Assert.assertNull(messageId); // Block by publish rate. ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setMessagePublishBufferSize(0L); @@ -153,15 +137,6 @@ public void testBlockByPublishRateLimiting() throws Exception { ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().disableCnxAutoRead(); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().enableCnxAutoRead(); - try { - messageId = producer.sendAsync(new byte[1024]).get(1, TimeUnit.SECONDS); - Assert.fail("should failed, because producer blocked by publish rate limiting"); - } catch (TimeoutException e) { - // No-op - } - - Assert.assertNull(messageId); - // Resume message publish. ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().setAutoReadDisabledRateLimiting(false); ((AbstractTopic)topicRef).producers.get("producer-name").getCnx().enableCnxAutoRead(); From 1c00b34f0246b91a313de8553e580ccc557994e4 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 22:17:33 +0800 Subject: [PATCH 09/13] CI Tests --- .github/workflows/ci-unit-broker-flaky.yml | 2 +- .github/workflows/ci-unit-broker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-unit-broker-flaky.yml b/.github/workflows/ci-unit-broker-flaky.yml index 4eaf30191145b..e23a89c951804 100644 --- a/.github/workflows/ci-unit-broker-flaky.yml +++ b/.github/workflows/ci-unit-broker-flaky.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker flaky tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest,UnAcknowledgedMessagesTimeoutTest,ConsumerPreciseDispatcherFlowControl,SimpleSchemaTest,ConsumerDedupPermitsUpdate,NamespaceOwnershipListenerTests' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest,UnAcknowledgedMessagesTimeoutTest,ConsumerPreciseDispatcherFlowControl,SimpleSchemaTest,ConsumerDedupPermitsUpdate,NamespaceOwnershipListenerTests,ServerCnxTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index d1a6006784183..195994d0cbb7f 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -73,7 +73,7 @@ jobs: run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests, !org/apache/pulsar/client/**/*.java, !org/apache/pulsar/broker/**/*.java' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests, !org/apache/pulsar/client/**/*.java, !org/apache/pulsar/broker/**/*.java, !ServerCnxTest' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() From da2a4fc61875b2fe3dcdcafce8460efd84f36404 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 23:15:40 +0800 Subject: [PATCH 10/13] CI Tests --- .github/workflows/ci-unit-broker-flaky.yml | 2 +- .github/workflows/ci-unit-broker.yml | 2 +- .../pulsar/broker/service/persistent/PersistentTopic.java | 6 ------ .../org/apache/pulsar/broker/service/ServerCnxTest.java | 8 +++++++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-unit-broker-flaky.yml b/.github/workflows/ci-unit-broker-flaky.yml index e23a89c951804..4eaf30191145b 100644 --- a/.github/workflows/ci-unit-broker-flaky.yml +++ b/.github/workflows/ci-unit-broker-flaky.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker flaky tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest,UnAcknowledgedMessagesTimeoutTest,ConsumerPreciseDispatcherFlowControl,SimpleSchemaTest,ConsumerDedupPermitsUpdate,NamespaceOwnershipListenerTests,ServerCnxTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=MessageIdTest,UnAcknowledgedMessagesTimeoutTest,ConsumerPreciseDispatcherFlowControl,SimpleSchemaTest,ConsumerDedupPermitsUpdate,NamespaceOwnershipListenerTests' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker.yml b/.github/workflows/ci-unit-broker.yml index 195994d0cbb7f..f61ec125812e0 100644 --- a/.github/workflows/ci-unit-broker.yml +++ b/.github/workflows/ci-unit-broker.yml @@ -73,7 +73,7 @@ jobs: run: | df -h free -h - mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest, !PulsarFunctionE2ESecurityTest, !SchemaCompatibilityCheckTest, !CompactedTopicTest, !LoadBalancerTest, !ConsumerBatchReceiveTest, !KeySharedSubscriptionTest, !MessagePublishBufferThrottleTest, !PrecisTopicPublishRateThrottleTest, !PublishRateLimiterTest, !MessagePublishThrottlingTest, !MessageIdTest, !UnAcknowledgedMessagesTimeoutTest, !ConsumerPreciseDispatcherFlowControl, !SimpleSchemaTest, !ConsumerDedupPermitsUpdate, !NamespaceOwnershipListenerTests, !org/apache/pulsar/client/**/*.java, !org/apache/pulsar/broker/**/*.java, !ServerCnxTest' -DfailIfNoTests=false -pl pulsar-broker + mvn test -e '-Dtest=!PersistentTransactionBufferTest,!ServerCnxTest,!AdminApiOffloadTest,!AdminApiSchemaValidationEnforced,!V1_AdminApiTest2,!ProxyPublishConsumeTlsTest,!PulsarFunctionE2ETest,!MessageIdSerialization,!AdminApiTest2,!PulsarFunctionLocalRunTest,!PartitionedProducerConsumerTest,!KafkaProducerSimpleConsumerTest,!MessagePublishThrottlingTest,!ReaderTest,!RackAwareTest,!SimpleProducerConsumerTest,!V1_ProducerConsumerTest,!PersistentFailoverE2ETest,!BrokerClientIntegrationTest,!ReplicatorRateLimiterTest,!PersistentDispatcherFailoverConsumerTest,!AdminApiTest,!V1_AdminApiTest,!CompactionTest,!BatchMessageTest,!PartitionedTopicsSchemaTest,!PulsarFunctionE2ESecurityTest,!SchemaCompatibilityCheckTest,!CompactedTopicTest,!LoadBalancerTest,!ConsumerBatchReceiveTest,!KeySharedSubscriptionTest,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest,!MessageIdTest,!UnAcknowledgedMessagesTimeoutTest,!ConsumerPreciseDispatcherFlowControl,!SimpleSchemaTest,!ConsumerDedupPermitsUpdate,!NamespaceOwnershipListenerTests,!org/apache/pulsar/client/**/*.java,!org/apache/pulsar/broker/**/*.java' -DfailIfNoTests=false -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java index f5cef0ee6f20c..a7fa72d762ff7 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/persistent/PersistentTopic.java @@ -147,16 +147,10 @@ public class PersistentTopic extends AbstractTopic implements Topic, AddEntryCal private static final double MESSAGE_EXPIRY_THRESHOLD = 1.5; private static final long POLICY_UPDATE_FAILURE_RETRY_TIME_SECONDS = 60; - - // Timestamp of when this topic was last seen active - private volatile long lastActive; // topic has every published chunked message since topic is loaded public boolean msgChunkPublished; - // Flag to signal that producer of this topic has published batch-message so, broker should not allow consumer which - // doesn't support batch-message - private volatile boolean hasBatchMessagePublished = false; private Optional dispatchRateLimiter = Optional.empty(); private Optional subscribeRateLimiter = Optional.empty(); public volatile long delayedDeliveryTickTimeMillis = 1000; diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java index 44a2966fe04b8..dd1176a7e8072 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/ServerCnxTest.java @@ -446,6 +446,7 @@ public void testDuplicateConcurrentProducerCommand() throws Exception { assertTrue(response instanceof CommandError); CommandError error = (CommandError) response; assertEquals(error.getError(), ServerError.ServiceNotReady); + channel.finish(); } @Test(timeOut = 30000) @@ -531,6 +532,7 @@ public void testNonExistentTopic() throws Exception { successSubName, 1 /* consumer id */, 1 /* request id */, SubType.Exclusive, 0, "test" /* consumer name */, 0); channel.writeInbound(newSubscribeCmd); assertTrue(getResponse() instanceof CommandError); + channel.finish(); } @Test(timeOut = 30000) @@ -560,6 +562,7 @@ public void testClusterAccess() throws Exception { "prod-name", Collections.emptyMap()); channel.writeInbound(clientCommand); assertTrue(getResponse() instanceof CommandError); + channel.finish(); } @Test(timeOut = 30000) @@ -598,6 +601,7 @@ public void testNonExistentTopicSuperUserAccess() throws Exception { assertTrue(topicRef.getSubscriptions().containsKey(successSubName)); assertTrue(topicRef.getSubscription(successSubName).getDispatcher().isConsumerConnected()); assertTrue(getResponse() instanceof CommandSuccess); + channel.finish(); } @Test(timeOut = 30000) @@ -752,6 +756,7 @@ public void testDuplicateConcurrentSubscribeCommand() throws Exception { assertTrue(response instanceof CommandError, "Response is not CommandError but " + response); CommandError error = (CommandError) response; assertEquals(error.getError(), ServerError.ServiceNotReady); + channel.finish(); } @Test(timeOut = 30000) @@ -1175,7 +1180,6 @@ public void testUnsupportedBatchMsgSubscribeCommand() throws Exception { PersistentTopic topicRef = (PersistentTopic) brokerService.getTopicReference(successTopicName).get(); topicRef.markBatchMessagePublished(); - // test SUBSCRIBE on topic and cursor creation success clientCommand = Commands.newSubscribe(successTopicName, failSubName, 2, 2, SubType.Exclusive, 0 /* priority */, "test" /* consumer name */, 0 /*avoid reseting cursor*/); @@ -1649,5 +1653,7 @@ public void testDelayedClosedProducer() throws Exception { Object response = getResponse(); assertTrue(response instanceof CommandSuccess); + + channel.finish(); } } From 6d8f68d83dadc06c82ffd6693b3f9383290ab768 Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 23:37:11 +0800 Subject: [PATCH 11/13] CI Tests --- .github/workflows/ci-unit-broker-broker.yml | 2 +- .github/workflows/ci-unit-broker-client.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml index 92ee0bb7b8043..81cda7e56c5f9 100644 --- a/.github/workflows/ci-unit-broker-broker.yml +++ b/.github/workflows/ci-unit-broker-broker.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java,!NamespaceOwnershipListenerTests,!AdminApiTest2,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-client.yml b/.github/workflows/ci-unit-broker-client.yml index 1b2dcfc12659c..633a54eea48b7 100644 --- a/.github/workflows/ci-unit-broker-client.yml +++ b/.github/workflows/ci-unit-broker-client.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker client tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java,!UnAcknowledgedMessagesTimeoutTest,!SimpleSchemaTest,!ConsumerDedupPermitsUpdate' -pl pulsar-broker - name: package surefire artifacts if: failure() From 0b5618bb5812b481948c5603443de8b14e132cfc Mon Sep 17 00:00:00 2001 From: penghui Date: Thu, 4 Jun 2020 23:49:20 +0800 Subject: [PATCH 12/13] Remove test from broker and client which is run in long time task. --- .github/workflows/ci-unit-broker-broker.yml | 2 +- .github/workflows/ci-unit-broker-client.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml index 81cda7e56c5f9..ccb860976163c 100644 --- a/.github/workflows/ci-unit-broker-broker.yml +++ b/.github/workflows/ci-unit-broker-broker.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java,!NamespaceOwnershipListenerTests,!AdminApiTest2,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java,!NamespaceOwnershipListenerTests,!AdminApiTest2,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest,!LoadBalancerTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-client.yml b/.github/workflows/ci-unit-broker-client.yml index 633a54eea48b7..74b6c65b711a0 100644 --- a/.github/workflows/ci-unit-broker-client.yml +++ b/.github/workflows/ci-unit-broker-client.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker client tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java,!UnAcknowledgedMessagesTimeoutTest,!SimpleSchemaTest,!ConsumerDedupPermitsUpdate' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/client/**/*.java,!UnAcknowledgedMessagesTimeoutTest,!SimpleSchemaTest,!ConsumerDedupPermitsUpdate,!ConsumerBatchReceiveTest,!KeySharedSubscriptionTest' -pl pulsar-broker - name: package surefire artifacts if: failure() From 8f3d7eb0a7306f136ee4bdfe4c2dda24893b46c4 Mon Sep 17 00:00:00 2001 From: penghui Date: Fri, 5 Jun 2020 00:32:33 +0800 Subject: [PATCH 13/13] Split transaction tests in broker module. --- .github/workflows/ci-unit-broker-broker.yml | 2 +- .../workflows/ci-unit-broker-transaction.yml | 89 +++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci-unit-broker-transaction.yml diff --git a/.github/workflows/ci-unit-broker-broker.yml b/.github/workflows/ci-unit-broker-broker.yml index ccb860976163c..5b66995d33393 100644 --- a/.github/workflows/ci-unit-broker-broker.yml +++ b/.github/workflows/ci-unit-broker-broker.yml @@ -70,7 +70,7 @@ jobs: - name: run unit tests pulsar broker tests if: steps.docs.outputs.changed_only == 'no' - run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java,!NamespaceOwnershipListenerTests,!AdminApiTest2,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest,!LoadBalancerTest' -pl pulsar-broker + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/**/*.java,!NamespaceOwnershipListenerTests,!AdminApiTest2,!MessagePublishBufferThrottleTest,!PrecisTopicPublishRateThrottleTest,!PublishRateLimiterTest,!MessagePublishThrottlingTest,!LoadBalancerTest,!org/apache/pulsar/broker/transaction/**/*.java,!AdminApiTest,!V1_AdminApiTest' -pl pulsar-broker - name: package surefire artifacts if: failure() diff --git a/.github/workflows/ci-unit-broker-transaction.yml b/.github/workflows/ci-unit-broker-transaction.yml new file mode 100644 index 0000000000000..4f63ecfd334c5 --- /dev/null +++ b/.github/workflows/ci-unit-broker-transaction.yml @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: CI - Unit - Brokers - Transaction +on: + pull_request: + branches: + - master + +jobs: + + unit-tests: + name: + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check if this pull request only changes documentation + id: docs + uses: apache/pulsar-test-infra/diff-only@master + with: + args: site2 .github deployment .asf.yaml .ci ct.yaml + + - name: Set up JDK 1.8 + uses: actions/setup-java@v1 + if: steps.docs.outputs.changed_only == 'no' + with: + java-version: 1.8 + + - name: Set up Maven + uses: apache/pulsar-test-infra/setup-maven@master + if: steps.docs.outputs.changed_only == 'no' + with: + maven-version: 3.6.1 + + - name: clean disk + if: steps.docs.outputs.changed_only == 'no' + run: | + sudo swapoff /swapfile + sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt clean + docker rmi $(docker images -q) -f + df -h + + - name: run unit tests install by skip tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn clean install -DskipTests -pl pulsar-broker -am + + - name: run unit tests pulsar broker transaction tests + if: steps.docs.outputs.changed_only == 'no' + run: mvn test -DfailIfNoTests=false '-Dtest=org/apache/pulsar/broker/transaction/**/*.java' -pl pulsar-broker + + - name: package surefire artifacts + if: failure() + run: | + df -h + free -h + rm -rf artifacts + mkdir artifacts + find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \; + zip -r artifacts.zip artifacts + - uses: actions/upload-artifact@master + name: upload surefire-artifacts + if: failure() + with: + name: surefire-artifacts + path: artifacts.zip