Merge form pulsar master

.github/workflows/ci-cpp.yaml

@@ -33,7 +33,10 @@ jobs:
     steps:
 
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-go-functions-style.yaml

@@ -32,7 +32,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs
@@ -58,7 +61,7 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: |
           cd pulsar-function-go
-          go build ./pf
+          go build ./...
 
       - name: CheckStyle
         if: steps.docs.outputs.changed_only == 'no'

.github/workflows/ci-go-functions-test.yaml

@@ -35,7 +35,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-backwards-compatibility.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-cli.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-function-state.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-messaging.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-process.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-schema.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-sql.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-standalone.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-thread.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-tiered-filesystem.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-integration-tiered-jcloud.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-license.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1

.github/workflows/ci-pulsarbot.yaml

@@ -31,7 +31,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
       - name: Execute pulsarbot command
         id:   pulsarbot
         env:

.github/workflows/ci-unit-adaptors.yml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-unit-broker-sasl.yml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-unit-broker.yml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-unit-flaky.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-unit-proxy.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

.github/workflows/ci-unit.yaml

@@ -32,7 +32,10 @@ jobs:
 
     steps:
       - name: checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Check if this pull request only changes documentation
         id:   docs

bin/pulsar-daemon

@@ -151,6 +151,7 @@ case $startStop in
 
     rotate_out_log $out
     echo starting $command, logging to $logfile
+    echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
     pulsar=$PULSAR_HOME/bin/pulsar
     nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
     echo $! > $pid

conf/broker.conf

@@ -91,6 +91,9 @@ allowAutoTopicCreation=true
 # The type of topic that is allowed to be automatically created.(partitioned/non-partitioned)
 allowAutoTopicCreationType=non-partitioned
 
+# Enable subscription auto creation if new consumer connected (disable auto creation with value false)
+allowAutoSubscriptionCreation=true
+
 # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
 defaultNumPartitions=1
 
@@ -324,6 +327,18 @@ replicatedSubscriptionsSnapshotTimeoutSeconds=30
 # Max number of snapshot to be cached per subscription.
 replicatedSubscriptionsSnapshotMaxCachedPerSubscription=10
 
+# Max memory size for broker handling messages sending from producers.
+# If the processing message size exceed this value, broker will stop read data
+# from the connection. The processing messages means messages are sends to broker
+# but broker have not send response to client, usually waiting to write to bookies.
+# It's shared across all the topics running in the same broker.
+# Use -1 to disable the memory limitation. Default is 1/2 of direct memory.
+maxMessagePublishBufferSizeInMB=
+
+# Interval between checks to see if message publish buffer size is exceed the max message publish buffer size
+# Use 0 or negative number to disable the max publish buffer limiting.
+messagePublishBufferCheckIntervalInMillis=100
+
 ### --- Authentication --- ###
 # Role names that are treated as "proxy roles". If the broker sees a request with
 #role as proxyRoles - it will demand to see a valid original principal.
@@ -708,7 +723,7 @@ loadManagerClassName=org.apache.pulsar.broker.loadbalance.impl.ModularLoadManage
 # Supported algorithms name for namespace bundle split.
 # "range_equally_divide" divides the bundle into two parts with the same hash range size.
 # "topic_count_equally_divide" divides the bundle into two parts with the same topics count.
-supportedNamespaceBundleSplitAlgorithms=[range_equally_divide,topic_count_equally_divide]
+supportedNamespaceBundleSplitAlgorithms=range_equally_divide,topic_count_equally_divide
 
 # Default algorithm name for namespace bundle split
 defaultNamespaceBundleSplitAlgorithm=range_equally_divide
@@ -778,6 +793,11 @@ exposePublisherStats=true
 statsUpdateFrequencyInSecs=60
 statsUpdateInitialDelayInSecs=60
 
+# Enable expose the precise backlog stats.
+# Set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate.
+# Default is false.
+exposePreciseBacklogInPrometheus=false
+
 ### --- Schema storage --- ###
 # The schema storage implementation used by this broker
 schemaRegistryStorageClassName=org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorageFactory

conf/standalone.conf

@@ -531,6 +531,11 @@ exposeTopicLevelMetricsInPrometheus=true
 # Enable topic level metrics
 exposePublisherStats=true
 
+# Enable expose the precise backlog stats.
+# Set false to use published counter and consumed counter to calculate, this would be more efficient but may be inaccurate.
+# Default is false.
+exposePreciseBacklogInPrometheus=false
+
 ### --- Deprecated config variables --- ###
 
 # Deprecated. Use configurationStoreServers
@@ -627,6 +632,9 @@ allowAutoTopicCreation=true
 # The type of topic that is allowed to be automatically created.(partitioned/non-partitioned)
 allowAutoTopicCreationType=non-partitioned
 
+# Enable subscription auto creation if new consumer connected (disable auto creation with value false)
+allowAutoSubscriptionCreation=true
+
 # The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
 defaultNumPartitions=1
 

dashboard/Dockerfile

@@ -19,7 +19,7 @@
 
 FROM python:3.7-stretch
 
-MAINTAINER Pulsar
+LABEL maintainer="Apache Pulsar <dev@pulsar.apache.org>"
 
 RUN apt-get update
 RUN apt-get -y install postgresql python sudo nginx supervisor

deployment/kubernetes/helm/pulsar/values-mini.yaml

@@ -181,7 +181,8 @@ bookkeeper:
   ## templates/bookkeeper-configmap.yaml
   ##
   configData:
-    PULSAR_MEM: "\"-Xms128m -Xmx256m -XX:MaxDirectMemorySize=128m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintHeapAtGC -verbosegc -XX:G1LogLevel=finest\""
+    BOOKIE_MEM: "\"-Xms128m -Xmx256m -XX:MaxDirectMemorySize=128m -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.linkCapacity=1024 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB -XX:+ExitOnOutOfMemoryError -XX:+PerfDisableSharedMem\""
+    BOOKIE_GC: "\"-XX:+UseG1GC -XX:MaxGCPauseMillis=10\""
     PULSAR_PREFIX_dbStorage_writeCacheMaxSizeMb: "32"
     PULSAR_PREFIX_dbStorage_readAheadCacheMaxSizeMb: "32"
     PULSAR_PREFIX_journalMaxSizeMB: "2048"
@@ -316,7 +317,7 @@ autoRecovery:
   ## templates/autorecovery-configmap.yaml
   ##
   configData:
-    PULSAR_MEM: "\" -Xms64m -Xmx128m \""
+    BOOKIE_MEM: "\" -Xms64m -Xmx128m \""
 
 ## Pulsar Extra: Dashboard
 ## templates/dashboard-deployment.yaml

deployment/kubernetes/helm/pulsar/values.yaml

@@ -318,7 +318,7 @@ autoRecovery:
   ## templates/autorecovery-configmap.yaml
   ##
   configData:
-    PULSAR_MEM: "\" -Xms1g -Xmx1g \""
+    BOOKIE_MEM: "\" -Xms1g -Xmx1g \""
 
 ## Pulsar Extra: Dashboard
 ## templates/dashboard-deployment.yaml