Merge remote-tracking branch 'apache/master' into PIP-157

Author: Andras Beni

.github/PULL_REQUEST_TEMPLATE.md

@@ -76,3 +76,20 @@ This change added tests and can be verified as follows:
 
 - [ ] `doc-complete`
 (Docs have been already added)
+
+### Matching PR in forked repository
+
+PR in forked repository: <!-- ENTER URL HERE 
+
+After opening this PR, the build in apache/pulsar will fail and instructions will
+be provided for opening a PR in the PR author's forked repository.
+
+apache/pulsar pull requests should be first tested in your own fork since the 
+apache/pulsar CI based on GitHub Actions has constrained resources and quota.
+GitHub Actions provides separate quota for pull requests that are executed in 
+a forked repository.
+
+The tests will be run in the forked repository until all PR review comments have
+been handled, the tests pass and the PR is approved by a reviewer.
+
+-->

.github/workflows/ci-cpp-build.yaml

@@ -33,12 +33,11 @@ concurrency:
 
 jobs:
   changed_files_job:
-    name: 'Changed files check'
+    name: Preconditions
     runs-on: ubuntu-20.04
     outputs:
       docs_only: ${{ needs.changed_files_job.outputs.docs_only }}
       cpp_only: ${{ needs.changed_files_job.outputs.cpp_only }}
-      changed_tests: ${{ steps.changes.outputs.tests_files }}
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -55,6 +54,14 @@ jobs:
         run: |
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        env:
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
+
   cpp-build-centos7:
     needs: changed_files_job
     name: Build CPP Client on CentOS7

.github/workflows/ci-documentbot.yml

@@ -33,7 +33,7 @@ concurrency:
 
 jobs:
   label:
-    if: ${{ github.repository == 'apache/pulsar' }}
+    if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open')
     permissions:
       pull-requests: write 
     runs-on: ubuntu-20.04

.github/workflows/ci-go-functions.yaml

@@ -36,12 +36,11 @@ env:
 
 jobs:
   changed_files_job:
-    name: 'Changed files check'
+    name: Preconditions
     runs-on: ubuntu-20.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       cpp_only: ${{ steps.check_changes.outputs.cpp_only }}
-      changed_tests: ${{ steps.changes.outputs.tests_files }}
     steps:
       - name: checkout
         uses: actions/checkout@v2
@@ -57,6 +56,15 @@ jobs:
         id: check_changes
         run: |
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+          echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
+
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        env:
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
 
   check-style:
     needs: changed_files_job

.github/workflows/pulsar-ci-flaky.yaml

@@ -37,7 +37,7 @@ env:
 
 jobs:
   changed_files_job:
-    name: 'Changed files check'
+    name: Preconditions
     runs-on: ubuntu-20.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,14 @@ jobs:
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
           echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
 
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        env:
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
+
   build-and-test:
     needs: changed_files_job
     name: Flaky tests suite

.github/workflows/pulsar-ci.yaml

@@ -37,7 +37,7 @@ env:
 
 jobs:
   changed_files_job:
-    name: 'Changed files check'
+    name: Preconditions
     runs-on: ubuntu-20.04
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
@@ -60,6 +60,14 @@ jobs:
           echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
           echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}"
 
+      - name: Check if the PR has been approved for testing
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
+        env:
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          build/pulsar_ci_tool.sh check_ready_to_test
+
   build-and-license-check:
     needs: changed_files_job
     name: Build and License check
@@ -470,7 +478,7 @@ jobs:
           gh-actions-artifact-client.js delete pulsar-java-test-image.zst
 
   cpp-tests:
-    name:
+    name: CI - CPP, Python Tests
     runs-on: ubuntu-20.04
     timeout-minutes: 120
     needs: [

.gitignore

@@ -27,7 +27,8 @@ pulsar-functions/worker/src/test/resources/
 .factorypath
 
 # IntelliJ
-.idea/
+.idea/*
+!.idea/vcs.xml
 *.iml
 *.iws
 
@@ -48,6 +49,7 @@ target/
 
 # Python
 *.pyc
+.python-version
 
 # Perf tools
 *.hgrm

.idea/vcs.xml

@@ -11,3 +11,7 @@ It is the responsibility of the security vulnerability handling project team (Ap
 ## Security Policy details and supported versions of Apache Pulsar
 
 The security policy and supported versions are outlined on the Pulsar website under [Security > Security Policy and Supported Versions](https://pulsar.apache.org/docs/security-policy-and-supported-versions/).
+
+## Security Advisories
+
+Please visit the [Security Advisories](https://github.com/apache/pulsar/wiki/Security-advisories) page.

SECURITY.md

@@ -300,8 +300,11 @@ if [[ -z "$IS_JAVA_8" ]]; then
   # https://github.com/netty/netty/issues/12265
   OPTS="$OPTS --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED"
   # netty.DnsResolverUtil
-  # JvmDefaultGCMetricsLogger
-  OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED --add-opens java.management/sun.management=ALL-UNNAMED"
+  OPTS="$OPTS --add-opens java.base/sun.net=ALL-UNNAMED"
+  # JvmDefaultGCMetricsLogger & MBeanStatsGenerator
+  OPTS="$OPTS --add-opens java.management/sun.management=ALL-UNNAMED"
+  # MBeanStatsGenerator
+  OPTS="$OPTS --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED"
 fi
 
 OPTS="-cp $PULSAR_CLASSPATH $OPTS"

bin/pulsar

@@ -34,6 +34,7 @@ BINDIR=$(dirname "$PRG")
 export PULSAR_HOME=`cd -P $BINDIR/..;pwd`
 . "$PULSAR_HOME/bin/pulsar-admin-common.sh"
 OPTS="-Dorg.jline.terminal.jansi=false $OPTS"
+OPTS="-Dpulsar.shell.working.dir=$(pwd) $OPTS"
 DEFAULT_CONFIG="-Dpulsar.shell.config.default=$PULSAR_CLIENT_CONF"
 
 #Change to PULSAR_HOME to support relative paths

bin/pulsar-shell

@@ -28,6 +28,7 @@ if ERRORLEVEL 1 (
 )
 
 set "OPTS=%OPTS% -Dorg.jline.terminal.jansi=false"
+set "OPTS=%OPTS% -Dpulsar.shell.config.default=%cd%"
 set "DEFAULT_CONFIG=-Dpulsar.shell.config.default="%PULSAR_CLIENT_CONF%""
 cd "%PULSAR_HOME%"
 "%JAVACMD%" %OPTS%  %DEFAULT_CONFIG%  org.apache.pulsar.shell.PulsarShell %*

bin/pulsar-shell.cmd

@@ -154,6 +154,81 @@ function ci_move_test_reports() {
   )
 }
 
+function ci_check_ready_to_test() {
+  if [[ -z "$GITHUB_EVENT_PATH" ]]; then
+    >&2 echo "GITHUB_EVENT_PATH isn't set"
+    return 1
+  fi
+
+  PR_JSON_URL=$(jq -r '.pull_request.url' "${GITHUB_EVENT_PATH}")
+  echo "Refreshing $PR_JSON_URL..."
+  PR_JSON=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" "${PR_JSON_URL}")
+
+  if printf "%s" "${PR_JSON}" | jq -e '.draft | select(. == true)' &> /dev/null; then
+    echo "PR is draft."
+  elif ! ( printf "%s" "${PR_JSON}" | jq -e '.mergeable | select(. == true)' &> /dev/null ); then
+    echo "PR isn't mergeable."
+  else
+    # check ready-to-test label
+    if printf "%s" "${PR_JSON}" | jq -e '.labels[] | .name | select(. == "ready-to-test")' &> /dev/null; then
+      echo "Found ready-to-test label."
+      return 0
+    else
+      echo "There is no ready-to-test label on the PR."
+    fi
+
+    # check if the PR has been approved
+    PR_NUM=$(jq -r '.pull_request.number' "${GITHUB_EVENT_PATH}")
+    REPO_FULL_NAME=$(jq -r '.repository.full_name' "${GITHUB_EVENT_PATH}")
+    REPO_NAME=$(basename "${REPO_FULL_NAME}")
+    REPO_OWNER=$(dirname "${REPO_FULL_NAME}")
+    # use graphql query to find out reviewDecision
+    PR_REVIEW_DECISION=$(curl -s -H "Authorization: Bearer $GITHUB_TOKEN" -X POST -d '{"query": "query { repository(name: \"'${REPO_NAME}'\", owner: \"'${REPO_OWNER}'\") { pullRequest(number: '${PR_NUM}') { reviewDecision } } }"}' https://api.github.com/graphql |jq -r '.data.repository.pullRequest.reviewDecision')
+    echo "Review decision for PR #${PR_NUM} in repository ${REPO_OWNER}/${REPO_NAME} is ${PR_REVIEW_DECISION}"
+    if [[ "$PR_REVIEW_DECISION" == "APPROVED" ]]; then
+      return 0
+    fi
+  fi
+
+  FORK_REPO_URL=$(jq -r '.pull_request.head.repo.html_url' "$GITHUB_EVENT_PATH")
+  PR_BRANCH_LABEL=$(jq -r '.pull_request.head.label' "$GITHUB_EVENT_PATH")
+  PR_URL=$(jq -r '.pull_request.html_url' "$GITHUB_EVENT_PATH")
+  FORK_PR_TITLE_URL_ENCODED=$(printf "%s" "${PR_JSON}" | jq -r '"[run-tests] " + .title | @uri')
+  FORK_PR_BODY_URL_ENCODED=$(jq -n -r "\"This PR is for running tests for upstream PR ${PR_URL}.\n\n<!-- Before creating this PR, please ensure that the fork $FORK_REPO_URL is up to date with https://github.com/apache/pulsar -->\" | @uri")
+  >&2 tee -a "$GITHUB_STEP_SUMMARY" <<EOF
+
+# Instructions for proceeding with the pull request:
+
+apache/pulsar pull requests should be first tested in your own fork since the apache/pulsar CI based on
+GitHub Actions has constrained resources and quota. GitHub Actions provides separate quota for
+pull requests that are executed in a forked repository.
+
+1. Go to ${FORK_REPO_URL} and ensure that your branch is up to date with https://github.com/apache/pulsar
+   Sync your fork if it's behind.
+2. Open a pull request to your own fork. You can use this link to create the pull request in
+   your own fork:
+   [Create PR in fork for running tests](${FORK_REPO_URL}/compare/master...${PR_BRANCH_LABEL}?expand=1&title=${FORK_PR_TITLE_URL_ENCODED}&body=${FORK_PR_BODY_URL_ENCODED})
+3. Edit the description of the pull request ${PR_URL} and add the link to the PR that you opened to your own fork
+   so that the reviewer can verify that tests pass in your own fork.
+4. Ensure that tests pass in your own fork. Your own fork will be used to run the tests during the PR review
+   and any changes made during the review. You as a PR author are responsible for following up on test failures.
+   Please report any flaky tests as new issues at https://github.com/apache/pulsar/issues
+   after checking that the flaky test isn't already reported.
+5. When the PR is approved, it will be possible to restart the Pulsar CI workflow within apache/pulsar
+   repository by adding a comment "/pulsarbot rerun-failure-checks" to the PR.
+   An alternative for the PR approval is to add a ready-to-test label to the PR. This can be done
+   by Apache Pulsar committers.
+6. When tests pass on the apache/pulsar side, the PR can be merged by a Apache Pulsar Committer.
+
+If you have any trouble you can get support in multiple ways:
+* by sending email to the [dev mailing list](mailto:dev@pulsar.apache.org) ([subscribe](mailto:dev-subscribe@pulsar.apache.org))
+* on the [#contributors channel on Pulsar Slack](https://apache-pulsar.slack.com/channels/contributors) ([join](https://pulsar.apache.org/community#section-discussions))
+* in apache/pulsar [GitHub discussions Q&A](https://github.com/apache/pulsar/discussions/categories/q-a)
+
+EOF
+  return 1
+}
+
 if [ -z "$1" ]; then
   echo "usage: $0 [ci_tool_function_name]"
   echo "Available ci tool functions:"

build/pulsar_ci_tool.sh

@@ -50,7 +50,7 @@
     <guice.version>4.2.3</guice.version>
     <guava.version>31.0.1-jre</guava.version>
     <ant.version>1.10.12</ant.version>
-    <snakeyaml.version>1.31</snakeyaml.version>
+    <snakeyaml.version>1.32</snakeyaml.version>
     <mockito.version>3.12.4</mockito.version>
     <!-- required for running tests on JDK11+ -->
     <test.additional.args>

buildtools/pom.xml

@@ -1529,12 +1529,22 @@ transactionCoordinatorEnabled=false
 transactionMetadataStoreProviderClassName=org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider
 
 # Transaction buffer takes a snapshot after the number of transaction operations reaches this value.
+# If transaction buffer enables snapshot segment, transaction buffer updates snapshot metadata
+# after the number of transaction operations reaches this value.
 transactionBufferSnapshotMaxTransactionCount=1000
 
-# Transaction buffer take snapshot interval time
+# The interval time for transaction buffer to take snapshots.
+# If transaction buffer enables snapshot segment, it is the interval time for transaction buffer to update snapshot metadata.
 # Unit : millisecond
 transactionBufferSnapshotMinTimeInMillis=5000
 
+# Whether to enable segmented transaction buffer snapshot to handle a large number of aborted transactions.
+transactionBufferSegmentedSnapshotEnabled=false
+
+# Transaction buffer stores the transaction ID of aborted transactions and takes snapshots.
+# This configuration determines the size of the snapshot segment. The default value is 256 KB (262144 bytes).
+transactionBufferSnapshotSegmentSize=262144
+
 # The max concurrent requests for transaction buffer client, default is 1000
 transactionBufferClientMaxConcurrentRequests=1000
 

conf/broker.conf

@@ -1119,12 +1119,22 @@ transactionCoordinatorEnabled=false
 transactionMetadataStoreProviderClassName=org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStoreProvider
 
 # Transaction buffer takes a snapshot after the number of transaction operations reaches this value.
+# If transaction buffer enables snapshot segment, transaction buffer updates snapshot metadata
+# after the number of transaction operations reaches this value.
 transactionBufferSnapshotMaxTransactionCount=1000
 
-# Transaction buffer take snapshot interval time
+# The interval time for transaction buffer to take snapshots.
+# If transaction buffer enables snapshot segment, it is the interval time for transaction buffer to update snapshot metadata.
 # Unit : millisecond
 transactionBufferSnapshotMinTimeInMillis=5000
 
+# Whether to enable segmented transaction buffer snapshot to handle a large number of aborted transactions.
+transactionBufferSegmentedSnapshotEnabled=false
+
+# Transaction buffer stores the transaction ID of aborted transactions and takes snapshots.
+# This configuration determines the size of the snapshot segment. The default value is 256 KB (262144 bytes).
+transactionBufferSnapshotSegmentSize=262144
+
 # The transaction buffer client's operation timeout in milliseconds.
 transactionBufferClientOperationTimeoutInMills=3000
 

conf/standalone.conf

@@ -454,7 +454,7 @@ The Apache Software License, Version 2.0
     - org.eclipse.jetty.websocket-websocket-servlet-9.4.48.v20220622.jar
     - org.eclipse.jetty-jetty-alpn-conscrypt-server-9.4.48.v20220622.jar
     - org.eclipse.jetty-jetty-alpn-server-9.4.48.v20220622.jar
- * SnakeYaml -- org.yaml-snakeyaml-1.31.jar
+ * SnakeYaml -- org.yaml-snakeyaml-1.32.jar
  * RocksDB - org.rocksdb-rocksdbjni-6.29.4.1.jar
  * Google Error Prone Annotations - com.google.errorprone-error_prone_annotations-2.5.1.jar
  * Apache Thrift - org.apache.thrift-libthrift-0.14.2.jar
@@ -547,6 +547,8 @@ The Apache Software License, Version 2.0
     - io.reactivex.rxjava3-rxjava-3.0.1.jar
   * RabbitMQ Java Client
     - com.rabbitmq-amqp-client-5.5.3.jar
+  * RoaringBitmap
+    - org.roaringbitmap-RoaringBitmap-0.9.15.jar
 
 BSD 3-clause "New" or "Revised" License
  * Google auth library