Skip to content

Commit 8b9e28c

Browse files
author
Yan Zhang
committed
Merge branch 'up/master' into website/branch-2.7.2-chapter-2
* up/master: (26 commits) [pulsar-admin] Allow setting --forward-source-message-property to false when updating a pulsar function (apache#12128) [website][upgrade]feat: docs migration - Development (apache#12320) Update delete inactive topic configuration documentation (apache#12350) [PIP 95][Issue 12040][broker] Multiple bind addresses for Pulsar protocol (apache#12056) Added Debezium Source for MS SQL Server (apache#12256) Fix: flaky oracle tests (apache#12306) [C++] Use URL encoded content type for OAuth 2.0 authentication (apache#12341) [C++] Handle OAuth 2.0 exceptional cases gracefully (apache#12335) feat(cli): add restart command to pulsar-daemon (apache#12279) [client-tools] Remove redundant initial value (apache#12296) Make AuthenticationTokenTest to run on windows (apache#12329) [offload] fix FileSystemManagedLedgerOffloader can not cleanup outdated ledger data (apache#12309) [Doc]--Update contents for Pulsar adaptor for Apache Spark (apache#12338) [PIP 95][Issue 12040][broker] Improved multi-listener in standalone mode (apache#12066) [website][upgrade]feat: docs migration - Cookbooks (apache#12319) [testclient] Make --payload-file take effect in PerformanceClient (apache#12187) [website][upgrade]feat: docs migration - adaptor (apache#12318) [pulsar-client] Add partition-change api for producer/consumer interceptors (apache#12287) [Transaction]Fix lowWaterMark of TopicTransactionBuffer (apache#12312) [pulsar-admin] New option takes precedence over deprecated option (apache#12260) ... # Conflicts: # site2/website-next/docusaurus.config.js # site2/website-next/versions.json
2 parents 7fdb854 + e316a53 commit 8b9e28c

File tree

158 files changed

+12926
-400
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

158 files changed

+12926
-400
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: CI - Integration - Pulsar-IO Oracle Source
21+
on:
22+
pull_request:
23+
branches:
24+
- master
25+
push:
26+
branches:
27+
- branch-*
28+
29+
env:
30+
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
31+
32+
jobs:
33+
34+
pulsar-io:
35+
name:
36+
runs-on: ubuntu-latest
37+
timeout-minutes: 120
38+
39+
steps:
40+
- name: checkout
41+
uses: actions/checkout@v2
42+
43+
- name: Tune Runner VM
44+
uses: ./.github/actions/tune-runner-vm
45+
46+
- name: Detect changed files
47+
id: changes
48+
uses: apache/pulsar-test-infra/paths-filter@master
49+
with:
50+
filters: .github/changes-filter.yaml
51+
52+
- name: Check changed files
53+
id: check_changes
54+
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
55+
56+
- name: Cache local Maven repository
57+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
58+
uses: actions/cache@v2
59+
with:
60+
path: |
61+
~/.m2/repository/*/*/*
62+
!~/.m2/repository/org/apache/pulsar
63+
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
64+
restore-keys: |
65+
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
66+
${{ runner.os }}-m2-dependencies-core-modules-
67+
68+
- name: Set up JDK 11
69+
uses: actions/setup-java@v2
70+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
71+
with:
72+
distribution: 'adopt'
73+
java-version: 11
74+
75+
- name: clean disk
76+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
77+
run: |
78+
sudo swapoff -a
79+
sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
80+
sudo apt clean
81+
docker rmi $(docker images -q) -f
82+
df -h
83+
84+
- name: run install by skip tests
85+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
86+
run: mvn -q -B -ntp clean install -DskipTests
87+
88+
- name: build pulsar image
89+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
90+
run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
91+
92+
- name: build pulsar-all image
93+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
94+
run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
95+
96+
- name: build artifacts and docker image
97+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
98+
run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
99+
100+
- name: run integration tests
101+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
102+
run: ./build/run_integration_group.sh PULSAR_IO_ORA
103+
104+
- name: Upload container logs
105+
uses: actions/upload-artifact@v2
106+
if: ${{ cancelled() || failure() }}
107+
continue-on-error: true
108+
with:
109+
name: container-logs
110+
path: tests/integration/target/container-logs
111+
112+
- name: Upload surefire-reports
113+
uses: actions/upload-artifact@v2
114+
if: ${{ cancelled() || failure() }}
115+
continue-on-error: true
116+
with:
117+
name: surefire-reports
118+
path: tests/integration/target/surefire-reports

bin/pulsar-daemon

+61-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020

2121
usage() {
2222
cat <<EOF
23-
Usage: pulsar-daemon (start|stop) <command> <args...>
23+
Usage: pulsar-daemon (start|stop|restart) <command> <args...>
2424
where command is one of:
2525
broker Run a broker server
2626
bookie Run a bookie server
@@ -205,6 +205,66 @@ case $startStop in
205205
fi
206206
;;
207207

208+
(restart)
209+
if [ -f $pid ]; then
210+
TARGET_PID=$(cat $pid)
211+
if kill -0 $TARGET_PID > /dev/null 2>&1; then
212+
echo "stopping $command"
213+
kill $TARGET_PID
214+
215+
count=0
216+
location=$PULSAR_LOG_DIR
217+
while ps -p $TARGET_PID > /dev/null;
218+
do
219+
echo "Shutdown is in progress... Please wait..."
220+
sleep 1
221+
count=`expr $count + 1`
222+
223+
if [ "$count" = "$PULSAR_STOP_TIMEOUT" ]; then
224+
break
225+
fi
226+
done
227+
228+
if [ "$count" != "$PULSAR_STOP_TIMEOUT" ]; then
229+
echo "Shutdown completed."
230+
fi
231+
232+
if kill -0 $TARGET_PID > /dev/null 2>&1; then
233+
fileName=$location/$command.out
234+
$JAVA_HOME/bin/jstack $TARGET_PID > $fileName
235+
echo "Thread dumps are taken for analysis at $fileName"
236+
if [ "$1" == "-force" ]
237+
then
238+
echo "forcefully stopping $command"
239+
kill -9 $TARGET_PID >/dev/null 2>&1
240+
echo Successfully stopped the process
241+
else
242+
echo "WARNNING : $command is not stopped completely."
243+
exit 1
244+
fi
245+
fi
246+
else
247+
echo "no $command to stop"
248+
fi
249+
rm $pid
250+
else
251+
echo no "$command to stop"
252+
fi
253+
sleep 3
254+
255+
rotate_out_log $out
256+
echo restarting $command, logging to $logfile
257+
echo Note: Set immediateFlush to true in conf/log4j2.yaml will guarantee the logging event is flushing to disk immediately. The default behavior is switched off due to performance considerations.
258+
pulsar=$PULSAR_HOME/bin/pulsar
259+
nohup $pulsar $command "$@" > "$out" 2>&1 < /dev/null &
260+
echo $! > $pid
261+
sleep 1; head $out
262+
sleep 2;
263+
if ! ps -p $! > /dev/null ; then
264+
exit 1
265+
fi
266+
;;
267+
208268
(*)
209269
usage
210270
exit 1

conf/bookkeeper.conf

+3
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,9 @@ autoRecoveryDaemonEnabled=true
252252
# How long to wait, in seconds, before starting auto recovery of a lost bookie
253253
lostBookieRecoveryDelay=0
254254

255+
# Use older Bookkeeper wire protocol (Before Version 3) for AutoRecovery. Default is false
256+
useV2WireProtocol=true
257+
255258
#############################################################################
256259
## Placement settings
257260
#############################################################################

conf/broker.conf

+5-1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,9 @@ webServicePortTls=
4040
# Hostname or IP address the service binds on, default is 0.0.0.0.
4141
bindAddress=0.0.0.0
4242

43+
# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
44+
bindAddresses=
45+
4346
# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
4447
advertisedAddress=
4548

@@ -155,7 +158,8 @@ allowAutoSubscriptionCreation=true
155158
# The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
156159
defaultNumPartitions=1
157160

158-
# Enable the deletion of inactive topics
161+
# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
162+
# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
159163
brokerDeleteInactiveTopicsEnabled=true
160164

161165
# How often to check for inactive topics

conf/standalone.conf

+5-1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ webServicePort=8080
3333
# Hostname or IP address the service binds on, default is 0.0.0.0.
3434
bindAddress=0.0.0.0
3535

36+
# Extra bind addresses for the service: <listener_name>:<scheme>://<host>:<port>,[...]
37+
bindAddresses=
38+
3639
# Hostname or IP address the service advertises to the outside world. If not set, the value of InetAddress.getLocalHost().getHostName() is used.
3740
advertisedAddress=
3841

@@ -99,7 +102,8 @@ backlogQuotaDefaultLimitSecond=-1
99102
# Default ttl for namespaces if ttl is not already configured at namespace policies. (disable default-ttl with value 0)
100103
ttlDurationDefaultInSeconds=0
101104

102-
# Enable the deletion of inactive topics
105+
# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
106+
# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
103107
brokerDeleteInactiveTopicsEnabled=true
104108

105109
# How often to check for inactive topics

deployment/terraform-ansible/templates/broker.conf

+2-1
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,8 @@ allowAutoSubscriptionCreation=true
119119
# The number of partitioned topics that is allowed to be automatically created if allowAutoTopicCreationType is partitioned.
120120
defaultNumPartitions=1
121121

122-
# Enable the deletion of inactive topics
122+
# Enable the deletion of inactive topics. This parameter need to cooperate with the allowAutoTopicCreation parameter.
123+
# If brokerDeleteInactiveTopicsEnabled is set to true, we should ensure that allowAutoTopicCreation is also set to true.
123124
brokerDeleteInactiveTopicsEnabled=true
124125

125126
# How often to check for inactive topics

distribution/io/src/assemble/io.xml

+1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
<file><source>${basedir}/../../pulsar-io/debezium/mysql/target/pulsar-io-debezium-mysql-${project.version}.nar</source></file>
7272
<file><source>${basedir}/../../pulsar-io/debezium/postgres/target/pulsar-io-debezium-postgres-${project.version}.nar</source></file>
7373
<file><source>${basedir}/../../pulsar-io/debezium/oracle/target/pulsar-io-debezium-oracle-${project.version}.nar</source></file>
74+
<file><source>${basedir}/../../pulsar-io/debezium/mssql/target/pulsar-io-debezium-mssql-${project.version}.nar</source></file>
7475
<file><source>${basedir}/../../pulsar-io/debezium/mongodb/target/pulsar-io-debezium-mongodb-${project.version}.nar</source></file>
7576
<file><source>${basedir}/../../pulsar-io/influxdb/target/pulsar-io-influxdb-${project.version}.nar</source></file>
7677
<file><source>${basedir}/../../pulsar-io/redis/target/pulsar-io-redis-${project.version}.nar</source></file>

managed-ledger/src/main/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerImpl.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -3083,9 +3083,13 @@ private void cleanupOffloaded(long ledgerId, UUID uuid, String offloadDriverName
30833083
Map<String, String> offloadDriverMetadata, String cleanupReason) {
30843084
log.info("[{}] Cleanup offload for ledgerId {} uuid {} because of the reason {}.",
30853085
name, ledgerId, uuid.toString(), cleanupReason);
3086+
Map<String, String> metadataMap = Maps.newHashMap();
3087+
metadataMap.putAll(offloadDriverMetadata);
3088+
metadataMap.put("ManagedLedgerName", name);
3089+
30863090
Retries.run(Backoff.exponentialJittered(TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toHours(1)).limit(10),
30873091
Retries.NonFatalPredicate,
3088-
() -> config.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, offloadDriverMetadata),
3092+
() -> config.getLedgerOffloader().deleteOffloaded(ledgerId, uuid, metadataMap),
30893093
scheduledExecutor, name).whenComplete((ignored, exception) -> {
30903094
if (exception != null) {
30913095
log.warn("[{}] Error cleaning up offload for {}, (cleanup reason: {})",

0 commit comments

Comments
 (0)