From ebc2b9748c050c8f8984a80e12e9d3970b7d4bc0 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 12:30:39 +0800 Subject: [PATCH 01/11] 1. Use ubuntu:20.04 base image for Pulsar docker images 2. Build python client by Python3.8 cherry-pick: 1. https://github.com/apache/pulsar/pull/11026 2. https://github.com/apache/pulsar/pull/11623 3. https://github.com/apache/pulsar/pull/11862 --- docker/pulsar-standalone/Dockerfile | 13 +++++++--- docker/pulsar/Dockerfile | 24 +++++++++++------- docker/pulsar/pom.xml | 4 +-- .../pulsar/scripts/install-pulsar-client.sh | 25 +++++++++++++++++++ 4 files changed, 51 insertions(+), 15 deletions(-) create mode 100755 docker/pulsar/scripts/install-pulsar-client.sh diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile index 0035de5e33d1b..600f747421860 100644 --- a/docker/pulsar-standalone/Dockerfile +++ b/docker/pulsar-standalone/Dockerfile @@ -24,16 +24,21 @@ FROM apachepulsar/pulsar-all:latest as pulsar FROM apachepulsar/pulsar-dashboard:latest as dashboard # Restart from -FROM openjdk:8-jdk +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get -y install wget gnupg && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN bash -c "echo deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main >> /etc/apt/sources.list.d/pgdg.list" # Note that the libpq-dev package is needed here in order to install # the required python psycopg2 package (for postgresql) later RUN apt-get update \ - && apt-get -y install python3.7 python3.7-dev python3-pip postgresql sudo nginx supervisor libpq-dev + && apt-get -y install openjdk-11-jdk-headless python3 python3-dev python3-pip postgresql-11 sudo nginx supervisor libpq-dev RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 RUN update-ca-certificates -RUN ln -s /usr/bin/pip3 /usr/bin/pip # Postgres configuration COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/11/main/postgresql.conf @@ -73,7 +78,7 @@ RUN pip install -r /pulsar/django/requirements.txt COPY --from=dashboard /pulsar/init-postgres.sh /pulsar/django/init-postgres.sh RUN mkdir /data RUN /pulsar/django/init-postgres.sh -RUN sudo -u postgres /etc/init.d/postgresql stop +RUN sudo -u postgres /etc/init.d/postgresql stop # Add postgresql to supervisord. Redirect logs to stdout RUN echo "\n[program:postgresql]\n\ command = /etc/init.d/postgresql start\n\ diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 37a022d965ff2..8464ea53d0232 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -32,13 +32,15 @@ COPY scripts/generate-zookeeper-config.sh /pulsar/bin COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin COPY scripts/watch-znode.py /pulsar/bin COPY scripts/set_python_version.sh /pulsar/bin -COPY scripts/install-pulsar-client-37.sh /pulsar/bin +COPY scripts/install-pulsar-client.sh /pulsar/bin -### Create 2nd stage from OpenJDK image -### and add Python dependencies (for Pulsar functions) +### Create 2nd stage from Ubuntu image +### and add OpenJDK and Python dependencies (for Pulsar functions) -FROM openjdk:8-jdk-slim +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive # Install software-properties-common RUN apt update \ @@ -47,21 +49,25 @@ RUN apt update \ # Install some utilities RUN apt-get update \ - && apt-get install -y netcat dnsutils less procps iputils-ping \ - python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \ + && apt-get -y dist-upgrade \ + && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \ + python3 python3-dev python3-setuptools python3-yaml python3-kazoo \ libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \ curl \ + && apt-get -y --purge autoremove \ + && apt-get autoclean \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python3.7 get-pip.py +RUN python3 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 ADD target/python-client/ /pulsar/pulsar-client ADD target/cpp-client/ /pulsar/cpp-client -RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security +ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 +RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security RUN apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ && apt-get clean \ @@ -75,4 +81,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar -RUN /pulsar/bin/install-pulsar-client-37.sh +RUN /pulsar/bin/install-pulsar-client.sh diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml index 1cc58615b6575..58f13904b4da7 100644 --- a/docker/pulsar/pom.xml +++ b/docker/pulsar/pom.xml @@ -63,8 +63,8 @@ ${project.basedir}/target ${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh - - 3.7 cp37-cp37m + + 3.8 cp38-cp38 diff --git a/docker/pulsar/scripts/install-pulsar-client.sh b/docker/pulsar/scripts/install-pulsar-client.sh new file mode 100755 index 0000000000000..56702b10db3d8 --- /dev/null +++ b/docker/pulsar/scripts/install-pulsar-client.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x + +PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/') +WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}") +pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all] From a79f59d95cfd2983fd8299d7ec4ddb0e0ca3c7c3 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 12:40:13 +0800 Subject: [PATCH 02/11] fix --- tests/docker-images/latest-version-image/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index eaa279a42cd11..124072eadb5af 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -75,3 +75,4 @@ RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go instal COPY target/java-test-functions.jar /pulsar/examples/ RUN cp -a /go/bin/. /pulsar/examples/go-examples/ + From 4a67684888d2bcbe9e0c7f2fcc3771e8fbb9fe42 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 12:40:42 +0800 Subject: [PATCH 03/11] fix --- .../scripts/install-pulsar-client-37.sh | 24 ------------------- 1 file changed, 24 deletions(-) delete mode 100755 docker/pulsar/scripts/install-pulsar-client-37.sh diff --git a/docker/pulsar/scripts/install-pulsar-client-37.sh b/docker/pulsar/scripts/install-pulsar-client-37.sh deleted file mode 100755 index 9d3714c0a979a..0000000000000 --- a/docker/pulsar/scripts/install-pulsar-client-37.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x - -WHEEL_FILE=$(ls /pulsar/pulsar-client | grep cp37) -pip3.7 install /pulsar/pulsar-client/${WHEEL_FILE} From 248f220b5f7e2bb91cf4cfca5b6fd5e00482a6c5 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 12:43:19 +0800 Subject: [PATCH 04/11] fix --- tests/docker-images/latest-version-image/Dockerfile | 1 - .../apache/pulsar/tests/integration/offload/TestBaseOffload.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index 124072eadb5af..eaa279a42cd11 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -75,4 +75,3 @@ RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go instal COPY target/java-test-functions.jar /pulsar/examples/ RUN cp -a /go/bin/. /pulsar/examples/go-examples/ - diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java index ca461566a7349..74190b17b2685 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java @@ -296,4 +296,5 @@ public void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String ad Thread.sleep(5000); Assert.assertTrue(ledgerExistsInBookKeeper(offloadedLedger)); } + } From 6493db377f95b5623e96c3e7bc14730f5118e433 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 12:49:43 +0800 Subject: [PATCH 05/11] change the ci trigger event, make sure all tests could be triggered when users pull request to branch-2.7 --- .github/workflows/ci-build-multi-os.yaml | 1 + .github/workflows/ci-cpp-build-windows.yaml | 1 + .github/workflows/ci-cpp.yaml | 1 + .github/workflows/ci-go-functions-style.yaml | 1 + .github/workflows/ci-go-functions-test.yaml | 1 + .github/workflows/ci-integration-backwards-compatibility.yaml | 1 + .github/workflows/ci-integration-cli.yaml | 1 + .github/workflows/ci-integration-function-state.yaml | 1 + .github/workflows/ci-integration-messaging.yaml | 1 + .github/workflows/ci-integration-process.yaml | 1 + .github/workflows/ci-integration-schema.yaml | 1 + .github/workflows/ci-integration-sql.yaml | 1 + .github/workflows/ci-integration-standalone.yaml | 1 + .github/workflows/ci-integration-thread.yaml | 1 + .github/workflows/ci-integration-tiered-filesystem.yaml | 1 + .github/workflows/ci-integration-tiered-jcloud.yaml | 1 + .github/workflows/ci-integration-transaction.yaml | 1 + .github/workflows/ci-shade-test.yaml | 1 + .github/workflows/ci-unit-broker-broker-gp1.yaml | 1 + 19 files changed, 19 insertions(+) diff --git a/.github/workflows/ci-build-multi-os.yaml b/.github/workflows/ci-build-multi-os.yaml index 9a2e11bf790cf..3433f1db2ba7c 100644 --- a/.github/workflows/ci-build-multi-os.yaml +++ b/.github/workflows/ci-build-multi-os.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml index 87de9936e7aa9..8d47873a0ab8c 100644 --- a/.github/workflows/ci-cpp-build-windows.yaml +++ b/.github/workflows/ci-cpp-build-windows.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index 99f5093f614a0..9b5655b9eac1d 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml index 05ed581f24040..1a7a06631a85d 100644 --- a/.github/workflows/ci-go-functions-style.yaml +++ b/.github/workflows/ci-go-functions-style.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - 'pulsar-function-go/**' push: diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml index 41151b179012f..7620f26b553d3 100644 --- a/.github/workflows/ci-go-functions-test.yaml +++ b/.github/workflows/ci-go-functions-test.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* paths: - 'pulsar-function-go/**' push: diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml index 085b8dc27fd2d..19f6809726720 100644 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ b/.github/workflows/ci-integration-backwards-compatibility.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml index 83de9867c6503..b2f40f956aedf 100644 --- a/.github/workflows/ci-integration-cli.yaml +++ b/.github/workflows/ci-integration-cli.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-function-state.yaml b/.github/workflows/ci-integration-function-state.yaml index 3a4d9bb46d3da..cf1715a657c0a 100644 --- a/.github/workflows/ci-integration-function-state.yaml +++ b/.github/workflows/ci-integration-function-state.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml index c31b77db21523..21c4a639a70ec 100644 --- a/.github/workflows/ci-integration-messaging.yaml +++ b/.github/workflows/ci-integration-messaging.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml index 2f9f234287cde..a80034b2f775c 100644 --- a/.github/workflows/ci-integration-process.yaml +++ b/.github/workflows/ci-integration-process.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml index 9b2d6d1cde82b..0a159a485360a 100644 --- a/.github/workflows/ci-integration-schema.yaml +++ b/.github/workflows/ci-integration-schema.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml index e96e293fa1ecf..9080db9d867fb 100644 --- a/.github/workflows/ci-integration-sql.yaml +++ b/.github/workflows/ci-integration-sql.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml index cf8929819682e..d18bccb3fac5e 100644 --- a/.github/workflows/ci-integration-standalone.yaml +++ b/.github/workflows/ci-integration-standalone.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml index 1adef813021ab..c4e7bec48f26a 100644 --- a/.github/workflows/ci-integration-thread.yaml +++ b/.github/workflows/ci-integration-thread.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml index a798f6c3164a6..0e7e337a00c13 100644 --- a/.github/workflows/ci-integration-tiered-filesystem.yaml +++ b/.github/workflows/ci-integration-tiered-filesystem.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml index 53561b5e5dcf2..21ca8dea59c7e 100644 --- a/.github/workflows/ci-integration-tiered-jcloud.yaml +++ b/.github/workflows/ci-integration-tiered-jcloud.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml index 683936b46b377..7daad33b96724 100644 --- a/.github/workflows/ci-integration-transaction.yaml +++ b/.github/workflows/ci-integration-transaction.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml index 98963ea274b41..134b7abaab646 100644 --- a/.github/workflows/ci-shade-test.yaml +++ b/.github/workflows/ci-shade-test.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* diff --git a/.github/workflows/ci-unit-broker-broker-gp1.yaml b/.github/workflows/ci-unit-broker-broker-gp1.yaml index 1d545dab886c7..eba8022bc021e 100644 --- a/.github/workflows/ci-unit-broker-broker-gp1.yaml +++ b/.github/workflows/ci-unit-broker-broker-gp1.yaml @@ -22,6 +22,7 @@ on: pull_request: branches: - master + - branch-* push: branches: - branch-* From c3be6e0dbaec35e4edae78fd058f7d706f78c2a0 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 14:27:59 +0800 Subject: [PATCH 06/11] remove add repo `ppa:deadsnakes/ppa` command --- docker/pulsar/Dockerfile | 5 ----- .../pulsar/tests/integration/offload/TestBaseOffload.java | 1 - 2 files changed, 6 deletions(-) diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 8464ea53d0232..c5c20bde0b519 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -42,11 +42,6 @@ FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive -# Install software-properties-common -RUN apt update \ - && apt install software-properties-common \ - && add-apt-repository ppa:deadsnakes/ppa - # Install some utilities RUN apt-get update \ && apt-get -y dist-upgrade \ diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java index 74190b17b2685..ca461566a7349 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/offload/TestBaseOffload.java @@ -296,5 +296,4 @@ public void testPublishOffloadAndConsumeDeletionLag(String serviceUrl, String ad Thread.sleep(5000); Assert.assertTrue(ledgerExistsInBookKeeper(offloadedLedger)); } - } From 822ff7907cf4a15d52a8ede43b38826d2593665f Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 17:14:14 +0800 Subject: [PATCH 07/11] sync the test latest DockerFile with branch master --- docker/pulsar/Dockerfile | 2 +- .../latest-version-image/Dockerfile | 84 +++++++++++++------ .../conf/presto/jvm.config | 30 +++++++ 3 files changed, 89 insertions(+), 27 deletions(-) create mode 100644 tests/docker-images/latest-version-image/conf/presto/jvm.config diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index c5c20bde0b519..6e7a636e9027b 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -76,4 +76,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar -RUN /pulsar/bin/install-pulsar-client.sh +RUN /pulsar/bin/install-pulsar-client.sh \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index eaa279a42cd11..f0022cb83298e 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -17,11 +17,47 @@ # under the License. # -FROM apachepulsar/pulsar-all:latest +# build go lang examples first in a separate layer + +FROM apachepulsar/pulsar:latest as pulsar-function-go + +USER root + +RUN rm -rf /var/lib/apt/lists/* && apt-get update +RUN apt-get install -y procps curl git + +ENV GOLANG_VERSION 1.13.3 + +RUN curl -sSL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz \ + | tar -C /usr/local -xz + +# RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && tar -xvf go1.13.3.linux-amd64.tar.gz && mv go /usr/local +# RUN export GOROOT=/usr/local/go && export GOPATH=$HOME/go && export PATH=$GOPATH/bin:$GOROOT/bin:$PATH +# RUN echo "export GOROOT=/usr/local/go" >> ~/.profile && echo "export GOPATH=$HOME/go" >> ~/.profile && echo "export PATH=$GOPATH/bin:$GOROOT/bin:$PATH" >> ~/.profile + +ENV PATH /usr/local/go/bin:$PATH + +RUN mkdir -p /go/src /go/bin && chmod -R 777 /go +ENV GOROOT /usr/local/go +ENV GOPATH /go +ENV PATH /go/bin:$PATH + +COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go +RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... +RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install +RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./... + +# Reference pulsar-all to copy connectors from there +FROM apachepulsar/pulsar-all:latest as pulsar-all + +######################################## +###### Main image build +######################################## +FROM apachepulsar/pulsar:latest RUN rm -rf /var/lib/apt/lists/* && apt update -RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl git +RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl RUN mkdir -p /var/log/pulsar && mkdir -p /var/run/supervisor/ && mkdir -p /pulsar/ssl @@ -41,26 +77,10 @@ COPY scripts/init-cluster.sh scripts/run-global-zk.sh scripts/run-local-zk.sh \ scripts/run-standalone.sh \ /pulsar/bin/ -# copy python test examples +COPY conf/presto/jvm.config /pulsar/conf/presto/ +# copy python test examples RUN mkdir -p /pulsar/instances/deps - -ENV GOLANG_VERSION 1.13.3 - -RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \ - | tar -v -C /usr/local -xz - -# RUN wget https://dl.google.com/go/go1.13.3.linux-amd64.tar.gz && tar -xvf go1.13.3.linux-amd64.tar.gz && mv go /usr/local -# RUN export GOROOT=/usr/local/go && export GOPATH=$HOME/go && export PATH=$GOPATH/bin:$GOROOT/bin:$PATH -# RUN echo "export GOROOT=/usr/local/go" >> ~/.profile && echo "export GOPATH=$HOME/go" >> ~/.profile && echo "export PATH=$GOPATH/bin:$GOROOT/bin:$PATH" >> ~/.profile - -ENV PATH /usr/local/go/bin:$PATH - -RUN mkdir -p /go/src /go/bin && chmod -R 777 /go -ENV GOROOT /usr/local/go -ENV GOPATH /go -ENV PATH /go/bin:$PATH - COPY python-examples/exclamation_lib.py /pulsar/instances/deps/ COPY python-examples/exclamation_with_extra_deps.py /pulsar/examples/python-examples/ COPY python-examples/exclamation.zip /pulsar/examples/python-examples/ @@ -68,10 +88,22 @@ COPY python-examples/producer_schema.py /pulsar/examples/python-examples/ COPY python-examples/consumer_schema.py /pulsar/examples/python-examples/ COPY python-examples/exception_function.py /pulsar/examples/python-examples/ -COPY target/pulsar-function-go/ /go/src/github.com/apache/pulsar/pulsar-function-go -RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go && go install ./... -RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/pf && go install -RUN cd /go/src/github.com/apache/pulsar/pulsar-function-go/examples && go install ./... - +# copy java test examples COPY target/java-test-functions.jar /pulsar/examples/ -RUN cp -a /go/bin/. /pulsar/examples/go-examples/ + +# copy go test examples +COPY --from=pulsar-function-go /go/bin /pulsar/examples/go-examples + +# Include all offloaders +COPY --from=pulsar-all /pulsar/offloaders /pulsar/offloaders + +# Include only the connectors needed by integration tests +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-cassandra-*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-debezium-*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-elastic-*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-hdfs*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-jdbc-postgres-*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-kafka-*.nar /pulsar/connectors/ +COPY --from=pulsar-all /pulsar/connectors/pulsar-io-rabbitmq-*.nar /pulsar/connectors/ + +CMD bash diff --git a/tests/docker-images/latest-version-image/conf/presto/jvm.config b/tests/docker-images/latest-version-image/conf/presto/jvm.config new file mode 100644 index 0000000000000..28db36a3937da --- /dev/null +++ b/tests/docker-images/latest-version-image/conf/presto/jvm.config @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +-server +-Xms128M +-Xmx1500M +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-Dpresto-temporarily-allow-java8=true +-Djdk.attach.allowAttachSelf=true From 83cf0f3f6201e04c436e428bc6b88eb66bd8da6e Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 18:30:46 +0800 Subject: [PATCH 08/11] 1. update docker build step in some CI workflow 2. update docker-build image file 3. upgrade go version for project pulsar-function-go --- .github/workflows/ci-cpp.yaml | 2 +- .../ci-integration-backwards-compatibility.yaml | 8 ++++++++ .github/workflows/ci-integration-cli.yaml | 8 ++++++++ .github/workflows/ci-integration-function-state.yaml | 8 ++++++++ .github/workflows/ci-integration-standalone.yaml | 8 ++++++++ build/docker/Dockerfile | 11 ++++++----- pulsar-client-cpp/docker-build.sh | 2 +- pulsar-client-cpp/docker-tests.sh | 2 +- pulsar-function-go/go.mod | 2 +- 9 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml index 9b5655b9eac1d..71b713bc3a97f 100644 --- a/.github/workflows/ci-cpp.yaml +++ b/.github/workflows/ci-cpp.yaml @@ -82,7 +82,7 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: | echo "Build C++ client library" - export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF" + export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so" pulsar-client-cpp/docker-build.sh - name: run c++ tests diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml index 19f6809726720..a244ceeb0fe58 100644 --- a/.github/workflows/ci-integration-backwards-compatibility.yaml +++ b/.github/workflows/ci-integration-backwards-compatibility.yaml @@ -77,6 +77,14 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn -q -B -ntp clean install -DskipTests + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + - name: build artifacts and docker image if: steps.docs.outputs.changed_only == 'no' run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml index b2f40f956aedf..264704fc5f292 100644 --- a/.github/workflows/ci-integration-cli.yaml +++ b/.github/workflows/ci-integration-cli.yaml @@ -77,6 +77,14 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn -q -B -ntp clean install -DskipTests + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + - name: build artifacts and docker image if: steps.docs.outputs.changed_only == 'no' run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/.github/workflows/ci-integration-function-state.yaml b/.github/workflows/ci-integration-function-state.yaml index cf1715a657c0a..991f954e5d121 100644 --- a/.github/workflows/ci-integration-function-state.yaml +++ b/.github/workflows/ci-integration-function-state.yaml @@ -77,6 +77,14 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn -q -B -ntp clean install -DskipTests + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + - name: build artifacts and docker image if: steps.docs.outputs.changed_only == 'no' run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml index d18bccb3fac5e..3547735ebfc7f 100644 --- a/.github/workflows/ci-integration-standalone.yaml +++ b/.github/workflows/ci-integration-standalone.yaml @@ -77,6 +77,14 @@ jobs: if: steps.docs.outputs.changed_only == 'no' run: mvn -q -B -ntp clean install -DskipTests + - name: build pulsar image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + + - name: build pulsar-all image + if: steps.docs.outputs.changed_only == 'no' + run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true + - name: build artifacts and docker image if: steps.docs.outputs.changed_only == 'no' run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 9ae75f530e9d8..5d9de1aa48146 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -25,10 +25,11 @@ ADD protobuf.patch /pulsar RUN apt-get update && \ apt-get install -y tig g++ cmake libssl-dev libcurl4-openssl-dev \ - liblog4cxx-dev libprotobuf-dev libboost-all-dev google-mock libgtest-dev \ - libjsoncpp-dev libxml2-utils protobuf-compiler wget \ + liblog4cxx-dev libprotobuf-dev google-mock libgtest-dev \ + libboost-dev libboost-program-options-dev libboost-system-dev libboost-python-dev \ + libxml2-utils protobuf-compiler wget \ curl doxygen openjdk-8-jdk-headless clang-format-5.0 \ - gnupg2 golang-1.10-go zip unzip libzstd-dev libsnappy-dev + gnupg2 golang-1.13-go zip unzip libzstd-dev libsnappy-dev python3-pip libpython-dev # Compile and install gtest RUN cd /usr/src/gtest && cmake . && make && cp libgtest.a /usr/lib @@ -62,8 +63,8 @@ RUN wget https://artifacts.crowdin.com/repo/deb/crowdin.deb -O crowdin.deb RUN dpkg -i crowdin.deb # Install PIP and PDoc -RUN wget https://bootstrap.pypa.io/get-pip.py && python get-pip.py -RUN pip install pdoc +RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python get-pip.py && rm get-pip.py +RUN pip3 install pdoc # Install Protobuf doc generator (requires Go) ENV GOPATH "$HOME/go" diff --git a/pulsar-client-cpp/docker-build.sh b/pulsar-client-cpp/docker-build.sh index 31fa5d9d1b31a..b615ae595985f 100755 --- a/pulsar-client-cpp/docker-build.sh +++ b/pulsar-client-cpp/docker-build.sh @@ -27,7 +27,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/pulsar-client-cpp BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" +BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04-py2}" IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" diff --git a/pulsar-client-cpp/docker-tests.sh b/pulsar-client-cpp/docker-tests.sh index e5fd3e80969f8..0969c1c4c53f9 100755 --- a/pulsar-client-cpp/docker-tests.sh +++ b/pulsar-client-cpp/docker-tests.sh @@ -34,7 +34,7 @@ ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/pulsar-client-cpp BUILD_IMAGE_NAME="${BUILD_IMAGE_NAME:-apachepulsar/pulsar-build}" -BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04}" +BUILD_IMAGE_VERSION="${BUILD_IMAGE_VERSION:-ubuntu-16.04-py2}" IMAGE="$BUILD_IMAGE_NAME:$BUILD_IMAGE_VERSION" diff --git a/pulsar-function-go/go.mod b/pulsar-function-go/go.mod index 560834533aad7..6170ae97699b6 100644 --- a/pulsar-function-go/go.mod +++ b/pulsar-function-go/go.mod @@ -3,7 +3,7 @@ module github.com/apache/pulsar/pulsar-function-go go 1.13 require ( - github.com/apache/pulsar-client-go v0.2.0 + github.com/apache/pulsar-client-go v0.6.0 github.com/golang/protobuf v1.4.2 github.com/prometheus/client_golang v1.7.1 github.com/prometheus/client_model v0.2.0 From abfb6d2920204b1cd957d45ff25404453475fdb5 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 23:30:22 +0800 Subject: [PATCH 09/11] cherry-pick 1. https://github.com/apache/pulsar/pull/9498 2. https://github.com/apache/pulsar/pull/9533 --- pulsar-client-cpp/CMakeLists.txt | 39 ++++++++++++++++--- pulsar-client-cpp/lib/NamedEntity.cc | 17 +++++++- pulsar-client-cpp/lib/NamedEntity.h | 9 +---- .../lib/PartitionedConsumerImpl.cc | 11 ++++-- pulsar-client-cpp/lib/Url.cc | 17 ++++++-- pulsar-client-cpp/lib/auth/AuthAthenz.cc | 10 ----- .../lib/auth/athenz/ZTSClient.cc | 21 +++++++++- pulsar-client-cpp/python/CMakeLists.txt | 8 ---- .../tests/ConsumerConfigurationTest.cc | 2 - pulsar-client-cpp/tests/ZTSClientTest.cc | 2 +- pulsar-function-go/pf/instance.go | 20 +++++----- 11 files changed, 102 insertions(+), 54 deletions(-) diff --git a/pulsar-client-cpp/CMakeLists.txt b/pulsar-client-cpp/CMakeLists.txt index 59b1c00a3a5ac..5a4ef7ea9afd3 100644 --- a/pulsar-client-cpp/CMakeLists.txt +++ b/pulsar-client-cpp/CMakeLists.txt @@ -17,7 +17,7 @@ # under the License. # -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.4) project (pulsar-cpp) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") @@ -43,6 +43,9 @@ MESSAGE(STATUS "BUILD_TESTS: " ${BUILD_TESTS}) option(BUILD_PYTHON_WRAPPER "Build Pulsar Python wrapper" ON) MESSAGE(STATUS "BUILD_PYTHON_WRAPPER: " ${BUILD_PYTHON_WRAPPER}) +option(BUILD_PERF_TOOLS "Build Pulsar CLI perf producer/consumer" OFF) +MESSAGE(STATUS "BUILD_PERF_TOOLS: " ${BUILD_PERF_TOOLS}) + option(LINK_STATIC "Link against static libraries" OFF) MESSAGE(STATUS "LINK_STATIC: " ${LINK_STATIC}) @@ -161,12 +164,34 @@ else() endif (USE_LOG4CXX) endif (LINK_STATIC) +find_package(Boost) + +if (NOT Boost_VERSION_MAJOR OR + (Boost_VERSION_MAJOR EQUAL 1 AND Boost_VERSION_MINOR LESS 69) + ) + # Boost System does not require linking since 1.69 + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} system) +endif() + if (MSVC) - find_package(Boost REQUIRED COMPONENTS program_options regex system date_time) + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} date_time) +endif() + +if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) + # GCC 4.8.2 implementation of std::regex is buggy + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} regex) + set(CMAKE_CXX_FLAGS " -DPULSAR_USE_BOOST_REGEX") + MESSAGE(STATUS "Using Boost::Regex") else() - find_package(Boost REQUIRED COMPONENTS program_options regex system) + MESSAGE(STATUS "Using std::regex") endif() +if(BUILD_PERF_TOOLS) + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} program_options) +endif() + +find_package(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) + if (BUILD_PYTHON_WRAPPER) find_package(PythonLibs REQUIRED) MESSAGE(STATUS "PYTHON: " ${PYTHONLIBS_VERSION_STRING}) @@ -283,7 +308,7 @@ if (NOT MSVC) ) else() set(COMMON_LIBS - ${COMMON_LIBS} + ${COMMON_LIBS} wldap32.lib Normaliz.lib) endif() @@ -328,7 +353,11 @@ set(CLIENT_LIBS ) add_subdirectory(lib) -add_subdirectory(perf) + +if(BUILD_PERF_TOOLS) + add_subdirectory(perf) +endif(BUILD_PERF_TOOLS) + add_subdirectory(examples) if (BUILD_TESTS) diff --git a/pulsar-client-cpp/lib/NamedEntity.cc b/pulsar-client-cpp/lib/NamedEntity.cc index cd68e5898eda9..f0836bab513b3 100644 --- a/pulsar-client-cpp/lib/NamedEntity.cc +++ b/pulsar-client-cpp/lib/NamedEntity.cc @@ -21,5 +21,20 @@ const boost::regex NamedEntity::pattern = boost::regex("^[-=:.\\w]*$"); bool NamedEntity::checkName(const std::string& name) { - return boost::regex_match(name, pattern) ? true : false; + for (char c : name) { + switch (c) { + case '=': + case ':': + case ' ': + case '!': + case '\t': + case '\r': + case '\n': + return false; + default: + break; + } + } + + return true; } diff --git a/pulsar-client-cpp/lib/NamedEntity.h b/pulsar-client-cpp/lib/NamedEntity.h index 44376026263b8..14b73d6069ca7 100644 --- a/pulsar-client-cpp/lib/NamedEntity.h +++ b/pulsar-client-cpp/lib/NamedEntity.h @@ -16,16 +16,11 @@ * specific language governing permissions and limitations * under the License. */ -#ifndef _PULSAR_NAMED_ENTITY_HEADER_ -#define _PULSAR_NAMED_ENTITY_HEADER_ +#pragma once -#include +#include class NamedEntity { - private: - static const boost::regex pattern; - public: static bool checkName(const std::string& name); }; -#endif diff --git a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc index bf123622b80ad..2686308ddf781 100644 --- a/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc +++ b/pulsar-client-cpp/lib/PartitionedConsumerImpl.cc @@ -229,7 +229,8 @@ ConsumerConfiguration PartitionedConsumerImpl::getSinglePartitionConsumerConfig( config.setBrokerConsumerStatsCacheTimeInMs(conf_.getBrokerConsumerStatsCacheTimeInMs()); const auto shared_this = const_cast(this)->shared_from_this(); - config.setMessageListener(std::bind(&PartitionedConsumerImpl::messageReceived, shared_this, _1, _2)); + config.setMessageListener(std::bind(&PartitionedConsumerImpl::messageReceived, shared_this, + std::placeholders::_1, std::placeholders::_2)); // Apply total limit of receiver queue size across partitions // NOTE: if it's called by handleGetPartitions(), the queue size of new internal consumers may be smaller @@ -250,8 +251,9 @@ ConsumerImplPtr PartitionedConsumerImpl::newInternalConsumer(unsigned int partit internalListenerExecutor_, true, Partitioned); const auto shared_this = const_cast(this)->shared_from_this(); - consumer->getConsumerCreatedFuture().addListener(std::bind( - &PartitionedConsumerImpl::handleSinglePartitionConsumerCreated, shared_this, _1, _2, partition)); + consumer->getConsumerCreatedFuture().addListener( + std::bind(&PartitionedConsumerImpl::handleSinglePartitionConsumerCreated, shared_this, + std::placeholders::_1, std::placeholders::_2, partition)); consumer->setPartitionIndex(partition); LOG_DEBUG("Creating Consumer for single Partition - " << topicPartitionName << "SubName - " @@ -573,7 +575,8 @@ void PartitionedConsumerImpl::runPartitionUpdateTask() { void PartitionedConsumerImpl::getPartitionMetadata() { using namespace std::placeholders; lookupServicePtr_->getPartitionMetadataAsync(topicName_) - .addListener(std::bind(&PartitionedConsumerImpl::handleGetPartitions, shared_from_this(), _1, _2)); + .addListener(std::bind(&PartitionedConsumerImpl::handleGetPartitions, shared_from_this(), + std::placeholders::_1, std::placeholders::_2)); } void PartitionedConsumerImpl::handleGetPartitions(Result result, diff --git a/pulsar-client-cpp/lib/Url.cc b/pulsar-client-cpp/lib/Url.cc index 0924652ee4424..f31e1fcc186de 100644 --- a/pulsar-client-cpp/lib/Url.cc +++ b/pulsar-client-cpp/lib/Url.cc @@ -18,9 +18,18 @@ */ #include "Url.h" -#include +#include + #include +#ifdef PULSAR_USE_BOOST_REGEX +#include +#define PULSAR_REGEX_NAMESPACE boost +#else +#include +#define PULSAR_REGEX_NAMESPACE std +#endif + namespace pulsar { static const std::map initDefaultPortsMap() { @@ -39,14 +48,14 @@ static const std::map& defaultPortsMap() { bool Url::parse(const std::string& urlStr, Url& url) { std::vector values; - static const boost::regex expression( + static const PULSAR_REGEX_NAMESPACE::regex expression( // proto host port "^(\?:([^:/\?#]+)://)\?(\\w+[^/\?#:]*)(\?::(\\d+))\?" // path file parameters "(/\?(\?:[^\?#/]*/)*)\?([^\?#]*)\?(\\\?(.*))\?"); - boost::cmatch groups; - if (!boost::regex_match(urlStr.c_str(), groups, expression)) { + PULSAR_REGEX_NAMESPACE::cmatch groups; + if (!PULSAR_REGEX_NAMESPACE::regex_match(urlStr.c_str(), groups, expression)) { // Invalid url return false; } diff --git a/pulsar-client-cpp/lib/auth/AuthAthenz.cc b/pulsar-client-cpp/lib/auth/AuthAthenz.cc index 3141fb30d16c1..82d12761c56c7 100644 --- a/pulsar-client-cpp/lib/auth/AuthAthenz.cc +++ b/pulsar-client-cpp/lib/auth/AuthAthenz.cc @@ -18,16 +18,6 @@ */ #include -#include -#include - -#include -#include -#include -#include - -#include - #include #include namespace ptree = boost::property_tree; diff --git a/pulsar-client-cpp/lib/auth/athenz/ZTSClient.cc b/pulsar-client-cpp/lib/auth/athenz/ZTSClient.cc index 045f9188e9db7..68a0b8070cec0 100644 --- a/pulsar-client-cpp/lib/auth/athenz/ZTSClient.cc +++ b/pulsar-client-cpp/lib/auth/athenz/ZTSClient.cc @@ -38,13 +38,18 @@ #include namespace ptree = boost::property_tree; -#include #include #include #include #include +#ifdef PULSAR_USE_BOOST_REGEX +#include +#else +#include +#endif + DECLARE_LOG_OBJECT() namespace pulsar { @@ -360,6 +365,8 @@ const std::string ZTSClient::getHeader() const { return roleHeader_; } PrivateKeyUri ZTSClient::parseUri(const char *uri) { PrivateKeyUri uriSt; // scheme mediatype[;base64] path file + +#ifdef PULSAR_USE_BOOST_REGEX static const boost::regex expression( "^(\?:([^:/\?#]+):)(\?:([;/\\-\\w]*),)\?(/\?(\?:[^\?#/]*/)*)\?([^\?#]*)"); boost::cmatch groups; @@ -367,8 +374,18 @@ PrivateKeyUri ZTSClient::parseUri(const char *uri) { uriSt.scheme = groups.str(1); uriSt.mediaTypeAndEncodingType = groups.str(2); uriSt.data = groups.str(4); - uriSt.path = groups.str(3) + groups.str(4); } +#else // !PULSAR_USE_BOOST_REGEX + static const std::regex expression( + R"(^(?:([A-Za-z]+):)(?:([/\w\-]+;\w+),([=\w]+))?(?:\/\/)?(\/[^?#]+)?)"); + std::cmatch groups; + if (std::regex_match(uri, groups, expression)) { + uriSt.scheme = groups.str(1); + uriSt.mediaTypeAndEncodingType = groups.str(2); + uriSt.data = groups.str(3); + uriSt.path = groups.str(4); + } +#endif // PULSAR_USE_BOOST_REGEX return uriSt; } } // namespace pulsar diff --git a/pulsar-client-cpp/python/CMakeLists.txt b/pulsar-client-cpp/python/CMakeLists.txt index 83bc63b35fb37..2a6d525c06a07 100644 --- a/pulsar-client-cpp/python/CMakeLists.txt +++ b/pulsar-client-cpp/python/CMakeLists.txt @@ -68,14 +68,6 @@ if (APPLE) ${Boost_PYTHON37-MT_LIBRARY_RELEASE} ${Boost_PYTHON38-MT_LIBRARY_RELEASE}) - if (LINK_STATIC) - # When linking statically on MacOS, include also libicu since it's now required by boost::regex - find_library(ICU_DATA REQUIRED NAMES libicudata.a PATHS /usr/local/opt/icu4c/lib) - find_library(ICU_I18N REQUIRED NAMES libicui18n.a PATHS /usr/local/opt/icu4c/lib) - find_library(ICU_UUC REQUIRED NAMES libicuuc.a PATHS /usr/local/opt/icu4c/lib) - - set(ICU_LIBS ${ICU_DATA} ${ICU_I18N} ${ICU_UUC}) - endif () endif() message(STATUS "Using Boost Python libs: ${PYTHON_WRAPPER_LIBS}") diff --git a/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc b/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc index 42e84534f1c87..379bcdca75c23 100644 --- a/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc +++ b/pulsar-client-cpp/tests/ConsumerConfigurationTest.cc @@ -18,8 +18,6 @@ */ #include #include -#include -#include #include DECLARE_LOG_OBJECT() diff --git a/pulsar-client-cpp/tests/ZTSClientTest.cc b/pulsar-client-cpp/tests/ZTSClientTest.cc index 9eccf17eccd79..01c2e03381668 100644 --- a/pulsar-client-cpp/tests/ZTSClientTest.cc +++ b/pulsar-client-cpp/tests/ZTSClientTest.cc @@ -39,7 +39,7 @@ TEST(ZTSClientTest, testZTSClient) { { PrivateKeyUri uri = ZTSClientWrapper::parseUri("file:///path/to/private.key"); ASSERT_EQ("file", uri.scheme); - ASSERT_EQ("///path/to/private.key", uri.path); + ASSERT_EQ("/path/to/private.key", uri.path); } { diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index afbf2992aa429..ae22056ffa82b 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -224,16 +224,16 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { gi.context.instanceConf.funcDetails.Namespace, gi.context.instanceConf.funcDetails.Name), gi.context.instanceConf.instanceID) - batchBuilderType := pulsar.DefaultBatchBuilder - - if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil { - batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder - if batchBuilder != "" { - if batchBuilder == "KEY_BASED" { - batchBuilderType = pulsar.KeyBasedBatchBuilder - } - } - } +// batchBuilderType := pulsar.DefaultBatchBuilder +// +// if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil { +// batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder +// if batchBuilder != "" { +// if batchBuilder == "KEY_BASED" { +// batchBuilderType = pulsar.KeyBasedBatchBuilder +// } +// } +// } producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{ Topic: topicName, From ef14b1d3c8275a5c14a806b43350849aa175d555 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Fri, 10 Sep 2021 23:31:17 +0800 Subject: [PATCH 10/11] revert 9124 --- pulsar-function-go/pf/instance.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pulsar-function-go/pf/instance.go b/pulsar-function-go/pf/instance.go index ae22056ffa82b..d3a45e08d3641 100644 --- a/pulsar-function-go/pf/instance.go +++ b/pulsar-function-go/pf/instance.go @@ -224,17 +224,6 @@ func (gi *goInstance) getProducer(topicName string) (pulsar.Producer, error) { gi.context.instanceConf.funcDetails.Namespace, gi.context.instanceConf.funcDetails.Name), gi.context.instanceConf.instanceID) -// batchBuilderType := pulsar.DefaultBatchBuilder -// -// if gi.context.instanceConf.funcDetails.Sink.ProducerSpec != nil { -// batchBuilder := gi.context.instanceConf.funcDetails.Sink.ProducerSpec.BatchBuilder -// if batchBuilder != "" { -// if batchBuilder == "KEY_BASED" { -// batchBuilderType = pulsar.KeyBasedBatchBuilder -// } -// } -// } - producer, err := gi.client.CreateProducer(pulsar.ProducerOptions{ Topic: topicName, Properties: properties, From cbb7652b30e5e125edc3b2e3df399c977d0d2712 Mon Sep 17 00:00:00 2001 From: gaoran10 Date: Sat, 11 Sep 2021 00:54:39 +0800 Subject: [PATCH 11/11] fix --- pulsar-client-cpp/lib/NamedEntity.cc | 2 -- pulsar-function-go/pf/mockMessage_test.go | 24 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pulsar-client-cpp/lib/NamedEntity.cc b/pulsar-client-cpp/lib/NamedEntity.cc index f0836bab513b3..ad7c385c6fe43 100644 --- a/pulsar-client-cpp/lib/NamedEntity.cc +++ b/pulsar-client-cpp/lib/NamedEntity.cc @@ -18,8 +18,6 @@ */ #include "NamedEntity.h" -const boost::regex NamedEntity::pattern = boost::regex("^[-=:.\\w]*$"); - bool NamedEntity::checkName(const std::string& name) { for (char c : name) { switch (c) { diff --git a/pulsar-function-go/pf/mockMessage_test.go b/pulsar-function-go/pf/mockMessage_test.go index 2e1cadeca62ae..af19556135413 100644 --- a/pulsar-function-go/pf/mockMessage_test.go +++ b/pulsar-function-go/pf/mockMessage_test.go @@ -64,6 +64,10 @@ func (m *MockMessage) Key() string { return "key" } +func (m *MockMessage) OrderingKey() string { + return "orderingKey" +} + func (m *MockMessage) RedeliveryCount() uint32 { return 1 } @@ -76,12 +80,32 @@ func (m *MockMessage) GetReplicatedFrom() string { return "mock-cluster" } +func (m *MockMessage) GetSchemaValue(v interface{}) error { + return nil +} + type MockMessageID struct{} func (m *MockMessageID) Serialize() []byte { return []byte(`message-id`) } +func (m *MockMessageID) LedgerID() int64 { + return 0 +} + +func (m *MockMessageID) EntryID() int64 { + return 0 +} + +func (m *MockMessageID) BatchIdx() int32 { + return 0 +} + +func (m *MockMessageID) PartitionIdx() int32 { + return 0 +} + type MockPulsarProducer struct{} func (producer *MockPulsarProducer) Topic() string {