[WIP][Branch-2.7][Build] Use ubuntu:20.04 base image for Pulsar docker images

.github/workflows/ci-build-multi-os.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-cpp-build-windows.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-cpp.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*
@@ -81,7 +82,7 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: |
           echo "Build C++ client library"
-          export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF"
+          export CMAKE_ARGS="-DCMAKE_BUILD_TYPE=Debug -DBUILD_DYNAMIC_LIB=OFF -DPYTHON_INCLUDE_DIR=/usr/include/python2.7 -DPYTHON_LIBRARY=/usr/lib/x86_64-linux-gnu/libpython2.7.so"
           pulsar-client-cpp/docker-build.sh
 
       - name: run c++ tests

.github/workflows/ci-go-functions-style.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
     paths:
       - 'pulsar-function-go/**'
   push:

.github/workflows/ci-go-functions-test.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
     paths:
       - 'pulsar-function-go/**'
   push:

.github/workflows/ci-integration-backwards-compatibility.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*
@@ -76,6 +77,14 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -q -B -ntp clean install -DskipTests
 
+      - name: build pulsar image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+      - name: build pulsar-all image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests

.github/workflows/ci-integration-cli.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*
@@ -76,6 +77,14 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -q -B -ntp clean install -DskipTests
 
+      - name: build pulsar image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+      - name: build pulsar-all image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests

.github/workflows/ci-integration-function-state.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*
@@ -76,6 +77,14 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -q -B -ntp clean install -DskipTests
 
+      - name: build pulsar image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+      - name: build pulsar-all image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests

.github/workflows/ci-integration-messaging.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-process.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-schema.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-sql.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-standalone.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*
@@ -76,6 +77,14 @@ jobs:
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -q -B -ntp clean install -DskipTests
 
+      - name: build pulsar image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
+      - name: build pulsar-all image
+        if: steps.docs.outputs.changed_only == 'no'
+        run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker -DskipTests -Ddocker.nocache=true
+
       - name: build artifacts and docker image
         if: steps.docs.outputs.changed_only == 'no'
         run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker -DskipTests

.github/workflows/ci-integration-thread.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-tiered-filesystem.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-tiered-jcloud.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-integration-transaction.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-shade-test.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

.github/workflows/ci-unit-broker-broker-gp1.yaml

@@ -22,6 +22,7 @@ on:
   pull_request:
     branches:
       - master
+      - branch-*
   push:
     branches:
       - branch-*

build/docker/Dockerfile

@@ -25,10 +25,11 @@ ADD protobuf.patch /pulsar
 
 RUN apt-get update && \
     apt-get install -y tig g++ cmake libssl-dev libcurl4-openssl-dev \
-                liblog4cxx-dev libprotobuf-dev libboost-all-dev google-mock libgtest-dev \
-                libjsoncpp-dev libxml2-utils protobuf-compiler wget \
+                liblog4cxx-dev libprotobuf-dev google-mock libgtest-dev \
+                libboost-dev libboost-program-options-dev libboost-system-dev libboost-python-dev \
+                libxml2-utils protobuf-compiler wget \
                 curl doxygen openjdk-8-jdk-headless clang-format-5.0 \
-                gnupg2 golang-1.10-go zip unzip libzstd-dev libsnappy-dev
+                gnupg2 golang-1.13-go zip unzip libzstd-dev libsnappy-dev python3-pip libpython-dev
 
 # Compile and install gtest
 RUN cd /usr/src/gtest && cmake . && make && cp libgtest.a /usr/lib
@@ -62,8 +63,8 @@ RUN wget https://artifacts.crowdin.com/repo/deb/crowdin.deb -O crowdin.deb
 RUN dpkg -i crowdin.deb
 
 # Install PIP and PDoc
-RUN wget https://bootstrap.pypa.io/get-pip.py && python get-pip.py
-RUN pip install pdoc
+RUN wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && python get-pip.py && rm get-pip.py
+RUN pip3 install pdoc
 
 # Install Protobuf doc generator (requires Go)
 ENV GOPATH "$HOME/go"

docker/pulsar-standalone/Dockerfile

@@ -24,16 +24,21 @@ FROM apachepulsar/pulsar-all:latest as pulsar
 FROM apachepulsar/pulsar-dashboard:latest as dashboard
 
 # Restart from
-FROM openjdk:8-jdk
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get -y install wget gnupg && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+RUN bash -c "echo deb http://apt.postgresql.org/pub/repos/apt/ focal-pgdg main >> /etc/apt/sources.list.d/pgdg.list"
 
 # Note that the libpq-dev package is needed here in order to install
 # the required python psycopg2 package (for postgresql) later
 RUN apt-get update \
-    && apt-get -y install python3.7 python3.7-dev python3-pip postgresql sudo nginx supervisor libpq-dev
+    && apt-get -y install openjdk-11-jdk-headless python3 python3-dev python3-pip postgresql-11 sudo nginx supervisor libpq-dev
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 RUN update-ca-certificates
-RUN ln -s /usr/bin/pip3 /usr/bin/pip
 
 # Postgres configuration
 COPY --from=dashboard /etc/postgresql/11/main/postgresql.conf /etc/postgresql/11/main/postgresql.conf
@@ -73,7 +78,7 @@ RUN pip install -r /pulsar/django/requirements.txt
 COPY --from=dashboard /pulsar/init-postgres.sh /pulsar/django/init-postgres.sh
 RUN mkdir /data
 RUN /pulsar/django/init-postgres.sh
-RUN sudo -u postgres /etc/init.d/postgresql stop 
+RUN sudo -u postgres /etc/init.d/postgresql stop
 # Add postgresql to supervisord. Redirect logs to stdout
 RUN echo "\n[program:postgresql]\n\
 command = /etc/init.d/postgresql start\n\

docker/pulsar/Dockerfile

@@ -32,36 +32,37 @@ COPY scripts/generate-zookeeper-config.sh /pulsar/bin
 COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin
 COPY scripts/watch-znode.py /pulsar/bin
 COPY scripts/set_python_version.sh /pulsar/bin
-COPY scripts/install-pulsar-client-37.sh /pulsar/bin
+COPY scripts/install-pulsar-client.sh /pulsar/bin
 
 
-### Create 2nd stage from OpenJDK image
-### and add Python dependencies (for Pulsar functions)
+### Create 2nd stage from Ubuntu image
+### and add OpenJDK and Python dependencies (for Pulsar functions)
 
-FROM openjdk:8-jdk-slim
+FROM ubuntu:20.04
 
-# Install software-properties-common
-RUN apt update \
-    && apt install software-properties-common \
-    && add-apt-repository ppa:deadsnakes/ppa
+ARG DEBIAN_FRONTEND=noninteractive
 
 # Install some utilities
 RUN apt-get update \
-     && apt-get install -y netcat dnsutils less procps iputils-ping \
-                 python3.7 python3.7-dev python3-setuptools python3-yaml python3-kazoo \
+     && apt-get -y dist-upgrade \
+     && apt-get -y install openjdk-11-jdk-headless netcat dnsutils less procps iputils-ping \
+                 python3 python3-dev python3-setuptools python3-yaml python3-kazoo \
                  libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
                  curl \
+     && apt-get -y --purge autoremove \
+     && apt-get autoclean \
      && apt-get clean \
      && rm -rf /var/lib/apt/lists/*
 
 RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python3.7 get-pip.py
+RUN python3 get-pip.py
 
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 
 ADD target/python-client/ /pulsar/pulsar-client
 ADD target/cpp-client/ /pulsar/cpp-client
-RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security
+ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
+RUN echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/java-11-openjdk-amd64/conf/security/java.security
 RUN apt-get update \
      && apt install -y /pulsar/cpp-client/*.deb \
      && apt-get clean \
@@ -75,4 +76,4 @@ ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE
 COPY --from=pulsar /pulsar /pulsar
 WORKDIR /pulsar
 
-RUN /pulsar/bin/install-pulsar-client-37.sh
+RUN /pulsar/bin/install-pulsar-client.sh

docker/pulsar/pom.xml

@@ -63,8 +63,8 @@
                   <workingDirectory>${project.basedir}/target</workingDirectory>
                   <executable>${project.basedir}/../../pulsar-client-cpp/docker/build-wheels.sh</executable>
                   <arguments>
-                    <!-- build python 3.7 -->
-                    <argument>3.7 cp37-cp37m</argument>
+                    <!-- build python 3.8 -->
+                    <argument>3.8 cp38-cp38</argument>
                   </arguments>
                 </configuration>
               </execution>

docker/pulsar/scripts/install-pulsar-client-37.sh → docker/pulsar/scripts/install-pulsar-client.sh

@@ -20,5 +20,6 @@
 
 set -x
 
-WHEEL_FILE=$(ls /pulsar/pulsar-client | grep cp37)
-pip3.7 install /pulsar/pulsar-client/${WHEEL_FILE}
+PYTHON_MAJOR_MINOR=$(python3 -V | sed -E 's/.* ([[:digit:]]+)\.([[:digit:]]+).*/\1\2/')
+WHEEL_FILE=$(ls /pulsar/pulsar-client | grep "cp${PYTHON_MAJOR_MINOR}")
+pip3 install /pulsar/pulsar-client/${WHEEL_FILE}[all]