[improve] Allow to build and push multi-arch Docker images (apache#19432

) Co-authored-by: Lari Hotari <lhotari@users.noreply.github.com> Co-authored-by: Yong Zhang <zhangyong1025.zy@gmail.com> Co-authored-by: Zixuan Liu <nodeces@gmail.com> Co-authored-by: tison <wander4096@gmail.com>
ascentstream · Apr 22, 2023 · 4190e40 · 4190e40
1 parent c58da14
commit 4190e40
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 18 deletions.
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -798,6 +798,7 @@ jobs:
           # build docker image
           # include building of Pulsar SQL, Connectors, Offloaders and server distros
           mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
+          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
           -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 

diff --git a/build/build_java_test_image.sh b/build/build_java_test_image.sh
@@ -27,5 +27,6 @@ if [[ "$(docker version -f '{{.Server.Experimental}}' 2>/dev/null)" == "true" ]]
   SQUASH_PARAM="-Ddocker.squash=true"
 fi
 mvn -am -pl tests/docker-images/java-test-image -Pcore-modules,-main,integrationTests,docker \
+  -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
   -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true $SQUASH_PARAM \
   "$@" install
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
@@ -46,7 +46,8 @@ function ci_print_thread_dumps() {
 
 # runs maven
 function _ci_mvn() {
-  mvn -B -ntp "$@"
+  mvn -B -ntp -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
+        "$@"
 }
 
 # runs OWASP Dependency Check for all projects

diff --git a/docker/pulsar-all/pom.xml b/docker/pulsar-all/pom.xml
@@ -68,10 +68,6 @@
   <profiles>
     <profile>
       <id>docker</id>
-      <properties>
-        <docker.buildArg.PULSAR_IO_DIR>target/apache-pulsar-io-connectors-${project.version}-bin</docker.buildArg.PULSAR_IO_DIR>
-        <docker.buildArg.PULSAR_OFFLOADER_TARBALL>target/pulsar-offloader-distribution-${project.version}-bin.tar.gz</docker.buildArg.PULSAR_OFFLOADER_TARBALL>
-      </properties>
       <!-- include the docker image only when docker profile is active -->
       <dependencies>
         <dependency>
@@ -143,23 +139,55 @@
                 <configuration>
                   <images>
                     <image>
-                      <name>pulsar-all</name>
+                      <name>${docker.organization}/pulsar-all</name>
                       <build>
                         <contextDir>${project.basedir}</contextDir>
                         <tags>
                           <tag>latest</tag>
+                          <tag>${project.version}</tag>
                         </tags>
+                        <args>
+                          <PULSAR_IO_DIR>target/apache-pulsar-io-connectors-${project.version}-bin</PULSAR_IO_DIR>
+                          <PULSAR_OFFLOADER_TARBALL>target/pulsar-offloader-distribution-${project.version}-bin.tar.gz</PULSAR_OFFLOADER_TARBALL>
+                        </args>
+                        <buildx>
+                          <platforms>
+                            <platform>${docker.platforms}</platform>
+                          </platforms>
+                        </buildx>
                       </build>
                     </image>
                   </images>
-                  <tagName>latest</tagName>
-                  <repo>${docker.organization}</repo>
                 </configuration>
               </execution>
             </executions>
           </plugin>
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>docker-push</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>io.fabric8</groupId>
+            <artifactId>docker-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>build</goal>
+                  <goal>tag</goal>
+                  <goal>push</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
   </profiles>
 </project>
diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
@@ -47,15 +47,14 @@
     </dependency>
   </dependencies>
 
+  <properties>
+      <UBUNTU_MIRROR>mirror://mirrors.ubuntu.com/mirrors.txt</UBUNTU_MIRROR>
+      <UBUNTU_SECURITY_MIRROR>http://security.ubuntu.com/ubuntu/</UBUNTU_SECURITY_MIRROR>
+  </properties>
+
   <profiles>
     <profile>
       <id>docker</id>
-      <properties>
-        <docker.buildArg.PULSAR_TARBALL>target/pulsar-server-distribution-${project.version}-bin.tar.gz</docker.buildArg.PULSAR_TARBALL>
-        <docker.buildArg.PULSAR_CLIENT_PYTHON_VERSION>${pulsar.client.python.version}</docker.buildArg.PULSAR_CLIENT_PYTHON_VERSION>
-        <docker.buildArg.UBUNTU_MIRROR>${env.UBUNTU_MIRROR}</docker.buildArg.UBUNTU_MIRROR>
-        <docker.buildArg.UBUNTU_SECURITY_MIRROR>${env.UBUNTU_SECURITY_MIRROR}</docker.buildArg.UBUNTU_SECURITY_MIRROR>
-      </properties>
       <build>
         <plugins>
           <plugin>
@@ -72,17 +71,27 @@
                 <configuration>
                   <images>
                     <image>
-                      <name>pulsar</name>
+                      <name>${docker.organization}/pulsar</name>
                       <build>
+                        <args>
+                          <PULSAR_TARBALL>target/pulsar-server-distribution-${project.version}-bin.tar.gz</PULSAR_TARBALL>
+                          <PULSAR_CLIENT_PYTHON_VERSION>${pulsar.client.python.version}</PULSAR_CLIENT_PYTHON_VERSION>
+                          <UBUNTU_MIRROR>${UBUNTU_MIRROR}</UBUNTU_MIRROR>
+                          <UBUNTU_SECURITY_MIRROR>${UBUNTU_SECURITY_MIRROR}</UBUNTU_SECURITY_MIRROR>
+                        </args>
                         <contextDir>${project.basedir}</contextDir>
                         <tags>
                           <tag>latest</tag>
+                          <tag>${project.version}</tag>
                         </tags>
+                        <buildx>
+                          <platforms>
+                            <platform>${docker.platforms}</platform>
+                          </platforms>
+                        </buildx>
                       </build>
                     </image>
                   </images>
-                  <tagName>latest</tagName>
-                  <repo>${docker.organization}</repo>
                 </configuration>
               </execution>
             </executions>
@@ -108,5 +117,29 @@
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>docker-push</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>io.fabric8</groupId>
+            <artifactId>docker-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>default</id>
+                <phase>package</phase>
+                <goals>
+                  <goal>build</goal>
+                  <goal>tag</goal>
+                  <goal>push</goal>
+                </goals>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
   </profiles>
 </project>
diff --git a/pom.xml b/pom.xml
@@ -94,6 +94,12 @@ flexible messaging model and an intuitive client API.</description>
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <project.build.outputTimestamp>${maven.build.timestamp}</project.build.outputTimestamp>
     <redirectTestOutputToFile>true</redirectTestOutputToFile>
+
+    <!-- By default build for local Docker platform.
+         To create multi-arch image, pass -Ddocker.platforms=linux/arm64,linux/amd64
+    -->
+    <docker.platforms></docker.platforms>
+
     <!-- required for running tests on JDK11+ -->
     <test.additional.args>
       --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
@@ -152,7 +158,7 @@ flexible messaging model and an intuitive client API.</description>
     <reflections.version>0.10.2</reflections.version>
     <swagger.version>1.6.2</swagger.version>
     <puppycrawl.checkstyle.version>8.37</puppycrawl.checkstyle.version>
-    <docker-maven.version>0.40.2</docker-maven.version>
+    <docker-maven.version>0.42.1</docker-maven.version>
     <docker.verbose>true</docker.verbose>
     <typetools.version>0.5.0</typetools.version>
     <protobuf3.version>3.19.6</protobuf3.version>

diff --git a/tests/docker-images/latest-version-image/pom.xml b/tests/docker-images/latest-version-image/pom.xml
@@ -147,6 +147,7 @@
                 <phase>package</phase>
                 <goals>
                   <goal>build</goal>
+                  <goal>tag</goal>
                 </goals>
                 <configuration>
                   <images>
@@ -159,6 +160,11 @@
                           <tag>${project.version}</tag>
                         </tags>
                         <noCache>true</noCache>
+                        <buildx>
+                          <platforms>
+                            <platform>${docker.platforms}</platform>
+                          </platforms>
+                        </buildx>
                       </build>
                     </image>
                   </images>