ci: refactor docker ci script and enable docker job in presubmit

.azure-pipelines/pipelines.yml

@@ -123,8 +123,7 @@ jobs:
 
   - job: docker
     displayName: "Linux multi-arch docker"
-    dependsOn: ["release","release_arm64"]
-    condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
+    dependsOn: ["release", "release_arm64"]
     pool:
       vmImage: "ubuntu-18.04"
     steps:
@@ -153,6 +152,11 @@ jobs:
           AZP_SHA1: $(Build.SourceVersion)
           DOCKERHUB_USERNAME: $(DockerUsername)
           DOCKERHUB_PASSWORD: $(DockerPassword)
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: "$(Build.StagingDirectory)/build_images"
+          artifactName: docker
+        condition: always()
 
   - job: macOS
     dependsOn: ["format"]

ci/Dockerfile-envoy-google-vrp

@@ -1,4 +1,5 @@
-FROM envoyproxy/envoy:local
+ARG ENVOY_VRP_BASE_IMAGE
+FROM $ENVOY_VRP_BASE_IMAGE
 
 RUN apt-get update \
     && apt-get upgrade -y \

ci/docker_ci.sh

@@ -4,46 +4,90 @@
 # CI logs.
 set -e
 
+ENVOY_DOCKER_IMAGE_DIRECTORY="${ENVOY_DOCKER_IMAGE_DIRECTORY:-${BUILD_STAGINGDIRECTORY:-.}/build_images}"
+
 # Setting environments for buildx tools
-config_env(){
-    # Qemu configurations
-    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+config_env() {
+  # Qemu configurations
+  docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+
+  # Remove older build instance
+  docker buildx rm multi-builder | true
+  docker buildx create --use --name multi-builder --platform linux/arm64,linux/amd64
+}
+
+build_platforms() {
+  TYPE=$1
+  if [[ -z "${TYPE}" ]]; then
+    echo "linux/arm64,linux/amd64"
+  else
+    echo "linux/amd64"
+  fi
+}
 
-    # Remove older build instance
-    docker buildx rm  multi-builder | true
-    docker buildx create --use --name multi-builder --platform linux/arm64,linux/amd64
+build_args() {
+  TYPE=$1
+  if [[ "${TYPE}" == "-google-vrp" ]]; then
+    echo "--build-arg ENVOY_VRP_BASE_IMAGE=${VRP_BASE_IMAGE}"
+  fi
 }
 
-build_images(){
-    TYPE=$1
-    BUILD_TAG=$2
-
-    # Only build/push envoyproxy/envoy multi-arch images since others still do not support.
-    if [ -z "${TYPE}" ]; then
-        docker buildx build --platform linux/arm64 -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
-        # Export envoyproxy/envoy amd64 image which will be used for building envoyproxy/envoy-google-vrp
-        docker buildx build --platform linux/amd64 -f ci/Dockerfile-envoy"${TYPE}" -o type=docker -t ${BUILD_TAG} .
-    elif [ "${TYPE}" == "-google-vrp" ]; then
-        # The envoyproxy/envoy-google-vrp is based on envoyproxy/envoy image. So it is built from cache envoyproxy/envoy:local
-        docker build -f ci/Dockerfile-envoy"${TYPE}" --cache-from "${DOCKER_IMAGE_PREFIX}:local" -t ${BUILD_TAG} .
-    else
-        docker build -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
-    fi
+use_builder() {
+  TYPE=$1
+  if [[ "${TYPE}" == "-google-vrp" ]]; then
+    docker buildx use default
+  else
+    docker buildx use multi-builder
+  fi
+}
+
+IMAGES_TO_SAVE=()
+
+build_images() {
+  TYPE=$1
+  BUILD_TAG=$2
+  BASE=$3
+
+  use_builder "${TYPE}"
+  ARGS="$(build_args ${TYPE})"
+  PLATFORM="$(build_platforms ${TYPE})"
+
+  docker buildx build --platform "${PLATFORM}" -f ci/Dockerfile-envoy"${TYPE}" ${ARGS} -t "${BUILD_TAG}" .
+
+  PLATFORM="$(build_platforms ${TYPE} | tr ',' ' ')"
+  # docker buildx load cannot have multiple platform, load individually
+  for ARCH in ${PLATFORM}; do
+    IMAGE_TAG="${BUILD_TAG}-${ARCH/linux\//}"
+    docker buildx build --platform "${ARCH}" -f ci/Dockerfile-envoy"${TYPE}" ${ARGS} -t "${IMAGE_TAG}" . --load
+    IMAGES_TO_SAVE+=("${IMAGE_TAG}")
+  done
 }
 
-push_images(){
-    TYPE=$1
-    BUILD_TAG=$2
-
-    if [ -z "${TYPE}" ]; then
-        # Only push envoyproxy/envoy multi-arch images since others still do not support.
-        docker buildx build --platform linux/arm64,linux/amd64 --push -f ci/Dockerfile-envoy"${TYPE}" -t ${BUILD_TAG} .
-    else
-        docker tag "${DOCKER_IMAGE_PREFIX}${TYPE}:local" ${BUILD_TAG}
-        docker push ${BUILD_TAG}
-    fi
+push_images() {
+  TYPE=$1
+  BUILD_TAG=$2
+  BASE=$3
+
+  use_builder "${TYPE}"
+  ARGS="$(build_args ${TYPE})"
+  PLATFORM="$(build_platforms ${TYPE})"
+  docker buildx build --platform "${PLATFORM}" -f ci/Dockerfile-envoy"${TYPE}" ${ARGS} -t ${BUILD_TAG} . --push
 }
 
+MASTER_BRANCH="refs/heads/master"
+RELEASE_BRANCH_REGEX="^refs/heads/release/v.*"
+RELEASE_TAG_REGEX="^refs/tags/v.*"
+
+# For master builds and release branch builds use the dev repo. Otherwise we assume it's a tag and
+# we push to the primary repo.
+if [[ "${AZP_BRANCH}" =~ "${RELEASE_TAG_REGEX}" ]]; then
+  IMAGE_POSTFIX=""
+  IMAGE_NAME="${AZP_BRANCH/refs\/tags\//}"
+else
+  IMAGE_POSTFIX="-dev"
+  IMAGE_NAME="${AZP_SHA1}"
+fi
+
 # This prefix is altered for the private security images on setec builds.
 DOCKER_IMAGE_PREFIX="${DOCKER_IMAGE_PREFIX:-envoyproxy/envoy}"
 
@@ -53,48 +97,41 @@ BUILD_TYPES=("" "-alpine" "-alpine-debug" "-google-vrp")
 # Configure docker-buildx tools
 config_env
 
+# VRP base image is only for amd64
+VRP_BASE_IMAGE="${DOCKER_IMAGE_PREFIX}${IMAGE_POSTFIX}:${IMAGE_NAME}-amd64"
+
 # Test the docker build in all cases, but use a local tag that we will overwrite before push in the
 # cases where we do push.
 for BUILD_TYPE in "${BUILD_TYPES[@]}"; do
-    build_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}:local"
+  build_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
 done
 
-MASTER_BRANCH="refs/heads/master"
-RELEASE_BRANCH_REGEX="^refs/heads/release/v.*"
-RELEASE_TAG_REGEX="^refs/tags/v.*"
+mkdir -p "${ENVOY_DOCKER_IMAGE_DIRECTORY}"
+ENVOY_DOCKER_TAR="${ENVOY_DOCKER_IMAGE_DIRECTORY}/envoy-docker-images.tar.xz"
+echo "Saving built images to ${ENVOY_DOCKER_TAR}."
+docker save "${IMAGES_TO_SAVE[@]}" | xz -T0 -2 >"${ENVOY_DOCKER_TAR}"
 
 # Only push images for master builds, release branch builds, and tag builds.
-if [[ "${AZP_BRANCH}" != "${MASTER_BRANCH}" ]] && \
-   ! [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]] && \
-   ! [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
-    echo 'Ignoring non-master branch or tag for docker push.'
-    exit 0
-fi
-
-# For master builds and release branch builds use the dev repo. Otherwise we assume it's a tag and
-# we push to the primary repo.
-if [[ "${AZP_BRANCH}" == "${MASTER_BRANCH}" ]] || \
-   [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]]; then
-  IMAGE_POSTFIX="-dev"
-  IMAGE_NAME="$AZP_SHA1"
-else
-  IMAGE_POSTFIX=""
-  IMAGE_NAME="${AZP_BRANCH/refs\/tags\//}"
+if [[ "${AZP_BRANCH}" != "${MASTER_BRANCH}" ]] &&
+  ! [[ "${AZP_BRANCH}" =~ ${RELEASE_BRANCH_REGEX} ]] &&
+  ! [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+  echo 'Ignoring non-master branch or tag for docker push.'
+  exit 0
 fi
 
 docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD"
 
 for BUILD_TYPE in "${BUILD_TYPES[@]}"; do
-    push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}"
-
-    # Only push latest on master builds.
-    if [[ "${AZP_BRANCH}" == "${MASTER_BRANCH}" ]]; then
-        push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:latest"
-    fi
-
-    # Push vX.Y-latest to tag the latest image in a release line
-    if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
-      RELEASE_LINE=$(echo "$IMAGE_NAME" | sed -E 's/(v[0-9]+\.[0-9]+)\.[0-9]+/\1-latest/')
-        push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${RELEASE_LINE}"
-    fi
+  push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${IMAGE_NAME}" "${BASE_IMAGE}"
+
+  # Only push latest on master builds.
+  if [[ "${AZP_BRANCH}" == "${MASTER_BRANCH}" ]]; then
+    push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:latest" "${BASE_IMAGE}"
+  fi
+
+  # Push vX.Y-latest to tag the latest image in a release line
+  if [[ "${AZP_BRANCH}" =~ ${RELEASE_TAG_REGEX} ]]; then
+    RELEASE_LINE=$(echo "$IMAGE_NAME" | sed -E 's/(v[0-9]+\.[0-9]+)\.[0-9]+/\1-latest/')
+    push_images "${BUILD_TYPE}" "${DOCKER_IMAGE_PREFIX}${BUILD_TYPE}${IMAGE_POSTFIX}:${RELEASE_LINE}" "${BASE_IMAGE}"
+  fi
 done

ci/docker_rebuild_google-vrp.sh

@@ -19,35 +19,30 @@
 
 set -e
 
-# This should match your local machine if you are building custom Envoy binaries outside of Docker.
-BASE_DOCKER_IMAGE="ubuntu:20.04"
+# Don't use the local envoy-dev, but pull from Docker Hub instead, this avoids having to rebuild
+# this local dep which is fairly stable.
+BASE_DOCKER_IMAGE="envoyproxy/envoy-dev:latest"
 
 declare -r BUILD_DIR="$(mktemp -d)"
 cp ci/Dockerfile-envoy-google-vrp "${BUILD_DIR}"
 declare -r DOCKER_BUILD_FILE="${BUILD_DIR}"/Dockerfile-envoy-google-vrp
 
 # If we have a local Envoy binary, use a variant of the build environment that supports it.
-if [[ -n "$1" ]]
-then
-  # Switch to a base image similar to the local build environment. This provides compatibility of
-  # locally built Envoy and glibc in the Docker env.
-  sed -i -e "s#envoyproxy/envoy:local#${BASE_DOCKER_IMAGE}#" "${DOCKER_BUILD_FILE}"
+if [[ -n "$1" ]]; then
+  # This should match your local machine if you are building custom Envoy binaries outside of Docker.
+  # This provides compatibility of locally built Envoy and glibc in the Docker env.
+  BASE_DOCKER_IMAGE="ubuntu:20.04"
   # Copy the binary to deal with symlinks in Bazel cache and Docker daemon confusion.
   declare -r LOCAL_ENVOY="envoy-binary"
   cp -f "$1" "${PWD}/${LOCAL_ENVOY}"
   sed -i -e "s@# ADD %local envoy bin%@ADD ${LOCAL_ENVOY}@" "${DOCKER_BUILD_FILE}"
-else
-  # Don't use the local envoy-dev, but pull from Docker Hub instead, this avoids having to rebuild
-  # this local dep which is fairly stable.
-  sed -i -e "s#envoyproxy/envoy:local#envoyproxy/envoy-dev:latest#" "${DOCKER_BUILD_FILE}"
 fi
 
 cat "${DOCKER_BUILD_FILE}"
 
-docker build -t "envoy-google-vrp:local" -f "${DOCKER_BUILD_FILE}" .
+docker build -t "envoy-google-vrp:local" --build-arg "ENVOY_VRP_BASE_IMAGE=${BASE_DOCKER_IMAGE}" -f "${DOCKER_BUILD_FILE}" .
 
-if [[ -n "$1" ]]
-then
+if [[ -n "$1" ]]; then
   rm -f "${LOCAL_ENVOY}"
 fi
 rm -r "${BUILD_DIR}"