Merge pull request #289 from zwpaper/build-nerdctl-without-docker

build: add builder args to support build with nerdctl
kubernetes-sigs · Feb 22, 2023 · 1423efb · 1423efb
2 parents 42ff90d + 899e6b9
commit 1423efb
Show file tree

Hide file tree

Showing 6 changed files with 227 additions and 44 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,6 +16,11 @@ concurrency:
 jobs:
   test-build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        builder:
+          - docker
+          - nerdctl
     steps:
       - uses: actions/checkout@v3
       - name: Set up Go
@@ -38,15 +43,25 @@ jobs:
         run: |
           make build
 
+      - name: Install Nerdctl and Start Containerd
+        if: ${{ matrix.builder == 'nerdctl' }}
+        shell: bash
+        run: |
+          curl -sSL https://github.com/containerd/nerdctl/releases/download/v1.1.0/nerdctl-full-1.1.0-linux-amd64.tar.gz -o - | sudo tar -xz -C /usr/local
+          sudo systemctl daemon-reload
+          sudo systemctl enable --now containerd
+          containerd-rootless-setuptool.sh install
+          containerd-rootless-setuptool.sh install-buildkit-containerd
+
       - name: Build Image
         shell: bash
         run: |
-          make build-image
+          BUILDER=${{ matrix.builder }} make build-image
 
       - name: Build Cluster Image
         shell: bash
         run: |
-          make build-cluster-image
+          BUILDER=${{ matrix.builder }} make build-cluster-image
 
   test-kwok:
     runs-on: ubuntu-latest
@@ -175,6 +190,7 @@ jobs:
           sudo systemctl daemon-reload
           sudo systemctl enable --now containerd
           containerd-rootless-setuptool.sh install
+          containerd-rootless-setuptool.sh install-buildkit-containerd
 
       - name: Make pki directory
         if: ${{ matrix.kwokctl-runtime == 'binary' }}

diff --git a/Makefile b/Makefile
@@ -64,6 +64,7 @@ IMAGE_PLATFORMS ?= linux/amd64 linux/arm64
 
 BINARY_PLATFORMS ?= linux/amd64 linux/arm64 darwin/amd64 darwin/arm64 windows/amd64 windows/arm64
 
+BUILDER ?= docker
 DOCKER_CLI_EXPERIMENTAL ?= enabled
 
 .PHONY: default
@@ -157,6 +158,7 @@ image:
 		--version=${VERSION} \
 		--staging-prefix=${STAGING_PREFIX} \
 		--dry-run=${DRY_RUN} \
+		--builder=${BUILDER} \
 		--push=${PUSH}
 
 ## cross-image: Build kwok images for all supported platforms
@@ -169,6 +171,7 @@ cross-image:
 		--version=${VERSION} \
 		--staging-prefix=${STAGING_PREFIX} \
 		--dry-run=${DRY_RUN} \
+		--builder=${BUILDER} \
 		--push=${PUSH}
 
 ## cluster-image: Build cluster image
@@ -181,6 +184,7 @@ cluster-image:
 		--version=${VERSION} \
 		--staging-prefix=${STAGING_PREFIX} \
 		--dry-run=${DRY_RUN} \
+		--builder=${BUILDER} \
 		--push=${PUSH}
 
 ## cross-cluster-image: Build cluster images for all supported platforms and all supported Kubernetes versions.
@@ -194,6 +198,7 @@ cross-cluster-image:
 		--version=${VERSION} \
 		--staging-prefix=${STAGING_PREFIX} \
 		--dry-run=${DRY_RUN} \
+		--builder=${BUILDER} \
 		--push=${PUSH}
 
 ## integration-tests: Run integration tests

diff --git a/images/cluster/build.sh b/images/cluster/build.sh
@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -o errexit
+set -o nounset
+set -o pipefail
+
 DIR="$(dirname "${BASH_SOURCE[0]}")"
 DIR="$(realpath "${DIR}")"
 ROOT_DIR="$(realpath "${DIR}/../..")"
@@ -26,6 +30,7 @@ PLATFORMS=()
 VERSION=""
 STAGING_PREFIX=""
 KUBE_VERSIONS=()
+BUILDER="docker"
 
 function usage() {
   echo "Usage: ${0} [--help] [--version <version>] [--kube-version <kube-version> ...] [--image <image> ...] [--extra-tag <extra-tag> ...] [--staging-prefix <staging-prefix>] [--platform <platform> ...] [--push] [--dry-run]"
@@ -37,6 +42,7 @@ function usage() {
   echo "  --platform <platform> is multi-platform capable for image"
   echo "  --push will push image to registry"
   echo "  --dry-run just show what would be done"
+  echo "  --builder <builder> specify image builder, default: docker. available options: docker, nerdctl"
 }
 
 function args() {
@@ -76,6 +82,10 @@ function args() {
       [[ "${arg#*=}" != "${arg}" ]] && DRY_RUN="${arg#*=}" || DRY_RUN="true"
       shift
       ;;
+    --builder | --builder=*)
+      [[ "${arg#*=}" != "${arg}" ]] && BUILDER="${arg#*=}" || { BUILDER="${2}" && shift; }
+      shift
+      ;;
     --help)
       usage
       exit 0
@@ -106,7 +116,7 @@ function args() {
     exit 1
   fi
 
-  if [[ "${#PLATFORMS}" -eq 0 ]]; then
+  if [[ "${#PLATFORMS[*]}" -eq 0 ]]; then
     PLATFORMS+=(
       linux/amd64
     )
@@ -122,13 +132,17 @@ function dry_run() {
 
 function main() {
   local extra_args
+  local platform_args
+  local images
   local suffix
   local tag
 
   for kube_version in "${KUBE_VERSIONS[@]}"; do
+    images=()
     extra_args=(
       "--build-arg=kube_version=${kube_version}"
     )
+    platform_args=()
 
     suffix="-k8s.${kube_version}"
 
@@ -137,39 +151,62 @@ function main() {
       if [[ "${STAGING_PREFIX}" != "" ]]; then
         tag="${STAGING_PREFIX}-${VERSION}${suffix}"
       fi
-      extra_args+=(
-        "--tag=${image}:${tag}"
-      )
+      extra_args+=("--tag=${image}:${tag}")
+      images+=("${image}:${tag}")
+
       if [[ "${#EXTRA_TAGS[@]}" -ne 0 ]]; then
         for extra_tag in "${EXTRA_TAGS[@]}"; do
           tag="${extra_tag}${suffix}"
           if [[ "${STAGING_PREFIX}" != "" ]]; then
             tag="${STAGING_PREFIX}-${extra_tag}${suffix}"
           fi
-          extra_args+=(
-            "--tag=${image}:${tag}"
-          )
+          extra_args+=("--tag=${image}:${tag}")
+          images+=("${image}:${tag}")
         done
       fi
     done
 
     for platform in "${PLATFORMS[@]}"; do
-      extra_args+=(
-        "--platform=${platform}"
-      )
+      extra_args+=("--platform=${platform}")
+      platform_args+=("--platform=${platform}")
     done
-    if [[ "${PUSH}" == "true" ]]; then
-      extra_args+=("--push")
+
+    if [[ "${BUILDER}" == "nerdctl" ]]; then
+      build_with_nerdctl "${extra_args[@]}"
+      if [[ "${PUSH}" == "true" ]]; then
+        for image in "${images[@]}"; do
+          dry_run nerdctl push "${platform_args[@]}" "${image}"
+        done
+      fi
     else
-      extra_args+=("--load")
+      if [[ "${PUSH}" == "true" ]]; then
+        extra_args+=("--push")
+      else
+        extra_args+=("--load")
+      fi
+      build_with_docker "${extra_args[@]}"
     fi
-    dry_run docker buildx build \
-      "${extra_args[@]}" \
-      -f "${DOCKERFILE}" \
-      .
   done
 }
 
+function build_with_docker() {
+  local extra_args
+  extra_args=("$@")
+  dry_run docker buildx build \
+    "${extra_args[@]}" \
+    -f "${DOCKERFILE}" \
+    .
+}
+
+function build_with_nerdctl() {
+  local extra_args
+  extra_args=("$@")
+  dry_run nerdctl build \
+    "${extra_args[@]}" \
+    -f "${DOCKERFILE}" \
+    .
+}
+
 args "$@"
 
 cd "${ROOT_DIR}" && main
diff --git a/images/kwok/build.sh b/images/kwok/build.sh
@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -o errexit
+set -o nounset
+set -o pipefail
+
 DIR="$(dirname "${BASH_SOURCE[0]}")"
 DIR="$(realpath "${DIR}")"
 ROOT_DIR="$(realpath "${DIR}/../..")"
@@ -25,16 +29,18 @@ EXTRA_TAGS=()
 PLATFORMS=()
 VERSION=""
 STAGING_PREFIX=""
+BUILDER="docker"
 
 function usage() {
-  echo "Usage: ${0} [--help] [--version <version>] [--image <image> ...] [--extra-tag <extra-tag> ...] [--staging-prefix <staging-prefix>] [--platform <platform> ...] [--push] [--dry-run]"
+  echo "Usage: ${0} [--help] [--version <version>] [--image <image> ...] [--extra-tag <extra-tag> ...] [--staging-prefix <staging-prefix>] [--platform <platform> ...] [--push] [--dry-run] [--builder <builder>]"
   echo "  --version <version> is kwok version, is required"
   echo "  --image <image> is image, is required"
   echo "  --extra-tag <extra-tag> is extra tag"
   echo "  --staging-prefix <staging-prefix> is staging prefix for tag"
   echo "  --platform <platform> is multi-platform capable for image"
   echo "  --push will push image to registry"
   echo "  --dry-run just show what would be done"
+  echo "  --builder <builder> specify image builder, default: docker. available options: docker, nerdctl"
 }
 
 function args() {
@@ -70,6 +76,10 @@ function args() {
       [[ "${arg#*=}" != "${arg}" ]] && DRY_RUN="${arg#*=}" || DRY_RUN="true"
       shift
       ;;
+    --builder | --builder=*)
+      [[ "${arg#*=}" != "${arg}" ]] && BUILDER="${arg#*=}" || { BUILDER="${2}" && shift; }
+      shift
+      ;;
     --help)
       usage
       exit 0
@@ -94,7 +104,7 @@ function args() {
     exit 1
   fi
 
-  if [[ "${#PLATFORMS}" -eq 0 ]]; then
+  if [[ "${#PLATFORMS[*]}" -eq 0 ]]; then
     PLATFORMS+=(
       linux/amd64
     )
@@ -110,46 +120,72 @@ function dry_run() {
 
 function main() {
   local extra_args
+  local platform_args
+  local images
   local tag
 
   extra_args=()
+  images=()
   for image in "${IMAGES[@]}"; do
     tag="${VERSION}"
     if [[ "${STAGING_PREFIX}" != "" ]]; then
       tag="${STAGING_PREFIX}-${VERSION}"
     fi
-    extra_args+=(
-      "--tag=${image}:${tag}"
-    )
+    extra_args+=("--tag=${image}:${tag}")
+    images+=("${image}:${tag}")
+
     if [[ "${#EXTRA_TAGS[@]}" -ne 0 ]]; then
       for extra_tag in "${EXTRA_TAGS[@]}"; do
         tag="${extra_tag}"
         if [[ "${STAGING_PREFIX}" != "" ]]; then
           tag="${STAGING_PREFIX}-${extra_tag}"
         fi
-        extra_args+=(
-          "--tag=${image}:${tag}"
-        )
+        extra_args+=("--tag=${image}:${tag}")
+        images+=("${image}:${tag}")
       done
     fi
   done
 
   for platform in "${PLATFORMS[@]}"; do
-    extra_args+=(
-      "--platform=${platform}"
-    )
+    extra_args+=("--platform=${platform}")
+    platform_args+=("--platform=${platform}")
   done
-  if [[ "${PUSH}" == "true" ]]; then
-    extra_args+=("--push")
+
+  if [[ "${BUILDER}" == "nerdctl" ]]; then
+    build_with_nerdctl "${extra_args[@]}"
+    if [[ "${PUSH}" == "true" ]]; then
+      for image in "${images[@]}"; do
+        dry_run nerdctl push "${platform_args[@]}" "${image}"
+      done
+    fi
   else
-    extra_args+=("--load")
+    if [[ "${PUSH}" == "true" ]]; then
+      extra_args+=("--push")
+    else
+      extra_args+=("--load")
+    fi
+    build_with_docker "${extra_args[@]}"
   fi
+}
+
+function build_with_docker() {
+  local extra_args
+  extra_args=("$@")
   dry_run docker buildx build \
     "${extra_args[@]}" \
     -f "${DOCKERFILE}" \
     .
 }
 
+function build_with_nerdctl() {
+  local extra_args
+  extra_args=("$@")
+  dry_run nerdctl build \
+    "${extra_args[@]}" \
+    -f "${DOCKERFILE}" \
+    .
+}
+
 args "$@"
 
 cd "${ROOT_DIR}" && main