test: test build on docker and nerdctl

Signed-off-by: Wei Zhang <kweizh@gmail.com>
kubernetes-sigs · Feb 21, 2023 · dd27508 · dd27508
1 parent e484ec1
commit dd27508
Show file tree

Hide file tree

Showing 4 changed files with 124 additions and 12 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,6 +16,11 @@ concurrency:
 jobs:
   test-build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        builder:
+          - docker
+          - nerdctl
     steps:
       - uses: actions/checkout@v3
       - name: Set up Go
@@ -33,15 +38,25 @@ jobs:
         run: |
           make build
 
+      - name: Install Nerdctl and Start Containerd
+        if: ${{ matrix.builder == 'nerdctl' }}
+        shell: bash
+        run: |
+          curl -sSL https://github.com/containerd/nerdctl/releases/download/v1.1.0/nerdctl-full-1.1.0-linux-amd64.tar.gz -o - | sudo tar -xz -C /usr/local
+          sudo systemctl daemon-reload
+          sudo systemctl enable --now containerd
+          containerd-rootless-setuptool.sh install
+          containerd-rootless-setuptool.sh install-buildkit-containerd
+
       - name: Build Image
         shell: bash
         run: |
-          make build-image
+          BUILDER=${{ matrix.builder }} make build-image
 
       - name: Build Cluster Image
         shell: bash
         run: |
-          make build-cluster-image
+          BUILDER=${{ matrix.builder }} make build-cluster-image
 
   test-kwok:
     runs-on: ubuntu-latest
@@ -170,6 +185,7 @@ jobs:
           sudo systemctl daemon-reload
           sudo systemctl enable --now containerd
           containerd-rootless-setuptool.sh install
+          containerd-rootless-setuptool.sh install-buildkit-containerd
 
       - name: Make pki directory
         if: ${{ matrix.kwokctl-runtime == 'binary' }}

diff --git a/images/cluster/build.sh b/images/cluster/build.sh
@@ -178,6 +178,11 @@ function main() {
         done
       fi
     else
+      if [[ "${PUSH}" == "true" ]]; then
+        extra_args+=("--push")
+      else
+        extra_args+=("--load")
+      fi
       build_with_docker "${extra_args[@]}"
     fi
   done
@@ -186,11 +191,6 @@ function main() {
 function build_with_docker() {
   local extra_args
   extra_args=("$@")
-  if [[ "${PUSH}" == "true" ]]; then
-    extra_args+=("--push")
-  else
-    extra_args+=("--load")
-  fi
   dry_run docker buildx build \
     "${extra_args[@]}" \
     -f "${DOCKERFILE}" \

diff --git a/images/kwok/build.sh b/images/kwok/build.sh
@@ -159,18 +159,18 @@ function main() {
       done
     fi
   else
+    if [[ "${PUSH}" == "true" ]]; then
+      extra_args+=("--push")
+    else
+      extra_args+=("--load")
+    fi
     build_with_docker "${extra_args[@]}"
   fi
 }
 
 function build_with_docker() {
   local extra_args
   extra_args=("$@")
-  if [[ "${PUSH}" == "true" ]]; then
-    extra_args+=("--push")
-  else
-    extra_args+=("--load")
-  fi
   dry_run docker buildx build \
     "${extra_args[@]}" \
     -f "${DOCKERFILE}" \

diff --git a/test/release/build.test.sh b/test/release/build.test.sh
@@ -73,36 +73,76 @@ docker buildx build --tag=kwok:${VERSION} --platform=linux/amd64 --load -f image
 EOF
 }
 
+function want_image_nerdctl() {
+  cat <<EOF
+nerdctl build --tag=kwok:${VERSION} --platform=linux/amd64 -f images/kwok/Dockerfile .
+EOF
+}
+
 function want_image_with_push() {
   cat <<EOF
 docker buildx build --tag=kwok:${VERSION} --platform=linux/amd64 --push -f images/kwok/Dockerfile .
 EOF
 }
 
+function want_image_nerdctl_with_push() {
+  cat <<EOF
+nerdctl build --tag=kwok:${VERSION} --platform=linux/amd64 -f images/kwok/Dockerfile .
+nerdctl push --platform=linux/amd64 kwok:${VERSION}
+EOF
+}
+
 function want_image_with_push_staging() {
   cat <<EOF
 docker buildx build --tag=${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION} --platform=linux/amd64 --push -f images/kwok/Dockerfile .
 EOF
 }
 
+function want_image_nerdctl_with_push_staging() {
+  cat <<EOF
+nerdctl build --tag=${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION} --platform=linux/amd64 -f images/kwok/Dockerfile .
+nerdctl push --platform=linux/amd64 ${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION}
+EOF
+}
+
 function want_cluster_image() {
   cat <<EOF
 docker buildx build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=cluster:${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 --load -f images/cluster/Dockerfile .
 EOF
 }
 
+function want_cluster_image_nerdctl() {
+  cat <<EOF
+nerdctl build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=cluster:${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 -f images/cluster/Dockerfile .
+EOF
+}
+
 function want_cluster_image_with_push() {
   cat <<EOF
 docker buildx build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=cluster:${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 --push -f images/cluster/Dockerfile .
 EOF
 }
 
+function want_cluster_image_nerdctl_with_push() {
+  cat <<EOF
+nerdctl build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=cluster:${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 -f images/cluster/Dockerfile .
+nerdctl push --platform=linux/amd64 cluster:${VERSION}-k8s.v${LAST_KUBE_RELEASE}
+EOF
+}
+
 function want_cluster_image_with_push_staging() {
   cat <<EOF
 docker buildx build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 --push -f images/cluster/Dockerfile .
 EOF
 }
 
+function want_cluster_image_nerdctl_with_push_staging() {
+  cat <<EOF
+nerdctl build --build-arg=kube_version=v${LAST_KUBE_RELEASE} --tag=${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${LAST_KUBE_RELEASE} --platform=linux/amd64 -f images/cluster/Dockerfile .
+nerdctl push --platform=linux/amd64 ${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${LAST_KUBE_RELEASE}
+EOF
+}
+
 function want_cross_build() {
   cat <<EOF
 GOOS=linux GOARCH=amd64 go build -ldflags '-X sigs.k8s.io/kwok/pkg/consts.Version=${VERSION} -X sigs.k8s.io/kwok/pkg/consts.KubeVersion=v${LAST_KUBE_RELEASE}' -o ./bin/linux/amd64/kwok ./cmd/kwok
@@ -225,36 +265,76 @@ docker buildx build --tag=kwok:${VERSION} --platform=linux/amd64 --platform=linu
 EOF
 }
 
+function want_cross_image_nerdctl() {
+  cat <<EOF
+nerdctl build --tag=kwok:${VERSION} --platform=linux/amd64 --platform=linux/arm64 -f images/kwok/Dockerfile .
+EOF
+}
+
 function want_cross_image_with_push() {
   cat <<EOF
 docker buildx build --tag=kwok:${VERSION} --platform=linux/amd64 --platform=linux/arm64 --push -f images/kwok/Dockerfile .
 EOF
 }
 
+function want_cross_image_nerdctl_with_push() {
+  cat <<EOF
+nerdctl build --tag=kwok:${VERSION} --platform=linux/amd64 --platform=linux/arm64 -f images/kwok/Dockerfile .
+nerdctl push --platform=linux/amd64 --platform=linux/arm64 kwok:${VERSION}
+EOF
+}
+
 function want_cross_image_with_push_staging() {
   cat <<EOF
 docker buildx build --tag=${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION} --platform=linux/amd64 --platform=linux/arm64 --push -f images/kwok/Dockerfile .
 EOF
 }
 
+function want_cross_image_nerdctl_with_push_staging() {
+  cat <<EOF
+nerdctl build --tag=${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION} --platform=linux/amd64 --platform=linux/arm64 -f images/kwok/Dockerfile .
+nerdctl push --platform=linux/amd64 --platform=linux/arm64 ${IMAGE_PREFIX}/kwok:${PREFIX}-${VERSION}
+EOF
+}
+
 function want_cross_cluster_image() {
   for v in ${SUPPORTED_RELEASES} ; do
     echo "docker buildx build --build-arg=kube_version=v${v} --tag=cluster:${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 --load -f images/cluster/Dockerfile ."
   done
 }
 
+function want_cross_cluster_image_nerdctl() {
+  for v in ${SUPPORTED_RELEASES} ; do
+    echo "nerdctl build --build-arg=kube_version=v${v} --tag=cluster:${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 -f images/cluster/Dockerfile ."
+  done
+}
+
 function want_cross_cluster_image_with_push() {
   for v in ${SUPPORTED_RELEASES} ; do
     echo "docker buildx build --build-arg=kube_version=v${v} --tag=cluster:${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 --push -f images/cluster/Dockerfile ."
   done
 }
 
+function want_cross_cluster_image_nerdctl_with_push() {
+  for v in ${SUPPORTED_RELEASES} ; do
+    echo "nerdctl build --build-arg=kube_version=v${v} --tag=cluster:${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 -f images/cluster/Dockerfile ."
+    echo "nerdctl push --platform=linux/amd64 --platform=linux/arm64 cluster:${VERSION}-k8s.v${v}"
+  done
+}
+
 function want_cross_cluster_image_with_push_staging() {
   for v in ${SUPPORTED_RELEASES} ; do
     echo "docker buildx build --build-arg=kube_version=v${v} --tag=${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 --push -f images/cluster/Dockerfile ."
   done
 }
 
+function want_cross_cluster_image_nerdctl_with_push_staging() {
+  for v in ${SUPPORTED_RELEASES} ; do
+    echo "nerdctl build --build-arg=kube_version=v${v} --tag=${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${v} --platform=linux/amd64 --platform=linux/arm64 -f images/cluster/Dockerfile ."
+    echo "nerdctl --platform=linux/amd64 --platform=linux/arm64 push ${IMAGE_PREFIX}/cluster:${PREFIX}-${VERSION}-k8s.v${v}"
+  done
+}
+
 function main() {
   failed=()
   export DRY_RUN=true
@@ -267,10 +347,18 @@ function main() {
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true image | diff -u <(want_image_with_push) - || failed+=("image-with-push")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} image | diff -u <(want_image_with_push_staging) - || failed+=("image-with-push-staging")
 
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl image | diff -u <(want_image_nerdctl) - || failed+=("image-nerdctl")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true image | diff -u <(want_image_nerdctl_with_push) - || failed+=("image-nerdctl-with-push")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} image | diff -u <(want_image_nerdctl_with_push_staging) - || failed+=("image--nerdctl-with-push-staging")
+
   make --no-print-directory -C "${ROOT_DIR}" cluster-image | diff -u <(want_cluster_image) - || failed+=("cluster-image")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true cluster-image | diff -u <(want_cluster_image_with_push) - || failed+=("cluster-image-with-push")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cluster-image | diff -u <(want_cluster_image_with_push_staging) - || failed+=("cluster-image-with-push-staging")
 
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl cluster-image | diff -u <(want_cluster_image_nerdctl) - || failed+=("cluster-image-nerdctl")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true cluster-image | diff -u <(want_cluster_image_nerdctl_with_push) - || failed+=("cluster-image-nerdctl-with-push")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cluster-image | diff -u <(want_cluster_image_nerdctl_with_push_staging) - || failed+=("cluster-image-nerdctl-with-push-staging")
+
   make --no-print-directory -C "${ROOT_DIR}" cross-build | diff -u <(want_cross_build) - || failed+=("cross-build")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket cross-build | diff -u <(want_cross_build_with_push_bucket) - || failed+=("cross-build-with-push-bucket")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} cross-build | diff -u <(want_cross_build_with_push_bucket_staging) - || failed+=("cross-build-with-push-bucket-staging")
@@ -280,10 +368,18 @@ function main() {
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true cross-image | diff -u <(want_cross_image_with_push) - || failed+=("cross-image-with-push")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cross-image | diff -u <(want_cross_image_with_push_staging) - || failed+=("cross-image-with-push-staging")
 
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl cross-image | diff -u <(want_cross_image_nerdctl) - || failed+=("cross-image-nerdctl")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true cross-image | diff -u <(want_cross_image_nerdctl_with_push) - || failed+=("cross-image-nerdctl-with-push")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cross-image | diff -u <(want_cross_image_nerdctl_with_push_staging) - || failed+=("cross-image-nerdctl-with-push-staging")
+
   make --no-print-directory -C "${ROOT_DIR}" cross-cluster-image | diff -u <(want_cross_cluster_image) - || failed+=("cross-cluster-image")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true cross-cluster-image | diff -u <(want_cross_cluster_image_with_push) - || failed+=("cross-cluster-image-with-push")
   make --no-print-directory -C "${ROOT_DIR}" PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cross-cluster-image | diff -u <(want_cross_cluster_image_with_push_staging) - || failed+=("cross-cluster-image-with-push-staging")
 
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl cross-cluster-image | diff -u <(want_cross_cluster_image_nerdctl) - || failed+=("cross-cluster-image-nerdctl")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true cross-cluster-image | diff -u <(want_cross_cluster_image_nerdctl_with_push) - || failed+=("cross-cluster-image-nerdctl-with-push")
+  make --no-print-directory -C "${ROOT_DIR}" BUILDER=nerdctl PUSH=true BUCKET=bucket STAGING=true STAGING_PREFIX=${PREFIX} STAGING_IMAGE_PREFIX=${IMAGE_PREFIX} cross-cluster-image | diff -u <(want_cross_cluster_image_nerdctl_with_push_staging) - || failed+=("cross-cluster-image-nerdctl-with-push-staging")
+
   if [[ "${#failed[@]}" -ne 0 ]]; then
     echo "Error: Some tests failed"
     for test in "${failed[@]}"; do