Merge pull request #2176 from BenTheElder/new-node-image

support cross-compiling node-image

hack/release/build/push-node.sh

@@ -22,25 +22,47 @@ cd "${REPO_ROOT}"
 # ensure we have up to date kind
 make build
 
-# generate tag
-DATE="$(date +v%Y%m%d)"
-TAG="${DATE}-$(git describe --always --dirty)"
-
-# build
+# path to kubernetes sources
 KUBEROOT="${KUBEROOT:-${GOPATH}/src/k8s.io/kubernetes}"
 
+# ensure we have qemu setup (de-duped logic with setting up buildx for multi-arch)
+"${REPO_ROOT}/hack/build/init-buildx.sh"
+
+# kubernetes build option(s)
 GOFLAGS="${GOFLAGS:-}"
 if [ -z "${GOFLAGS}" ]; then
     # TODO: add dockerless when 1.19 or greater
     GOFLAGS="-tags=providerless"
 fi
+
+# NOTE: adding platforms is costly in terms of build time
+# we will consider expanding this in the future, for now the aim is to prove
+# multi-arch and enable developers working on commonly available hardware
+# Other users are free to build their own images on additional platforms using
+# their own time and resources. Please see our docs.
+ARCHES="${ARCHES:-amd64 arm64}"
+IFS=" " read -r -a __arches__ <<< "$ARCHES"
+
 set -x
-"${REPO_ROOT}/bin/kind" build node-image --image="kindest/node:${TAG}" --kube-root="${KUBEROOT}"
+# get kubernetes version
+version_line="$(cd "${KUBEROOT}"; ./hack/print-workspace-status.sh | grep 'gitVersion')"
+kube_version="${version_line#"gitVersion "}"
 
-# re-tag with kubernetes version
-IMG="kindest/node:${TAG}"
-KUBE_VERSION="$(docker run --rm --entrypoint=cat "${IMG}" /kind/version)"
-docker tag "${IMG}" "kindest/node:${KUBE_VERSION}"
+# build for each arch
+IMAGE="kindest/node:${kube_version}"
+images=()
+for arch in "${__arches__[@]}"; do
+    image="kindest/node-${arch}:${kube_version}"
+    "${REPO_ROOT}/bin/kind" build node-image --image="${image}" --arch="${arch}" "${KUBEROOT}"
+    images+=("${image}")
+done
 
-# push
-docker push kindest/node:"${KUBE_VERSION}"
+# combine to manifest list tagged with kubernetes version
+export DOCKER_CLI_EXPERIMENTAL=enabled
+# images must be pushed to be referenced by docker manifest
+# we push only after all builds have succeeded
+for image in "${images[@]}"; do
+    docker push "${image}"
+done
+docker manifest create "${IMAGE}" "${images[@]}"
+docker manifest push "${IMAGE}"

pkg/apis/config/defaults/image.go

@@ -18,4 +18,4 @@ limitations under the License.
 package defaults
 
 // Image is the default for the Config.Image field, aka the default node image.
-const Image = "kindest/node:v1.20.2@sha256:15d3b5c4f521a84896ed1ead1b14e4774d02202d5c65ab68f30eeaf310a3b1a7"
+const Image = "kindest/node:v1.21.1@sha256:c6eead46eaba71017e290f696fa675187133d7953e9291900e384f711b6cf8ed"

pkg/build/nodeimage/build.go

@@ -31,8 +31,7 @@ func Build(options ...Option) error {
 		image:     DefaultImage,
 		baseImage: DefaultBaseImage,
 		logger:    log.NoopLogger{},
-		// TODO: only host arch supported. changing this will be tricky
-		arch: runtime.GOARCH,
+		arch:      runtime.GOARCH,
 	}
 
 	// apply user options
@@ -44,7 +43,7 @@ func Build(options ...Option) error {
 
 	// verify that we're using a supported arch
 	if !supportedArch(ctx.arch) {
-		return errors.Errorf("unsupported architecture %q", ctx.arch)
+		ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
 	}
 
 	// locate sources if no kubernetes source was specified
@@ -78,16 +77,3 @@ func supportedArch(arch string) bool {
 	}
 	return true
 }
-
-// buildContext is used to build the kind node image, and contains
-// build configuration
-type buildContext struct {
-	// option fields
-	image     string
-	baseImage string
-	logger    log.Logger
-	// non-option fields
-	arch     string // TODO(bentheelder): this should be an option
-	kubeRoot string
-	builder  kube.Builder
-}

pkg/build/nodeimage/build_impl.go → pkg/build/nodeimage/buildcontext.go

@@ -33,8 +33,22 @@ import (
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/fs"
+	"sigs.k8s.io/kind/pkg/log"
 )
 
+// buildContext is used to build the kind node image, and contains
+// build configuration
+type buildContext struct {
+	// option fields
+	image     string
+	baseImage string
+	logger    log.Logger
+	arch      string
+	kubeRoot  string
+	// non-option fields
+	builder kube.Builder
+}
+
 // Build builds the cluster node image, the sourcedir must be set on
 // the buildContext
 func (c *buildContext) Build() (err error) {
@@ -188,7 +202,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	fixRepository := func(repository string) string {
 		if strings.HasSuffix(repository, archSuffix) {
 			fixed := strings.TrimSuffix(repository, archSuffix)
-			fmt.Println("fixed: " + repository + " -> " + fixed)
+			c.logger.V(1).Info("fixed: " + repository + " -> " + fixed)
 			repository = fixed
 		}
 		return repository
@@ -206,7 +220,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		fixedImages.Insert(registry + ":" + tag)
 	}
 	builtImages = fixedImages
-	c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
+	c.logger.V(1).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
 
 	// gets the list of images required by kubeadm
 	requiredImages, err := exec.OutputLines(cmder.Command(
@@ -262,38 +276,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		return nil, errors.Wrap(err, "failed to make images dir")
 	}
 
-	fns := []func() error{}
-	pulledImages := make(chan string, len(requiredImages))
-	for i, image := range requiredImages {
-		i, image := i, image // https://golang.org/doc/faq#closures_and_goroutines
-		fns = append(fns, func() error {
-			if !builtImages.Has(image) {
-				fmt.Printf("Pulling: %s\n", image)
-				err := docker.Pull(c.logger, image, 2)
-				if err != nil {
-					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
-				}
-				// TODO(bentheelder): generate a friendlier name
-				pullName := fmt.Sprintf("%d.tar", i)
-				pullTo := path.Join(imagesDir, pullName)
-				err = docker.Save(image, pullTo)
-				if err != nil {
-					return err
-				}
-				pulledImages <- pullTo
-			}
-			return nil
-		})
-	}
-	if err := errors.AggregateConcurrent(fns); err != nil {
-		return nil, err
-	}
-	close(pulledImages)
-	pulled := []string{}
-	for image := range pulledImages {
-		pulled = append(pulled, image)
-	}
-
 	// setup image importer
 	importer := newContainerdImporter(cmder)
 	if err := importer.Prepare(); err != nil {
@@ -308,19 +290,32 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		}
 	}()
 
-	// create a plan of image loading
-	loadFns := []func() error{}
-	for _, image := range pulled {
-		image := image // capture loop var
-		loadFns = append(loadFns, func() error {
-			f, err := os.Open(image)
-			if err != nil {
-				return err
+	fns := []func() error{}
+	for _, image := range requiredImages {
+		image := image // https://golang.org/doc/faq#closures_and_goroutines
+		fns = append(fns, func() error {
+			if !builtImages.Has(image) {
+				/*
+					TODO: show errors when we have real errors. See comments in
+					importer implementation
+					err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
+					if err != nil {
+						c.logger.Warnf("Failed to pull %s with error: %v", image, err)
+						runE := exec.RunErrorForError(err)
+						c.logger.Warn(string(runE.Output))
+					}
+				*/
+				_ = importer.Pull(image, dockerBuildOsAndArch(c.arch))
 			}
-			defer f.Close()
-			return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout).SetStdin(f).Run()
+			return nil
 		})
 	}
+	if err := errors.AggregateConcurrent(fns); err != nil {
+		return nil, err
+	}
+
+	// create a plan of image loading
+	loadFns := []func() error{}
 	for _, image := range bits.ImagePaths() {
 		image := image // capture loop var
 		loadFns = append(loadFns, func() error {
@@ -332,7 +327,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 			//return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run()
 			// we will rewrite / correct the tags as we load the image
 			if err := exec.RunWithStdinWriter(importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout), func(w io.Writer) error {
-				return docker.EditArchiveRepositories(f, w, fixRepository)
+				return docker.EditArchive(f, w, fixRepository, c.arch)
 			}); err != nil {
 				return err
 			}
@@ -352,7 +347,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 func (c *buildContext) createBuildContainer() (id string, err error) {
 	// attempt to explicitly pull the image if it doesn't exist locally
 	// we don't care if this errors, we'll still try to run which also pulls
-	_, _ = docker.PullIfNotPresent(c.logger, c.baseImage, 4)
+	_ = docker.Pull(c.logger, c.baseImage, dockerBuildOsAndArch(c.arch), 4)
 	// this should be good enough: a specific prefix, the current unix time,
 	// and a little random bits in case we have multiple builds simultaneously
 	random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
@@ -364,6 +359,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 			// the container should hang forever so we can exec in it
 			"--entrypoint=sleep",
 			"--name=" + id,
+			"--platform=" + dockerBuildOsAndArch(c.arch),
 		},
 		[]string{
 			"infinity", // sleep infinitely to keep the container around
@@ -374,3 +370,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 	}
 	return id, nil
 }
+
+func dockerBuildOsAndArch(arch string) string {
+	return "linux/" + arch
+}

pkg/build/nodeimage/const_cni.go

@@ -20,7 +20,7 @@ package nodeimage
 The default CNI manifest and images are our own tiny kindnet
 */
 
-var defaultCNIImages = []string{"kindest/kindnetd:v20210326-1e038dc5"}
+var defaultCNIImages = []string{"docker.io/kindest/kindnetd:v20210326-1e038dc5"}
 
 // TODO: migrate to fully patching and deprecate the template
 const defaultCNIManifest = `
@@ -94,7 +94,7 @@ spec:
       serviceAccountName: kindnet
       containers:
       - name: kindnet-cni
-        image: kindest/kindnetd:v20210326-1e038dc5
+        image: docker.io/kindest/kindnetd:v20210326-1e038dc5
         env:
         - name: HOST_IP
           valueFrom:

pkg/build/nodeimage/const_storage.go

@@ -25,7 +25,7 @@ NOTE: we have customized it in the following ways:
 - install as the default storage class
 */
 
-var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
+var defaultStorageImages = []string{"docker.io/rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
 
 const defaultStorageManifest = `
 # kind customized https://github.com/rancher/local-path-provisioner manifest
@@ -95,7 +95,7 @@ spec:
       serviceAccountName: local-path-provisioner-service-account
       containers:
       - name: local-path-provisioner
-        image: rancher/local-path-provisioner:v0.0.14
+        image: docker.io/rancher/local-path-provisioner:v0.0.14
         imagePullPolicy: IfNotPresent
         command:
         - local-path-provisioner

pkg/build/nodeimage/defaults.go

@@ -20,4 +20,4 @@ package nodeimage
 const DefaultImage = "kindest/node:latest"
 
 // DefaultBaseImage is the default base image used
-const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961@sha256:2b96d0b11c80e7cb1096ea89e2ffbe26ae72b2b08a177e088fa0edeff9fad516"
+const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961"

pkg/build/nodeimage/imageimporter.go

@@ -17,28 +17,21 @@ limitations under the License.
 package nodeimage
 
 import (
+	"io/ioutil"
+
 	"sigs.k8s.io/kind/pkg/exec"
 )
 
-type imageImporter interface {
-	Prepare() error
-	LoadCommand() exec.Cmd
-	ListImported() ([]string, error)
-	End() error
-}
-
 type containerdImporter struct {
 	containerCmder exec.Cmder
 }
 
-func newContainerdImporter(containerCmder exec.Cmder) imageImporter {
+func newContainerdImporter(containerCmder exec.Cmder) *containerdImporter {
 	return &containerdImporter{
 		containerCmder: containerCmder,
 	}
 }
 
-var _ imageImporter = &containerdImporter{}
-
 func (c *containerdImporter) Prepare() error {
 	if err := c.containerCmder.Command(
 		"bash", "-c", "nohup containerd > /dev/null 2>&1 &",
@@ -53,6 +46,14 @@ func (c *containerdImporter) End() error {
 	return c.containerCmder.Command("pkill", "containerd").Run()
 }
 
+func (c *containerdImporter) Pull(image, platform string) error {
+	// TODO: this should exist with a --no-unpack and some way to operate quietly
+	// without discarding output
+	return c.containerCmder.Command(
+		"ctr", "--namespace=k8s.io", "images", "pull", "--platform="+platform, image,
+	).SetStdout(ioutil.Discard).SetStderr(ioutil.Discard).Run()
+}
+
 func (c *containerdImporter) LoadCommand() exec.Cmd {
 	return c.containerCmder.Command(
 		// TODO: ideally we do not need this in the future. we have fixed at least one image

pkg/build/nodeimage/internal/container/docker/archive.go

@@ -87,7 +87,7 @@ func GetArchiveTags(path string) ([]string, error) {
 // https://github.com/moby/moby/blob/master/image/spec/v1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.2.md
-func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositories func(string) string) error {
+func EditArchive(reader io.Reader, writer io.Writer, editRepositories func(string) string, architectureOverride string) error {
 	tarReader := tar.NewReader(reader)
 	tarWriter := tar.NewWriter(writer)
 	// iterate all entries in the tarball
@@ -117,6 +117,15 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 				return err
 			}
 			hdr.Size = int64(len(b))
+			// edit image config when we find that
+		} else if strings.HasSuffix(hdr.Name, ".json") {
+			if architectureOverride != "" {
+				b, err = editConfigArchitecture(b, architectureOverride)
+				if err != nil {
+					return err
+				}
+				hdr.Size = int64(len(b))
+			}
 		}
 
 		// write to the output tarball
@@ -133,6 +142,19 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 
 /* helpers */
 
+func editConfigArchitecture(raw []byte, architectureOverride string) ([]byte, error) {
+	var cfg map[string]interface{}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, err
+	}
+	const architecture = "architecture"
+	if _, ok := cfg[architecture]; !ok {
+		return raw, nil
+	}
+	cfg[architecture] = architectureOverride
+	return json.Marshal(cfg)
+}
+
 // archiveRepositories represents repository:tag:ref
 //
 // https://github.com/moby/moby/blob/master/image/spec/v1.md