From 6c96945beb5201324f25313d62802594da3f49df Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Wed, 31 Mar 2021 22:08:36 -0700
Subject: [PATCH 1/9] move kube-root to args[0], deprecate flag

this will make more sense as we expand the accepted values and as GOPATH goes away in 1.17 and setting this essentially becomes required when we can't autodetect the kubernetes source checkout reliably
---
 pkg/cmd/kind/build/nodeimage/nodeimage.go | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/pkg/cmd/kind/build/nodeimage/nodeimage.go b/pkg/cmd/kind/build/nodeimage/nodeimage.go
index c77e608e61..223ee7223b 100644
--- a/pkg/cmd/kind/build/nodeimage/nodeimage.go
+++ b/pkg/cmd/kind/build/nodeimage/nodeimage.go
@@ -37,16 +37,22 @@ type flagpole struct {
 func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command {
 	flags := &flagpole{}
 	cmd := &cobra.Command{
-		Args: cobra.NoArgs,
+		Args: cobra.MaximumNArgs(1),
 		// TODO(bentheelder): more detailed usage
-		Use:   "node-image",
+		Use:   "node-image [kubernetes-source]",
 		Short: "Build the node image",
 		Long:  "Build the node image which contains Kubernetes build artifacts and other kind requirements",
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if cmd.Flags().Lookup("kube-root").Changed {
+				if len(args) != 0 {
+					return errors.New("passing an argument and deprecated --kube-root is not supported, please switch to just the argument")
+				}
+				logger.Warn("--kube-root is deprecated, please switch to passing this as an argument")
+			}
 			if cmd.Flags().Lookup("type").Changed {
 				return errors.New("--type is no longer supported, please remove this flag")
 			}
-			return runE(logger, flags)
+			return runE(logger, flags, args)
 		},
 	}
 	cmd.Flags().StringVar(
@@ -71,11 +77,15 @@ func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command {
 	return cmd
 }
 
-func runE(logger log.Logger, flags *flagpole) error {
+func runE(logger log.Logger, flags *flagpole, args []string) error {
+	kubeRoot := flags.KubeRoot
+	if len(args) > 0 {
+		kubeRoot = args[0]
+	}
 	if err := nodeimage.Build(
 		nodeimage.WithImage(flags.Image),
 		nodeimage.WithBaseImage(flags.BaseImage),
-		nodeimage.WithKuberoot(flags.KubeRoot),
+		nodeimage.WithKuberoot(kubeRoot),
 		nodeimage.WithLogger(logger),
 	); err != nil {
 		return errors.Wrap(err, "error building node image")

From 4eafefe33ec0d32b9f73ad563220681227f6d39b Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 2 Apr 2021 01:39:51 -0700
Subject: [PATCH 2/9] implement kind build node-image --arch

- switch to pulling images with containerd in the running base, instead of to the host. this breaks offline kubernetes development but the previous approach with docker pull / save / export cannot be used reliably for multi-arch
- fully qualify images to prepull (needed by ctr)
- fix image architecture in upstream built images, some of which are have the wrong metadata
- move architecture support to warning
---
 pkg/build/nodeimage/build.go                  | 18 +---
 .../{build_impl.go => buildcontext.go}        | 88 +++++++++----------
 pkg/build/nodeimage/const_cni.go              |  4 +-
 pkg/build/nodeimage/const_storage.go          |  4 +-
 pkg/build/nodeimage/imageimporter.go          | 17 ++--
 .../internal/container/docker/archive.go      | 24 ++++-
 .../internal/container/docker/pull.go         | 24 +----
 .../nodeimage/internal/kube/builder_docker.go |  4 +-
 pkg/build/nodeimage/options.go                | 10 +++
 pkg/cmd/kind/build/nodeimage/nodeimage.go     |  7 ++
 10 files changed, 101 insertions(+), 99 deletions(-)
 rename pkg/build/nodeimage/{build_impl.go => buildcontext.go} (90%)

diff --git a/pkg/build/nodeimage/build.go b/pkg/build/nodeimage/build.go
index 67e9146ffc..d643cad624 100644
--- a/pkg/build/nodeimage/build.go
+++ b/pkg/build/nodeimage/build.go
@@ -31,8 +31,7 @@ func Build(options ...Option) error {
 		image:     DefaultImage,
 		baseImage: DefaultBaseImage,
 		logger:    log.NoopLogger{},
-		// TODO: only host arch supported. changing this will be tricky
-		arch: runtime.GOARCH,
+		arch:      runtime.GOARCH,
 	}
 
 	// apply user options
@@ -44,7 +43,7 @@ func Build(options ...Option) error {
 
 	// verify that we're using a supported arch
 	if !supportedArch(ctx.arch) {
-		return errors.Errorf("unsupported architecture %q", ctx.arch)
+		ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
 	}
 
 	// locate sources if no kubernetes source was specified
@@ -78,16 +77,3 @@ func supportedArch(arch string) bool {
 	}
 	return true
 }
-
-// buildContext is used to build the kind node image, and contains
-// build configuration
-type buildContext struct {
-	// option fields
-	image     string
-	baseImage string
-	logger    log.Logger
-	// non-option fields
-	arch     string // TODO(bentheelder): this should be an option
-	kubeRoot string
-	builder  kube.Builder
-}
diff --git a/pkg/build/nodeimage/build_impl.go b/pkg/build/nodeimage/buildcontext.go
similarity index 90%
rename from pkg/build/nodeimage/build_impl.go
rename to pkg/build/nodeimage/buildcontext.go
index 89cc24a87c..c7c636fbfd 100644
--- a/pkg/build/nodeimage/build_impl.go
+++ b/pkg/build/nodeimage/buildcontext.go
@@ -33,8 +33,22 @@ import (
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/fs"
+	"sigs.k8s.io/kind/pkg/log"
 )
 
+// buildContext is used to build the kind node image, and contains
+// build configuration
+type buildContext struct {
+	// option fields
+	image     string
+	baseImage string
+	logger    log.Logger
+	arch      string
+	kubeRoot  string
+	// non-option fields
+	builder kube.Builder
+}
+
 // Build builds the cluster node image, the sourcedir must be set on
 // the buildContext
 func (c *buildContext) Build() (err error) {
@@ -188,7 +202,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 	fixRepository := func(repository string) string {
 		if strings.HasSuffix(repository, archSuffix) {
 			fixed := strings.TrimSuffix(repository, archSuffix)
-			fmt.Println("fixed: " + repository + " -> " + fixed)
+			c.logger.V(1).Info("fixed: " + repository + " -> " + fixed)
 			repository = fixed
 		}
 		return repository
@@ -206,7 +220,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		fixedImages.Insert(registry + ":" + tag)
 	}
 	builtImages = fixedImages
-	c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
+	c.logger.V(1).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
 
 	// gets the list of images required by kubeadm
 	requiredImages, err := exec.OutputLines(cmder.Command(
@@ -262,38 +276,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		return nil, errors.Wrap(err, "failed to make images dir")
 	}
 
-	fns := []func() error{}
-	pulledImages := make(chan string, len(requiredImages))
-	for i, image := range requiredImages {
-		i, image := i, image // https://golang.org/doc/faq#closures_and_goroutines
-		fns = append(fns, func() error {
-			if !builtImages.Has(image) {
-				fmt.Printf("Pulling: %s\n", image)
-				err := docker.Pull(c.logger, image, 2)
-				if err != nil {
-					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
-				}
-				// TODO(bentheelder): generate a friendlier name
-				pullName := fmt.Sprintf("%d.tar", i)
-				pullTo := path.Join(imagesDir, pullName)
-				err = docker.Save(image, pullTo)
-				if err != nil {
-					return err
-				}
-				pulledImages <- pullTo
-			}
-			return nil
-		})
-	}
-	if err := errors.AggregateConcurrent(fns); err != nil {
-		return nil, err
-	}
-	close(pulledImages)
-	pulled := []string{}
-	for image := range pulledImages {
-		pulled = append(pulled, image)
-	}
-
 	// setup image importer
 	importer := newContainerdImporter(cmder)
 	if err := importer.Prepare(); err != nil {
@@ -308,19 +290,28 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		}
 	}()
 
-	// create a plan of image loading
-	loadFns := []func() error{}
-	for _, image := range pulled {
-		image := image // capture loop var
-		loadFns = append(loadFns, func() error {
-			f, err := os.Open(image)
-			if err != nil {
-				return err
+	fns := []func() error{}
+	for _, image := range requiredImages {
+		image := image // https://golang.org/doc/faq#closures_and_goroutines
+		fns = append(fns, func() error {
+			if !builtImages.Has(image) {
+				err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
+				if err != nil {
+					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
+					runE := exec.RunErrorForError(err)
+					c.logger.Warn(string(runE.Output))
+				}
+				// TODO(bentheelder): generate a friendlier name
 			}
-			defer f.Close()
-			return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout).SetStdin(f).Run()
+			return nil
 		})
 	}
+	if err := errors.AggregateConcurrent(fns); err != nil {
+		return nil, err
+	}
+
+	// create a plan of image loading
+	loadFns := []func() error{}
 	for _, image := range bits.ImagePaths() {
 		image := image // capture loop var
 		loadFns = append(loadFns, func() error {
@@ -332,7 +323,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 			//return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run()
 			// we will rewrite / correct the tags as we load the image
 			if err := exec.RunWithStdinWriter(importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout), func(w io.Writer) error {
-				return docker.EditArchiveRepositories(f, w, fixRepository)
+				return docker.EditArchive(f, w, fixRepository, c.arch)
 			}); err != nil {
 				return err
 			}
@@ -352,7 +343,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 func (c *buildContext) createBuildContainer() (id string, err error) {
 	// attempt to explicitly pull the image if it doesn't exist locally
 	// we don't care if this errors, we'll still try to run which also pulls
-	_, _ = docker.PullIfNotPresent(c.logger, c.baseImage, 4)
+	_ = docker.Pull(c.logger, c.baseImage, dockerBuildOsAndArch(c.arch), 4)
 	// this should be good enough: a specific prefix, the current unix time,
 	// and a little random bits in case we have multiple builds simultaneously
 	random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
@@ -364,6 +355,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 			// the container should hang forever so we can exec in it
 			"--entrypoint=sleep",
 			"--name=" + id,
+			"--platform=" + dockerBuildOsAndArch(c.arch),
 		},
 		[]string{
 			"infinity", // sleep infinitely to keep the container around
@@ -374,3 +366,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
 	}
 	return id, nil
 }
+
+func dockerBuildOsAndArch(arch string) string {
+	return "linux/" + arch
+}
diff --git a/pkg/build/nodeimage/const_cni.go b/pkg/build/nodeimage/const_cni.go
index 0e36489bb3..2afd7e1425 100644
--- a/pkg/build/nodeimage/const_cni.go
+++ b/pkg/build/nodeimage/const_cni.go
@@ -20,7 +20,7 @@ package nodeimage
 The default CNI manifest and images are our own tiny kindnet
 */
 
-var defaultCNIImages = []string{"kindest/kindnetd:v20210326-1e038dc5"}
+var defaultCNIImages = []string{"docker.io/kindest/kindnetd:v20210326-1e038dc5"}
 
 // TODO: migrate to fully patching and deprecate the template
 const defaultCNIManifest = `
@@ -94,7 +94,7 @@ spec:
       serviceAccountName: kindnet
       containers:
       - name: kindnet-cni
-        image: kindest/kindnetd:v20210326-1e038dc5
+        image: docker.io/kindest/kindnetd:v20210326-1e038dc5
         env:
         - name: HOST_IP
           valueFrom:
diff --git a/pkg/build/nodeimage/const_storage.go b/pkg/build/nodeimage/const_storage.go
index 50f096310f..aede99a83a 100644
--- a/pkg/build/nodeimage/const_storage.go
+++ b/pkg/build/nodeimage/const_storage.go
@@ -25,7 +25,7 @@ NOTE: we have customized it in the following ways:
 - install as the default storage class
 */
 
-var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
+var defaultStorageImages = []string{"docker.io/rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
 
 const defaultStorageManifest = `
 # kind customized https://github.com/rancher/local-path-provisioner manifest
@@ -95,7 +95,7 @@ spec:
       serviceAccountName: local-path-provisioner-service-account
       containers:
       - name: local-path-provisioner
-        image: rancher/local-path-provisioner:v0.0.14
+        image: docker.io/rancher/local-path-provisioner:v0.0.14
         imagePullPolicy: IfNotPresent
         command:
         - local-path-provisioner
diff --git a/pkg/build/nodeimage/imageimporter.go b/pkg/build/nodeimage/imageimporter.go
index 1d0b31060a..c961711208 100644
--- a/pkg/build/nodeimage/imageimporter.go
+++ b/pkg/build/nodeimage/imageimporter.go
@@ -20,25 +20,16 @@ import (
 	"sigs.k8s.io/kind/pkg/exec"
 )
 
-type imageImporter interface {
-	Prepare() error
-	LoadCommand() exec.Cmd
-	ListImported() ([]string, error)
-	End() error
-}
-
 type containerdImporter struct {
 	containerCmder exec.Cmder
 }
 
-func newContainerdImporter(containerCmder exec.Cmder) imageImporter {
+func newContainerdImporter(containerCmder exec.Cmder) *containerdImporter {
 	return &containerdImporter{
 		containerCmder: containerCmder,
 	}
 }
 
-var _ imageImporter = &containerdImporter{}
-
 func (c *containerdImporter) Prepare() error {
 	if err := c.containerCmder.Command(
 		"bash", "-c", "nohup containerd > /dev/null 2>&1 &",
@@ -53,6 +44,12 @@ func (c *containerdImporter) End() error {
 	return c.containerCmder.Command("pkill", "containerd").Run()
 }
 
+func (c *containerdImporter) Pull(image, platform string) error {
+	return c.containerCmder.Command(
+		"ctr", "--namespace=k8s.io", "images", "pull", "--platform="+platform, image,
+	).Run()
+}
+
 func (c *containerdImporter) LoadCommand() exec.Cmd {
 	return c.containerCmder.Command(
 		// TODO: ideally we do not need this in the future. we have fixed at least one image
diff --git a/pkg/build/nodeimage/internal/container/docker/archive.go b/pkg/build/nodeimage/internal/container/docker/archive.go
index 49f9511244..73277896d5 100644
--- a/pkg/build/nodeimage/internal/container/docker/archive.go
+++ b/pkg/build/nodeimage/internal/container/docker/archive.go
@@ -87,7 +87,7 @@ func GetArchiveTags(path string) ([]string, error) {
 // https://github.com/moby/moby/blob/master/image/spec/v1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.1.md
 // https://github.com/moby/moby/blob/master/image/spec/v1.2.md
-func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositories func(string) string) error {
+func EditArchive(reader io.Reader, writer io.Writer, editRepositories func(string) string, architectureOverride string) error {
 	tarReader := tar.NewReader(reader)
 	tarWriter := tar.NewWriter(writer)
 	// iterate all entries in the tarball
@@ -117,6 +117,15 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 				return err
 			}
 			hdr.Size = int64(len(b))
+			// edit image config when we find that
+		} else if strings.HasSuffix(hdr.Name, ".json") {
+			if architectureOverride != "" {
+				b, err = editConfigArchitecture(b, architectureOverride)
+				if err != nil {
+					return err
+				}
+				hdr.Size = int64(len(b))
+			}
 		}
 
 		// write to the output tarball
@@ -133,6 +142,19 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
 
 /* helpers */
 
+func editConfigArchitecture(raw []byte, architectureOverride string) ([]byte, error) {
+	var cfg map[string]interface{}
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return nil, err
+	}
+	const architecture = "architecture"
+	if _, ok := cfg[architecture]; !ok {
+		return raw, nil
+	}
+	cfg[architecture] = architectureOverride
+	return json.Marshal(cfg)
+}
+
 // archiveRepositories represents repository:tag:ref
 //
 // https://github.com/moby/moby/blob/master/image/spec/v1.md
diff --git a/pkg/build/nodeimage/internal/container/docker/pull.go b/pkg/build/nodeimage/internal/container/docker/pull.go
index 6a556db045..0ac49de6b4 100644
--- a/pkg/build/nodeimage/internal/container/docker/pull.go
+++ b/pkg/build/nodeimage/internal/container/docker/pull.go
@@ -23,33 +23,17 @@ import (
 	"sigs.k8s.io/kind/pkg/log"
 )
 
-// PullIfNotPresent will pull an image if it is not present locally
-// retrying up to retries times
-// it returns true if it attempted to pull, and any errors from pulling
-func PullIfNotPresent(logger log.Logger, image string, retries int) (pulled bool, err error) {
-	// TODO(bentheelder): switch most (all) of the logging here to debug level
-	// once we have configurable log levels
-	// if this did not return an error, then the image exists locally
-	cmd := exec.Command("docker", "inspect", "--type=image", image)
-	if err := cmd.Run(); err == nil {
-		logger.V(1).Infof("Image: %s present locally", image)
-		return false, nil
-	}
-	// otherwise try to pull it
-	return true, Pull(logger, image, retries)
-}
-
 // Pull pulls an image, retrying up to retries times
-func Pull(logger log.Logger, image string, retries int) error {
-	logger.V(1).Infof("Pulling image: %s ...", image)
-	err := exec.Command("docker", "pull", image).Run()
+func Pull(logger log.Logger, image string, platform string, retries int) error {
+	logger.V(1).Infof("Pulling image: %s for platform %s ...", image, platform)
+	err := exec.Command("docker", "pull", "--platform="+platform, image).Run()
 	// retry pulling up to retries times if necessary
 	if err != nil {
 		for i := 0; i < retries; i++ {
 			time.Sleep(time.Second * time.Duration(i+1))
 			logger.V(1).Infof("Trying again to pull image: %q ... %v", image, err)
 			// TODO(bentheelder): add some backoff / sleep?
-			err = exec.Command("docker", "pull", image).Run()
+			err = exec.Command("docker", "pull", "--platform="+platform, image).Run()
 			if err == nil {
 				break
 			}
diff --git a/pkg/build/nodeimage/internal/kube/builder_docker.go b/pkg/build/nodeimage/internal/kube/builder_docker.go
index 4cbb03a2b5..6eace3a230 100644
--- a/pkg/build/nodeimage/internal/kube/builder_docker.go
+++ b/pkg/build/nodeimage/internal/kube/builder_docker.go
@@ -21,11 +21,11 @@ import (
 	"path/filepath"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/util/version"
+
 	"sigs.k8s.io/kind/pkg/errors"
 	"sigs.k8s.io/kind/pkg/exec"
 	"sigs.k8s.io/kind/pkg/log"
-
-	"k8s.io/apimachinery/pkg/util/version"
 )
 
 // TODO(bentheelder): plumb through arch
diff --git a/pkg/build/nodeimage/options.go b/pkg/build/nodeimage/options.go
index e108bd6661..88906e25a5 100644
--- a/pkg/build/nodeimage/options.go
+++ b/pkg/build/nodeimage/options.go
@@ -62,3 +62,13 @@ func WithLogger(logger log.Logger) Option {
 		return nil
 	})
 }
+
+// WithArch sets the architecture to build for
+func WithArch(arch string) Option {
+	return optionAdapter(func(b *buildContext) error {
+		if arch != "" {
+			b.arch = arch
+		}
+		return nil
+	})
+}
diff --git a/pkg/cmd/kind/build/nodeimage/nodeimage.go b/pkg/cmd/kind/build/nodeimage/nodeimage.go
index 223ee7223b..8d62fa0722 100644
--- a/pkg/cmd/kind/build/nodeimage/nodeimage.go
+++ b/pkg/cmd/kind/build/nodeimage/nodeimage.go
@@ -31,6 +31,7 @@ type flagpole struct {
 	Image     string
 	BaseImage string
 	KubeRoot  string
+	Arch      string
 }
 
 // NewCommand returns a new cobra.Command for building the node image
@@ -74,6 +75,11 @@ func NewCommand(logger log.Logger, streams cmd.IOStreams) *cobra.Command {
 		nodeimage.DefaultBaseImage,
 		"name:tag of the base image to use for the build",
 	)
+	cmd.Flags().StringVar(
+		&flags.Arch, "arch",
+		"",
+		"architecture to build for, defaults to the host architecture",
+	)
 	return cmd
 }
 
@@ -87,6 +93,7 @@ func runE(logger log.Logger, flags *flagpole, args []string) error {
 		nodeimage.WithBaseImage(flags.BaseImage),
 		nodeimage.WithKuberoot(kubeRoot),
 		nodeimage.WithLogger(logger),
+		nodeimage.WithArch(flags.Arch),
 	); err != nil {
 		return errors.Wrap(err, "error building node image")
 	}

From 207a570fb88fbd86283b18c2e3016a29118e4b4a Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Thu, 29 Apr 2021 15:59:45 -0700
Subject: [PATCH 3/9] quiet output

---
 pkg/build/nodeimage/buildcontext.go  | 18 +++++++++++-------
 pkg/build/nodeimage/imageimporter.go |  6 +++++-
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/pkg/build/nodeimage/buildcontext.go b/pkg/build/nodeimage/buildcontext.go
index c7c636fbfd..5b409df0ce 100644
--- a/pkg/build/nodeimage/buildcontext.go
+++ b/pkg/build/nodeimage/buildcontext.go
@@ -295,13 +295,17 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
 		image := image // https://golang.org/doc/faq#closures_and_goroutines
 		fns = append(fns, func() error {
 			if !builtImages.Has(image) {
-				err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
-				if err != nil {
-					c.logger.Warnf("Failed to pull %s with error: %v", image, err)
-					runE := exec.RunErrorForError(err)
-					c.logger.Warn(string(runE.Output))
-				}
-				// TODO(bentheelder): generate a friendlier name
+				/*
+					TODO: show errors when we have real errors. See comments in
+					importer implementation
+					err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
+					if err != nil {
+						c.logger.Warnf("Failed to pull %s with error: %v", image, err)
+						runE := exec.RunErrorForError(err)
+						c.logger.Warn(string(runE.Output))
+					}
+				*/
+				_ = importer.Pull(image, dockerBuildOsAndArch(c.arch))
 			}
 			return nil
 		})
diff --git a/pkg/build/nodeimage/imageimporter.go b/pkg/build/nodeimage/imageimporter.go
index c961711208..c6744da7e9 100644
--- a/pkg/build/nodeimage/imageimporter.go
+++ b/pkg/build/nodeimage/imageimporter.go
@@ -17,6 +17,8 @@ limitations under the License.
 package nodeimage
 
 import (
+	"io/ioutil"
+
 	"sigs.k8s.io/kind/pkg/exec"
 )
 
@@ -45,9 +47,11 @@ func (c *containerdImporter) End() error {
 }
 
 func (c *containerdImporter) Pull(image, platform string) error {
+	// TODO: this should exist with a --no-unpack and some way to operate quietly
+	// without discarding output
 	return c.containerCmder.Command(
 		"ctr", "--namespace=k8s.io", "images", "pull", "--platform="+platform, image,
-	).Run()
+	).SetStdout(ioutil.Discard).SetStderr(ioutil.Discard).Run()
 }
 
 func (c *containerdImporter) LoadCommand() exec.Cmd {

From 1b57dbebdc6a71dc204bb4e1e70c712b96a7a556 Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 02:34:11 -0700
Subject: [PATCH 4/9] push multi-arch images with push-node.sh

---
 hack/release/build/push-node.sh | 37 ++++++++++++++++++++++++++-------
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/hack/release/build/push-node.sh b/hack/release/build/push-node.sh
index ddaa9d41db..a1a611c2cb 100755
--- a/hack/release/build/push-node.sh
+++ b/hack/release/build/push-node.sh
@@ -24,7 +24,6 @@ make build
 
 # generate tag
 DATE="$(date +v%Y%m%d)"
-TAG="${DATE}-$(git describe --always --dirty)"
 
 # build
 KUBEROOT="${KUBEROOT:-${GOPATH}/src/k8s.io/kubernetes}"
@@ -34,13 +33,35 @@ if [ -z "${GOFLAGS}" ]; then
     # TODO: add dockerless when 1.19 or greater
     GOFLAGS="-tags=providerless"
 fi
+
+# NOTE: adding platforms is costly in terms of build time
+# we will consider expanding this in the future, for now the aim is to prove
+# multi-arch and enable developers working on commonly available hardware
+# Other users are free to build their own images on additional platforms using
+# their own time and resources. Please see our docs.
+ARCHES="${ARCHES:-amd64 arm64}"
+__arches__=(${ARCHES})
+
 set -x
-"${REPO_ROOT}/bin/kind" build node-image --image="kindest/node:${TAG}" --kube-root="${KUBEROOT}"
+# get kubernetes version
+version_line="$(cd "${KUBEROOT}"; ./hack/print-workspace-status.sh | grep 'gitVersion')"
+kube_version="${version_line#"gitVersion "}"
 
-# re-tag with kubernetes version
-IMG="kindest/node:${TAG}"
-KUBE_VERSION="$(docker run --rm --entrypoint=cat "${IMG}" /kind/version)"
-docker tag "${IMG}" "kindest/node:${KUBE_VERSION}"
+# build for each arch
+image="kindest/node:${kube_version}"
+images=()
+for arch in "${__arches__[@]}"; do
+    image="kindest/node-${arch}:${kube_version}"
+    "${REPO_ROOT}/bin/kind" build node-image --image="${image}" --kube-root="${KUBEROOT}" --arch="${arch}"
+    images+=("${image}")
+done
 
-# push
-docker push kindest/node:"${KUBE_VERSION}"
+# combine to manifest list tagged with kubernetes version
+export DOCKER_CLI_EXPERIMENTAL=enabled
+# images must be pushed to be referenced by docker manifest
+# we push only after all builds have succeeded
+for image in "${images[@]}"; do
+    docker push "${image}"
+done
+docker manifest create "${image}" "${images[@]}"
+docker manifest push "${image}"

From da28a7d3f45ec3b68f6c04296b72e9426fee4924 Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 02:54:56 -0700
Subject: [PATCH 5/9] drop base image digest

---
 pkg/build/nodeimage/defaults.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/build/nodeimage/defaults.go b/pkg/build/nodeimage/defaults.go
index c7269b6be8..2cb21ed477 100644
--- a/pkg/build/nodeimage/defaults.go
+++ b/pkg/build/nodeimage/defaults.go
@@ -20,4 +20,4 @@ package nodeimage
 const DefaultImage = "kindest/node:latest"
 
 // DefaultBaseImage is the default base image used
-const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961@sha256:2b96d0b11c80e7cb1096ea89e2ffbe26ae72b2b08a177e088fa0edeff9fad516"
+const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961"

From 2d5eb98a79e15e3113adbca9a95e4424817b7279 Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 03:07:59 -0700
Subject: [PATCH 6/9] fix image pushing

---
 hack/release/build/push-node.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/hack/release/build/push-node.sh b/hack/release/build/push-node.sh
index a1a611c2cb..5d857de39d 100755
--- a/hack/release/build/push-node.sh
+++ b/hack/release/build/push-node.sh
@@ -48,7 +48,7 @@ version_line="$(cd "${KUBEROOT}"; ./hack/print-workspace-status.sh | grep 'gitVe
 kube_version="${version_line#"gitVersion "}"
 
 # build for each arch
-image="kindest/node:${kube_version}"
+IMAGE="kindest/node:${kube_version}"
 images=()
 for arch in "${__arches__[@]}"; do
     image="kindest/node-${arch}:${kube_version}"
@@ -63,5 +63,5 @@ export DOCKER_CLI_EXPERIMENTAL=enabled
 for image in "${images[@]}"; do
     docker push "${image}"
 done
-docker manifest create "${image}" "${images[@]}"
-docker manifest push "${image}"
+docker manifest create "${IMAGE}" "${images[@]}"
+docker manifest push "${IMAGE}"

From 229e50907f0e0ef3318e1a6bc5b2a6d1ba5de12b Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 03:18:09 -0700
Subject: [PATCH 7/9] bump node image

---
 pkg/apis/config/defaults/image.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkg/apis/config/defaults/image.go b/pkg/apis/config/defaults/image.go
index f811f36081..104ea88cdc 100644
--- a/pkg/apis/config/defaults/image.go
+++ b/pkg/apis/config/defaults/image.go
@@ -18,4 +18,4 @@ limitations under the License.
 package defaults
 
 // Image is the default for the Config.Image field, aka the default node image.
-const Image = "kindest/node:v1.20.2@sha256:15d3b5c4f521a84896ed1ead1b14e4774d02202d5c65ab68f30eeaf310a3b1a7"
+const Image = "kindest/node:v1.21.1@sha256:c6eead46eaba71017e290f696fa675187133d7953e9291900e384f711b6cf8ed"

From c2af3b4c2349ec7fd8bec2ae28e9cdc571b221fc Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 03:20:23 -0700
Subject: [PATCH 8/9] fix shell lints

---
 hack/release/build/push-node.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/hack/release/build/push-node.sh b/hack/release/build/push-node.sh
index 5d857de39d..4ee8e366a6 100755
--- a/hack/release/build/push-node.sh
+++ b/hack/release/build/push-node.sh
@@ -22,12 +22,10 @@ cd "${REPO_ROOT}"
 # ensure we have up to date kind
 make build
 
-# generate tag
-DATE="$(date +v%Y%m%d)"
-
-# build
+# path to kubernetes sources
 KUBEROOT="${KUBEROOT:-${GOPATH}/src/k8s.io/kubernetes}"
 
+# kubernetes build option(s)
 GOFLAGS="${GOFLAGS:-}"
 if [ -z "${GOFLAGS}" ]; then
     # TODO: add dockerless when 1.19 or greater
@@ -40,7 +38,7 @@ fi
 # Other users are free to build their own images on additional platforms using
 # their own time and resources. Please see our docs.
 ARCHES="${ARCHES:-amd64 arm64}"
-__arches__=(${ARCHES})
+IFS=" " read -r -a __arches__ <<< "$ARCHES"
 
 set -x
 # get kubernetes version

From 0a1d35d2e160ca44845d9e431ca752a637fcc6a0 Mon Sep 17 00:00:00 2001
From: Benjamin Elder <bentheelder@google.com>
Date: Fri, 14 May 2021 03:28:26 -0700
Subject: [PATCH 9/9] improve push-build

---
 hack/release/build/push-node.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/hack/release/build/push-node.sh b/hack/release/build/push-node.sh
index 4ee8e366a6..4d1033bde6 100755
--- a/hack/release/build/push-node.sh
+++ b/hack/release/build/push-node.sh
@@ -25,6 +25,9 @@ make build
 # path to kubernetes sources
 KUBEROOT="${KUBEROOT:-${GOPATH}/src/k8s.io/kubernetes}"
 
+# ensure we have qemu setup (de-duped logic with setting up buildx for multi-arch)
+"${REPO_ROOT}/hack/build/init-buildx.sh"
+
 # kubernetes build option(s)
 GOFLAGS="${GOFLAGS:-}"
 if [ -z "${GOFLAGS}" ]; then
@@ -50,7 +53,7 @@ IMAGE="kindest/node:${kube_version}"
 images=()
 for arch in "${__arches__[@]}"; do
     image="kindest/node-${arch}:${kube_version}"
-    "${REPO_ROOT}/bin/kind" build node-image --image="${image}" --kube-root="${KUBEROOT}" --arch="${arch}"
+    "${REPO_ROOT}/bin/kind" build node-image --image="${image}" --arch="${arch}" "${KUBEROOT}"
     images+=("${image}")
 done