Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

support cross-compiling node-image #2176

Merged
merged 9 commits into from
May 14, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 34 additions & 12 deletions hack/release/build/push-node.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,47 @@ cd "${REPO_ROOT}"
# ensure we have up to date kind
make build

# generate tag
DATE="$(date +v%Y%m%d)"
TAG="${DATE}-$(git describe --always --dirty)"

# build
# path to kubernetes sources
KUBEROOT="${KUBEROOT:-${GOPATH}/src/k8s.io/kubernetes}"

# ensure we have qemu setup (de-duped logic with setting up buildx for multi-arch)
"${REPO_ROOT}/hack/build/init-buildx.sh"

# kubernetes build option(s)
GOFLAGS="${GOFLAGS:-}"
if [ -z "${GOFLAGS}" ]; then
# TODO: add dockerless when 1.19 or greater
GOFLAGS="-tags=providerless"
fi

# NOTE: adding platforms is costly in terms of build time
# we will consider expanding this in the future, for now the aim is to prove
# multi-arch and enable developers working on commonly available hardware
# Other users are free to build their own images on additional platforms using
# their own time and resources. Please see our docs.
ARCHES="${ARCHES:-amd64 arm64}"
IFS=" " read -r -a __arches__ <<< "$ARCHES"

set -x
"${REPO_ROOT}/bin/kind" build node-image --image="kindest/node:${TAG}" --kube-root="${KUBEROOT}"
# get kubernetes version
version_line="$(cd "${KUBEROOT}"; ./hack/print-workspace-status.sh | grep 'gitVersion')"
kube_version="${version_line#"gitVersion "}"

# re-tag with kubernetes version
IMG="kindest/node:${TAG}"
KUBE_VERSION="$(docker run --rm --entrypoint=cat "${IMG}" /kind/version)"
docker tag "${IMG}" "kindest/node:${KUBE_VERSION}"
# build for each arch
IMAGE="kindest/node:${kube_version}"
images=()
for arch in "${__arches__[@]}"; do
image="kindest/node-${arch}:${kube_version}"
"${REPO_ROOT}/bin/kind" build node-image --image="${image}" --arch="${arch}" "${KUBEROOT}"
images+=("${image}")
done

# push
docker push kindest/node:"${KUBE_VERSION}"
# combine to manifest list tagged with kubernetes version
export DOCKER_CLI_EXPERIMENTAL=enabled
# images must be pushed to be referenced by docker manifest
# we push only after all builds have succeeded
for image in "${images[@]}"; do
docker push "${image}"
done
docker manifest create "${IMAGE}" "${images[@]}"
docker manifest push "${IMAGE}"
2 changes: 1 addition & 1 deletion pkg/apis/config/defaults/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ limitations under the License.
package defaults

// Image is the default for the Config.Image field, aka the default node image.
const Image = "kindest/node:v1.20.2@sha256:15d3b5c4f521a84896ed1ead1b14e4774d02202d5c65ab68f30eeaf310a3b1a7"
const Image = "kindest/node:v1.21.1@sha256:c6eead46eaba71017e290f696fa675187133d7953e9291900e384f711b6cf8ed"
18 changes: 2 additions & 16 deletions pkg/build/nodeimage/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,7 @@ func Build(options ...Option) error {
image: DefaultImage,
baseImage: DefaultBaseImage,
logger: log.NoopLogger{},
// TODO: only host arch supported. changing this will be tricky
arch: runtime.GOARCH,
arch: runtime.GOARCH,
}

// apply user options
Expand All @@ -44,7 +43,7 @@ func Build(options ...Option) error {

// verify that we're using a supported arch
if !supportedArch(ctx.arch) {
return errors.Errorf("unsupported architecture %q", ctx.arch)
ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
}

// locate sources if no kubernetes source was specified
Expand Down Expand Up @@ -78,16 +77,3 @@ func supportedArch(arch string) bool {
}
return true
}

// buildContext is used to build the kind node image, and contains
// build configuration
type buildContext struct {
// option fields
image string
baseImage string
logger log.Logger
// non-option fields
arch string // TODO(bentheelder): this should be an option
kubeRoot string
builder kube.Builder
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,22 @@ import (
"sigs.k8s.io/kind/pkg/errors"
"sigs.k8s.io/kind/pkg/exec"
"sigs.k8s.io/kind/pkg/fs"
"sigs.k8s.io/kind/pkg/log"
)

// buildContext is used to build the kind node image, and contains
// build configuration
type buildContext struct {
// option fields
image string
baseImage string
logger log.Logger
arch string
kubeRoot string
// non-option fields
builder kube.Builder
}

// Build builds the cluster node image, the sourcedir must be set on
// the buildContext
func (c *buildContext) Build() (err error) {
Expand Down Expand Up @@ -188,7 +202,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
fixRepository := func(repository string) string {
if strings.HasSuffix(repository, archSuffix) {
fixed := strings.TrimSuffix(repository, archSuffix)
fmt.Println("fixed: " + repository + " -> " + fixed)
c.logger.V(1).Info("fixed: " + repository + " -> " + fixed)
repository = fixed
}
return repository
Expand All @@ -206,7 +220,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
fixedImages.Insert(registry + ":" + tag)
}
builtImages = fixedImages
c.logger.V(0).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))
c.logger.V(1).Info("Detected built images: " + strings.Join(builtImages.List(), ", "))

// gets the list of images required by kubeadm
requiredImages, err := exec.OutputLines(cmder.Command(
Expand Down Expand Up @@ -262,38 +276,6 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
return nil, errors.Wrap(err, "failed to make images dir")
}

fns := []func() error{}
pulledImages := make(chan string, len(requiredImages))
for i, image := range requiredImages {
i, image := i, image // https://golang.org/doc/faq#closures_and_goroutines
fns = append(fns, func() error {
if !builtImages.Has(image) {
fmt.Printf("Pulling: %s\n", image)
err := docker.Pull(c.logger, image, 2)
if err != nil {
c.logger.Warnf("Failed to pull %s with error: %v", image, err)
}
// TODO(bentheelder): generate a friendlier name
pullName := fmt.Sprintf("%d.tar", i)
pullTo := path.Join(imagesDir, pullName)
err = docker.Save(image, pullTo)
if err != nil {
return err
}
pulledImages <- pullTo
}
return nil
})
}
if err := errors.AggregateConcurrent(fns); err != nil {
return nil, err
}
close(pulledImages)
pulled := []string{}
for image := range pulledImages {
pulled = append(pulled, image)
}

// setup image importer
importer := newContainerdImporter(cmder)
if err := importer.Prepare(); err != nil {
Expand All @@ -308,19 +290,32 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
}
}()

// create a plan of image loading
loadFns := []func() error{}
for _, image := range pulled {
image := image // capture loop var
loadFns = append(loadFns, func() error {
f, err := os.Open(image)
if err != nil {
return err
fns := []func() error{}
for _, image := range requiredImages {
image := image // https://golang.org/doc/faq#closures_and_goroutines
fns = append(fns, func() error {
if !builtImages.Has(image) {
/*
TODO: show errors when we have real errors. See comments in
importer implementation
err := importer.Pull(image, dockerBuildOsAndArch(c.arch))
if err != nil {
c.logger.Warnf("Failed to pull %s with error: %v", image, err)
runE := exec.RunErrorForError(err)
c.logger.Warn(string(runE.Output))
}
*/
_ = importer.Pull(image, dockerBuildOsAndArch(c.arch))
}
defer f.Close()
return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout).SetStdin(f).Run()
return nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this always returns nil

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes? Because the contract is a method that returns an error type but we are only running one thing and it will error and that's fine. We check later that the images are loaded.

Ctr pull does not have the --no-unpack option unlike the option I added to import.

})
}
if err := errors.AggregateConcurrent(fns); err != nil {
return nil, err
}

// create a plan of image loading
loadFns := []func() error{}
for _, image := range bits.ImagePaths() {
image := image // capture loop var
loadFns = append(loadFns, func() error {
Expand All @@ -332,7 +327,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
//return importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run()
// we will rewrite / correct the tags as we load the image
if err := exec.RunWithStdinWriter(importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stdout), func(w io.Writer) error {
return docker.EditArchiveRepositories(f, w, fixRepository)
return docker.EditArchive(f, w, fixRepository, c.arch)
}); err != nil {
return err
}
Expand All @@ -352,7 +347,7 @@ func (c *buildContext) prePullImages(bits kube.Bits, dir, containerID string) ([
func (c *buildContext) createBuildContainer() (id string, err error) {
// attempt to explicitly pull the image if it doesn't exist locally
// we don't care if this errors, we'll still try to run which also pulls
_, _ = docker.PullIfNotPresent(c.logger, c.baseImage, 4)
_ = docker.Pull(c.logger, c.baseImage, dockerBuildOsAndArch(c.arch), 4)
// this should be good enough: a specific prefix, the current unix time,
// and a little random bits in case we have multiple builds simultaneously
random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
Expand All @@ -364,6 +359,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
// the container should hang forever so we can exec in it
"--entrypoint=sleep",
"--name=" + id,
"--platform=" + dockerBuildOsAndArch(c.arch),
},
[]string{
"infinity", // sleep infinitely to keep the container around
Expand All @@ -374,3 +370,7 @@ func (c *buildContext) createBuildContainer() (id string, err error) {
}
return id, nil
}

func dockerBuildOsAndArch(arch string) string {
return "linux/" + arch
}
4 changes: 2 additions & 2 deletions pkg/build/nodeimage/const_cni.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package nodeimage
The default CNI manifest and images are our own tiny kindnet
*/

var defaultCNIImages = []string{"kindest/kindnetd:v20210326-1e038dc5"}
var defaultCNIImages = []string{"docker.io/kindest/kindnetd:v20210326-1e038dc5"}

// TODO: migrate to fully patching and deprecate the template
const defaultCNIManifest = `
Expand Down Expand Up @@ -94,7 +94,7 @@ spec:
serviceAccountName: kindnet
containers:
- name: kindnet-cni
image: kindest/kindnetd:v20210326-1e038dc5
image: docker.io/kindest/kindnetd:v20210326-1e038dc5
env:
- name: HOST_IP
valueFrom:
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/nodeimage/const_storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ NOTE: we have customized it in the following ways:
- install as the default storage class
*/

var defaultStorageImages = []string{"rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}
var defaultStorageImages = []string{"docker.io/rancher/local-path-provisioner:v0.0.14", "k8s.gcr.io/build-image/debian-base:v2.1.0"}

const defaultStorageManifest = `
# kind customized https://github.com/rancher/local-path-provisioner manifest
Expand Down Expand Up @@ -95,7 +95,7 @@ spec:
serviceAccountName: local-path-provisioner-service-account
containers:
- name: local-path-provisioner
image: rancher/local-path-provisioner:v0.0.14
image: docker.io/rancher/local-path-provisioner:v0.0.14
imagePullPolicy: IfNotPresent
command:
- local-path-provisioner
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/nodeimage/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@ package nodeimage
const DefaultImage = "kindest/node:latest"

// DefaultBaseImage is the default base image used
const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961@sha256:2b96d0b11c80e7cb1096ea89e2ffbe26ae72b2b08a177e088fa0edeff9fad516"
const DefaultBaseImage = "docker.io/kindest/base:v20210513-60cf6961"
21 changes: 11 additions & 10 deletions pkg/build/nodeimage/imageimporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,28 +17,21 @@ limitations under the License.
package nodeimage

import (
"io/ioutil"

"sigs.k8s.io/kind/pkg/exec"
)

type imageImporter interface {
Prepare() error
LoadCommand() exec.Cmd
ListImported() ([]string, error)
End() error
}

type containerdImporter struct {
containerCmder exec.Cmder
}

func newContainerdImporter(containerCmder exec.Cmder) imageImporter {
func newContainerdImporter(containerCmder exec.Cmder) *containerdImporter {
return &containerdImporter{
containerCmder: containerCmder,
}
}

var _ imageImporter = &containerdImporter{}

func (c *containerdImporter) Prepare() error {
if err := c.containerCmder.Command(
"bash", "-c", "nohup containerd > /dev/null 2>&1 &",
Expand All @@ -53,6 +46,14 @@ func (c *containerdImporter) End() error {
return c.containerCmder.Command("pkill", "containerd").Run()
}

func (c *containerdImporter) Pull(image, platform string) error {
// TODO: this should exist with a --no-unpack and some way to operate quietly
// without discarding output
return c.containerCmder.Command(
"ctr", "--namespace=k8s.io", "images", "pull", "--platform="+platform, image,
).SetStdout(ioutil.Discard).SetStderr(ioutil.Discard).Run()
}

func (c *containerdImporter) LoadCommand() exec.Cmd {
return c.containerCmder.Command(
// TODO: ideally we do not need this in the future. we have fixed at least one image
Expand Down
24 changes: 23 additions & 1 deletion pkg/build/nodeimage/internal/container/docker/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func GetArchiveTags(path string) ([]string, error) {
// https://github.com/moby/moby/blob/master/image/spec/v1.md
// https://github.com/moby/moby/blob/master/image/spec/v1.1.md
// https://github.com/moby/moby/blob/master/image/spec/v1.2.md
func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositories func(string) string) error {
func EditArchive(reader io.Reader, writer io.Writer, editRepositories func(string) string, architectureOverride string) error {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we have to do this because kubernetes has frequently produced images with the wrong architecture metadata when cross compiling images, which prevents success importing / unning them under containerd which is actually checking for the matching image. dockerd is pretty loose with using these but does throw warnings.

tarReader := tar.NewReader(reader)
tarWriter := tar.NewWriter(writer)
// iterate all entries in the tarball
Expand Down Expand Up @@ -117,6 +117,15 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie
return err
}
hdr.Size = int64(len(b))
// edit image config when we find that
} else if strings.HasSuffix(hdr.Name, ".json") {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is very obscure, so every json but manifest.json is an image config manifest?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, they will be named <hash>.json. we could get the names from the manifest.json, but then we need to do two passes over the tar or depend on ordering or ...

we could do this, but it doesn't seem necessary. images following the spac should only contain manifest.json and <hash>.json .json files where the latter are image config files.

links to the spec are in the comments above https://github.com/moby/moby/blob/master/image/spec/v1.2.md

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a typical image archive only has two .json files in it, but it's possible to create a tarball with multiple images, in which case there is one manifest.json and one config hash.json.

The manifest contains an array of structs each with the config file name for an image, it's tags, and it's layers. There is also the legacy repositories file for the tags.

The config file specifies the non-layer data like architecture/os, entrypoint, env, etc. for one image. We are patching the architecture field to avoid kind only working on highly patched kubernetes builds.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is a lot of information, do you think future us can figure it out?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are comments and there are links to the spec, which is not very lengthy :-)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

if architectureOverride != "" {
b, err = editConfigArchitecture(b, architectureOverride)
if err != nil {
return err
}
hdr.Size = int64(len(b))
}
}

// write to the output tarball
Expand All @@ -133,6 +142,19 @@ func EditArchiveRepositories(reader io.Reader, writer io.Writer, editRepositorie

/* helpers */

func editConfigArchitecture(raw []byte, architectureOverride string) ([]byte, error) {
var cfg map[string]interface{}
if err := json.Unmarshal(raw, &cfg); err != nil {
return nil, err
}
const architecture = "architecture"
if _, ok := cfg[architecture]; !ok {
return raw, nil
}
cfg[architecture] = architectureOverride
return json.Marshal(cfg)
}

// archiveRepositories represents repository:tag:ref
//
// https://github.com/moby/moby/blob/master/image/spec/v1.md
Expand Down
Loading