feat: implement cross-platform support for GCB build env (#7134)

integration/examples/cross-platform-builds/Dockerfile

@@ -1,8 +1,10 @@
-FROM golang:1.15 as builder
+FROM --platform=$BUILDPLATFORM golang:1.15 as builder
 COPY main.go .
 # `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG TARGETOS
+ARG TARGETARCH
 ARG SKAFFOLD_GO_GCFLAGS
-RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
+RUN GOOS=$TARGETOS GOARCH=$TARGETARCH go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o /app main.go
 
 FROM alpine:3
 # Define GOTRACEBACK to mark this container as using the Go language runtime

integration/examples/cross-platform-builds/skaffold.yaml

@@ -8,3 +8,8 @@ deploy:
   kubectl:
     manifests:
       - k8s-*
+profiles:
+  - name: googleCloudBuild
+    build:
+      googleCloudBuild:
+        projectId: k8s-skaffold

pkg/skaffold/build/gcb/buildpacks_test.go

@@ -22,6 +22,7 @@ import (
 
 	"google.golang.org/api/cloudbuild/v1"
 
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/GoogleContainerTools/skaffold/testutil"
@@ -194,7 +195,7 @@ func TestBuildpackBuildSpec(t *testing.T) {
 			builder := NewBuilder(&mockBuilderContext{artifactStore: store}, &latestV1.GoogleCloudBuild{
 				PackImage: "pack/image",
 			})
-			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
+			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", platform.Matcher{}, "bucket", "object")
 			t.CheckError(test.shouldErr, err)
 
 			if !test.shouldErr {

pkg/skaffold/build/gcb/cloud_build.go

@@ -152,7 +152,7 @@ func (b *Builder) buildArtifactWithCloudBuild(ctx context.Context, out io.Writer
 		})
 	}
 
-	buildSpec, err := b.buildSpec(ctx, artifact, tag, cbBucket, buildObject)
+	buildSpec, err := b.buildSpec(ctx, artifact, tag, platform, cbBucket, buildObject)
 	if err != nil {
 		return "", sErrors.NewErrorWithStatusCode(&proto.ActionableErr{
 			ErrCode: proto.StatusCode_BUILD_GCB_GENERATE_BUILD_DESCRIPTOR_ERR,

pkg/skaffold/build/gcb/docker.go

@@ -24,24 +24,29 @@ import (
 	cloudbuild "google.golang.org/api/cloudbuild/v1"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util/stringslice"
 )
 
 // dockerBuildSpec lists the build steps required to build a docker image.
-func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string) (cloudbuild.Build, error) {
+func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string, platforms platform.Matcher) (cloudbuild.Build, error) {
 	a = adjustCacheFrom(a, tag)
 
-	args, err := b.dockerBuildArgs(a, tag, a.Dependencies)
+	args, err := b.dockerBuildArgs(a, tag, a.Dependencies, platforms)
 	if err != nil {
 		return cloudbuild.Build{}, err
 	}
-
-	steps := b.cacheFromSteps(a.DockerArtifact)
-	steps = append(steps, &cloudbuild.BuildStep{
+	steps := b.cacheFromSteps(a.DockerArtifact, platforms)
+	buildStep := &cloudbuild.BuildStep{
 		Name: b.DockerImage,
 		Args: args,
-	})
+	}
+	if platforms.IsNotEmpty() {
+		// cross-platform build requires buildkit enabled
+		buildStep.Env = append(buildStep.Env, "DOCKER_BUILDKIT=1")
+	}
+	steps = append(steps, buildStep)
 
 	return cloudbuild.Build{
 		Steps:  steps,
@@ -50,22 +55,25 @@ func (b *Builder) dockerBuildSpec(a *latestV1.Artifact, tag string) (cloudbuild.
 }
 
 // cacheFromSteps pulls images used by `--cache-from`.
-func (b *Builder) cacheFromSteps(artifact *latestV1.DockerArtifact) []*cloudbuild.BuildStep {
+func (b *Builder) cacheFromSteps(artifact *latestV1.DockerArtifact, platforms platform.Matcher) []*cloudbuild.BuildStep {
 	var steps []*cloudbuild.BuildStep
-
+	argFmt := "docker pull %s || true"
+	if platforms.IsNotEmpty() {
+		argFmt = "docker pull --platform " + platforms.String() + " %s || true"
+	}
 	for _, cacheFrom := range artifact.CacheFrom {
 		steps = append(steps, &cloudbuild.BuildStep{
 			Name:       b.DockerImage,
 			Entrypoint: "sh",
-			Args:       []string{"-c", fmt.Sprintf("docker pull %s || true", cacheFrom)},
+			Args:       []string{"-c", fmt.Sprintf(argFmt, cacheFrom)},
 		})
 	}
 
 	return steps
 }
 
 // dockerBuildArgs lists the arguments passed to `docker` to build a given image.
-func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*latestV1.ArtifactDependency) ([]string, error) {
+func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*latestV1.ArtifactDependency, platforms platform.Matcher) ([]string, error) {
 	d := a.DockerArtifact
 	// TODO(nkubala): remove when buildkit is supported in GCB (#4773)
 	if len(d.Secrets) > 0 || d.SSH != "" {
@@ -83,6 +91,9 @@ func (b *Builder) dockerBuildArgs(a *latestV1.Artifact, tag string, deps []*late
 	}
 
 	args := []string{"build", "--tag", tag, "-f", d.DockerfilePath}
+	if platforms.IsNotEmpty() {
+		args = append(args, "--platform", platforms.String())
+	}
 	args = append(args, ba...)
 	args = append(args, ".")
 

pkg/skaffold/build/gcb/docker_test.go

@@ -20,10 +20,12 @@ import (
 	"context"
 	"testing"
 
-	cloudbuild "google.golang.org/api/cloudbuild/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"google.golang.org/api/cloudbuild/v1"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/GoogleContainerTools/skaffold/testutil"
@@ -33,6 +35,7 @@ func TestDockerBuildSpec(t *testing.T) {
 	tests := []struct {
 		description string
 		artifact    *latestV1.Artifact
+		platforms   platform.Matcher
 		expected    cloudbuild.Build
 		shouldErr   bool
 	}{
@@ -130,6 +133,42 @@ func TestDockerBuildSpec(t *testing.T) {
 			},
 			shouldErr: true,
 		},
+
+		{
+			description: "cross-platform build",
+			artifact: &latestV1.Artifact{
+				ArtifactType: latestV1.ArtifactType{
+					DockerArtifact: &latestV1.DockerArtifact{
+						DockerfilePath: "Dockerfile",
+						BuildArgs: map[string]*string{
+							"arg1": util.StringPtr("value1"),
+							"arg2": nil,
+						},
+					},
+				},
+			},
+			platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
+			expected: cloudbuild.Build{
+				LogsBucket: "bucket",
+				Source: &cloudbuild.Source{
+					StorageSource: &cloudbuild.StorageSource{
+						Bucket: "bucket",
+						Object: "object",
+					},
+				},
+				Steps: []*cloudbuild.BuildStep{{
+					Name: "docker/docker",
+					Args: []string{"build", "--tag", "nginx", "-f", "Dockerfile", "--platform", "freebsd/arm", "--build-arg", "arg1=value1", "--build-arg", "arg2", "."},
+					Env:  []string{"DOCKER_BUILDKIT=1"},
+				}},
+				Images: []string{"nginx"},
+				Options: &cloudbuild.BuildOptions{
+					DiskSizeGb:  100,
+					MachineType: "n1-standard-1",
+				},
+				Timeout: "10m",
+			},
+		},
 	}
 
 	for _, test := range tests {
@@ -157,7 +196,7 @@ func TestDockerBuildSpec(t *testing.T) {
 				Timeout:     "10m",
 			})
 
-			desc, err := builder.buildSpec(context.Background(), test.artifact, "nginx", "bucket", "object")
+			desc, err := builder.buildSpec(context.Background(), test.artifact, "nginx", test.platforms, "bucket", "object")
 			t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, desc)
 		})
 	}
@@ -168,6 +207,7 @@ func TestPullCacheFrom(t *testing.T) {
 		description string
 		artifact    *latestV1.Artifact
 		tag         string
+		platforms   platform.Matcher
 		expected    []*cloudbuild.BuildStep
 		shouldErr   bool
 	}{
@@ -216,6 +256,32 @@ func TestPullCacheFrom(t *testing.T) {
 				Args: []string{"build", "--tag", "gcr.io/k8s-skaffold/test:tagged", "-f", "Dockerfile", "--cache-from", "gcr.io/k8s-skaffold/test:tagged", "."},
 			}},
 		},
+		{
+			description: "cross-platform cache-from images",
+			artifact: &latestV1.Artifact{
+				ArtifactType: latestV1.ArtifactType{
+					DockerArtifact: &latestV1.DockerArtifact{
+						DockerfilePath: "Dockerfile",
+						CacheFrom:      []string{"from/image1", "from/image2"},
+					},
+				},
+			},
+			tag:       "nginx2",
+			platforms: platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
+			expected: []*cloudbuild.BuildStep{{
+				Name:       "docker/docker",
+				Entrypoint: "sh",
+				Args:       []string{"-c", "docker pull --platform freebsd/arm from/image1 || true"},
+			}, {
+				Name:       "docker/docker",
+				Entrypoint: "sh",
+				Args:       []string{"-c", "docker pull --platform freebsd/arm from/image2 || true"},
+			}, {
+				Name: "docker/docker",
+				Args: []string{"build", "--tag", "nginx2", "-f", "Dockerfile", "--platform", "freebsd/arm", "--cache-from", "from/image1", "--cache-from", "from/image2", "."},
+				Env:  []string{"DOCKER_BUILDKIT=1"},
+			}},
+		},
 	}
 
 	for _, test := range tests {
@@ -226,7 +292,7 @@ func TestPullCacheFrom(t *testing.T) {
 			builder := NewBuilder(&mockBuilderContext{}, &latestV1.GoogleCloudBuild{
 				DockerImage: "docker/docker",
 			})
-			desc, err := builder.dockerBuildSpec(test.artifact, test.tag)
+			desc, err := builder.dockerBuildSpec(test.artifact, test.tag, test.platforms)
 
 			t.CheckErrorAndDeepEqual(test.shouldErr, err, test.expected, desc.Steps)
 		})

pkg/skaffold/build/gcb/jib_test.go

@@ -22,9 +22,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	cloudbuild "google.golang.org/api/cloudbuild/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	"google.golang.org/api/cloudbuild/v1"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/jib"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 	"github.com/GoogleContainerTools/skaffold/testutil"
 )
@@ -40,6 +42,7 @@ func TestJibMavenBuildSpec(t *testing.T) {
 		description  string
 		skipTests    bool
 		baseImage    string
+		platforms    platform.Matcher
 		expectedArgs []string
 	}{
 		{
@@ -64,6 +67,16 @@ func TestJibMavenBuildSpec(t *testing.T) {
 			skipTests:    false,
 			expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersion + " --non-recursive prepare-package jib:build -Djib.from.image=img2:tag -Dimage=img"},
 		},
+		{
+			description:  "cross platform",
+			platforms:    platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
+			expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersionForCrossPlatform + " --non-recursive prepare-package jib:build -Djib.from.platforms=freebsd/arm -Dimage=img"},
+		},
+		{
+			description:  "multi platform",
+			platforms:    platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}, {Architecture: "arm64", OS: "linux"}}},
+			expectedArgs: []string{"-c", "mvn -Duser.home=$$HOME --batch-mode jib:_skaffold-fail-if-jib-out-of-date -Djib.requiredVersion=" + jib.MinimumJibMavenVersionForCrossPlatform + " --non-recursive prepare-package jib:build -Djib.from.platforms=freebsd/arm,linux/arm64 -Dimage=img"},
+		},
 	}
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
@@ -86,7 +99,7 @@ func TestJibMavenBuildSpec(t *testing.T) {
 			})
 			builder.skipTests = test.skipTests
 
-			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
+			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", test.platforms, "bucket", "object")
 			t.CheckNoError(err)
 
 			expected := []*cloudbuild.BuildStep{{
@@ -105,6 +118,7 @@ func TestJibGradleBuildSpec(t *testing.T) {
 	tests := []struct {
 		description  string
 		skipTests    bool
+		platforms    platform.Matcher
 		expectedArgs []string
 	}{
 		{
@@ -117,6 +131,16 @@ func TestJibGradleBuildSpec(t *testing.T) {
 			skipTests:    false,
 			expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersion + " :jib --image=img"},
 		},
+		{
+			description:  "cross platform",
+			platforms:    platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}}},
+			expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersionForCrossPlatform + " :jib -Djib.from.platforms=freebsd/arm --image=img"},
+		},
+		{
+			description:  "multi platform",
+			platforms:    platform.Matcher{Platforms: []v1.Platform{{Architecture: "arm", OS: "freebsd"}, {Architecture: "arm64", OS: "linux"}}},
+			expectedArgs: []string{"-c", "gradle -Duser.home=$$HOME --console=plain _skaffoldFailIfJibOutOfDate -Djib.requiredVersion=" + jib.MinimumJibGradleVersionForCrossPlatform + " :jib -Djib.from.platforms=freebsd/arm,linux/arm64 --image=img"},
+		},
 	}
 	for _, test := range tests {
 		testutil.Run(t, test.description, func(t *testutil.T) {
@@ -131,7 +155,7 @@ func TestJibGradleBuildSpec(t *testing.T) {
 			})
 			builder.skipTests = test.skipTests
 
-			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", "bucket", "object")
+			buildSpec, err := builder.buildSpec(context.Background(), artifact, "img", test.platforms, "bucket", "object")
 			t.CheckNoError(err)
 
 			expected := []*cloudbuild.BuildStep{{

pkg/skaffold/build/gcb/kaniko_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/graph"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 	"github.com/GoogleContainerTools/skaffold/testutil"
@@ -438,7 +439,7 @@ func TestKanikoBuildSpec(t *testing.T) {
 				}
 				return m, nil
 			})
-			desc, err := builder.buildSpec(context.Background(), artifact, "gcr.io/nginx", "bucket", "object")
+			desc, err := builder.buildSpec(context.Background(), artifact, "gcr.io/nginx", platform.Matcher{}, "bucket", "object")
 
 			expected := cloudbuild.Build{
 				LogsBucket: "bucket",

pkg/skaffold/build/gcb/spec.go

@@ -20,16 +20,17 @@ import (
 	"context"
 	"fmt"
 
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"google.golang.org/api/cloudbuild/v1"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/misc"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
 	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
 )
 
-func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, tag, bucket, object string) (cloudbuild.Build, error) {
+func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, tag string, platforms platform.Matcher, bucket, object string) (cloudbuild.Build, error) {
 	// Artifact specific build spec
-	buildSpec, err := b.buildSpecForArtifact(ctx, artifact, tag)
+	buildSpec, err := b.buildSpecForArtifact(ctx, artifact, tag, platforms)
 	if err != nil {
 		return buildSpec, err
 	}
@@ -57,18 +58,22 @@ func (b *Builder) buildSpec(ctx context.Context, artifact *latestV1.Artifact, ta
 	return buildSpec, nil
 }
 
-func (b *Builder) buildSpecForArtifact(ctx context.Context, a *latestV1.Artifact, tag string) (cloudbuild.Build, error) {
+func (b *Builder) buildSpecForArtifact(ctx context.Context, a *latestV1.Artifact, tag string, platforms platform.Matcher) (cloudbuild.Build, error) {
 	switch {
 	case a.KanikoArtifact != nil:
 		return b.kanikoBuildSpec(a, tag)
 
 	case a.DockerArtifact != nil:
-		return b.dockerBuildSpec(a, tag)
+		return b.dockerBuildSpec(a, tag, platforms)
 
 	case a.JibArtifact != nil:
-		return b.jibBuildSpec(ctx, a, tag, platform.Matcher{}) // TODO: pass correct platform matcher for GCB builds
+		return b.jibBuildSpec(ctx, a, tag, platforms)
 
 	case a.BuildpackArtifact != nil:
+		// TODO: Buildpacks only supports building for platform linux/amd64. See https://github.com/GoogleCloudPlatform/buildpacks/issues/112
+		if platforms.IsNotEmpty() && platforms.Intersect(platform.Matcher{Platforms: []v1.Platform{{OS: "linux", Architecture: "amd64"}}}).IsEmpty() {
+			return cloudbuild.Build{}, fmt.Errorf("buildpacks builder doesn't support building for platforms %s. Cannot build gcb artifact:\n%s", platforms.String(), misc.FormatArtifact(a))
+		}
 		return b.buildpackBuildSpec(a.BuildpackArtifact, tag, a.Dependencies)
 
 	default: