test: Add multi platform integration tests

integration/multiplatform_test.go

@@ -0,0 +1,332 @@
+/*
+Copyright 2022 The Skaffold Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+	k8sv1 "k8s.io/api/core/v1"
+
+	"github.com/GoogleContainerTools/skaffold/integration/skaffold"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
+	"github.com/GoogleContainerTools/skaffold/testutil"
+)
+
+const (
+	defaultRepo       = "gcr.io/k8s-skaffold"
+	hybridClusterName = "integration-tests-hybrid"
+	armClusterName    = "integration-tests-arm"
+)
+
+func TestMultiPlatformWithRun(t *testing.T) {
+	MarkIntegrationTest(t, NeedsGcp)
+	isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName
+	type image struct {
+		name string
+		pod  string
+	}
+
+	tests := []struct {
+		description       string
+		dir               string
+		images            []image
+		tag               string
+		expectedPlatforms []v1.Platform
+	}{
+		{
+			description:       "Run with multiplatform linux/arm64 and linux/amd64",
+			dir:               "examples/cross-platform-builds",
+			images:            []image{{name: "skaffold-example", pod: "getting-started"}},
+			tag:               "multiplatform-integration-test",
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			description: "Run with multiplatform linux/arm64 and linux/amd64 in a multi config project",
+			dir:         "testdata/multi-config-pods",
+			images: []image{
+				{name: "multi-config-module1", pod: "module1"},
+				{name: "multi-config-module2", pod: "module2"},
+			},
+			tag:               "multiplatform-integration-test",
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			platforms := platformsCliValue(test.expectedPlatforms)
+			ns, client := SetupNamespace(t)
+			args := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false"}
+			expectedPlatforms := expectedPlatformsForRunningCluster(test.expectedPlatforms)
+
+			skaffold.Run(args...).InDir(test.dir).InNs(ns.Name).RunOrFail(t)
+			defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunOrFail(t)
+
+			for _, image := range test.images {
+				checkRemoteImagePlatforms(t, fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag), expectedPlatforms)
+
+				if isRunningInHybridCluster {
+					pod := client.GetPod(image.pod)
+					checkNodeAffinity(t, test.expectedPlatforms, pod)
+				}
+			}
+		})
+	}
+}
+
+func TestMultiplatformWithDevAndDebug(t *testing.T) {
+	MarkIntegrationTest(t, NeedsGcp)
+	const platformsExpectedInNodeAffinity = 1
+	const platformsExpectedInCreatedImage = 1
+	isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName
+
+	type image struct {
+		name string
+		pod  string
+	}
+
+	tests := []struct {
+		description       string
+		dir               string
+		images            []image
+		tag               string
+		command           func(args ...string) *skaffold.RunBuilder
+		expectedPlatforms []v1.Platform
+	}{
+		{
+			description:       "Debug with multiplatform linux/arm64 and linux/amd64",
+			dir:               "examples/cross-platform-builds",
+			images:            []image{{name: "skaffold-example", pod: "getting-started"}},
+			tag:               "multiplatform-integration-test",
+			command:           skaffold.Debug,
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			description: "Debug with multiplatform linux/arm64 and linux/amd64 in a multi config project",
+			dir:         "testdata/multi-config-pods",
+			images: []image{
+				{name: "multi-config-module1", pod: "module1"},
+				{name: "multi-config-module2", pod: "module2"},
+			},
+			tag:               "multiplatform-integration-test",
+			command:           skaffold.Debug,
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			description:       "Dev with multiplatform linux/arm64 and linux/amd64",
+			dir:               "examples/cross-platform-builds",
+			images:            []image{{name: "skaffold-example", pod: "getting-started"}},
+			tag:               "multiplatform-integration-test",
+			command:           skaffold.Dev,
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			description: "Dev with multiplatform linux/arm64 and linux/amd64 in a multi config project",
+			dir:         "testdata/multi-config-pods",
+			images: []image{
+				{name: "multi-config-module1", pod: "module1"},
+				{name: "multi-config-module2", pod: "module2"},
+			},
+			tag:               "multiplatform-integration-test",
+			command:           skaffold.Dev,
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			platforms := platformsCliValue(test.expectedPlatforms)
+			ns, client := SetupNamespace(t)
+			args := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false"}
+			expectedPlatforms := expectedPlatformsForRunningCluster(test.expectedPlatforms)
+
+			test.command(args...).InDir(test.dir).InNs(ns.Name).RunBackground(t)
+			defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunBackground(t)
+
+			for _, image := range test.images {
+				client.WaitForPodsReady(image.pod)
+				createdImagePlatforms, err := docker.GetPlatforms(fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag))
+				failNowIfError(t, err)
+
+				if len(createdImagePlatforms) != platformsExpectedInCreatedImage {
+					t.Fatalf("there are more platforms in created Image than expected, found %v, expected %v", len(createdImagePlatforms), platformsExpectedInCreatedImage)
+				}
+
+				checkIfAPlatformMatch(t, expectedPlatforms, createdImagePlatforms[0])
+
+				if isRunningInHybridCluster {
+					pod := client.GetPod(image.pod)
+					failIfNodeAffinityNotSet(t, pod)
+					nodeAffinityPlatforms := getPlatformsFromNodeAffinity(pod)
+					platformsInNodeAffinity := len(nodeAffinityPlatforms)
+
+					if platformsInNodeAffinity != platformsExpectedInNodeAffinity {
+						t.Fatalf("there are more platforms in NodeAffinity than expected, found %v, expected %v", platformsInNodeAffinity, platformsExpectedInNodeAffinity)
+					}
+
+					checkIfAPlatformMatch(t, expectedPlatforms, nodeAffinityPlatforms[0])
+				}
+			}
+		})
+	}
+}
+
+func TestMultiplatformWithDeploy(t *testing.T) {
+	MarkIntegrationTest(t, NeedsGcp)
+	isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName
+	type image struct {
+		name string
+		pod  string
+	}
+
+	tests := []struct {
+		description       string
+		dir               string
+		images            []image
+		tag               string
+		expectedPlatforms []v1.Platform
+	}{
+		{
+			description:       "Deploy with multiplatform linux/arm64 and linux/amd64",
+			dir:               "examples/cross-platform-builds",
+			images:            []image{{name: "skaffold-example", pod: "getting-started"}},
+			tag:               "multiplatform-integration-test",
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+		{
+			description: "Deploy with multiplatform linux/arm64 and linux/amd64 in a multi config project",
+			dir:         "testdata/multi-config-pods",
+			images: []image{
+				{name: "multi-config-module1", pod: "module1"},
+				{name: "multi-config-module2", pod: "module2"},
+			},
+			tag:               "multiplatform-integration-test",
+			expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			tmpfile := testutil.TempFile(t, "", []byte{})
+			platforms := platformsCliValue(test.expectedPlatforms)
+			argsBuild := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false", "--file-output", tmpfile}
+			argsDeploy := []string{"--build-artifacts", tmpfile, "--default-repo", defaultRepo, "--enable-platform-node-affinity=true"}
+
+			skaffold.Build(argsBuild...).InDir(test.dir).RunOrFail(t)
+			ns, client := SetupNamespace(t)
+			skaffold.Deploy(argsDeploy...).InDir(test.dir).InNs(ns.Name).RunOrFail(t)
+			defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunOrFail(t)
+
+			for _, image := range test.images {
+				checkRemoteImagePlatforms(t, fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag), test.expectedPlatforms)
+
+				if isRunningInHybridCluster {
+					pod := client.GetPod(image.pod)
+					checkNodeAffinity(t, test.expectedPlatforms, pod)
+				}
+			}
+		})
+	}
+}
+
+func checkNodeAffinity(t *testing.T, expectedPlatforms []v1.Platform, pod *k8sv1.Pod) {
+	failIfNodeAffinityNotSet(t, pod)
+	nodeAffinityPlatforms := getPlatformsFromNodeAffinity(pod)
+	checkPlatformsEqual(t, nodeAffinityPlatforms, expectedPlatforms)
+}
+
+func failIfNodeAffinityNotSet(t *testing.T, pod *k8sv1.Pod) {
+	if pod.Spec.Affinity == nil {
+		t.Fatalf("Affinity not defined in spec")
+	}
+
+	if pod.Spec.Affinity.NodeAffinity == nil {
+		t.Fatalf("NodeAffinity not defined in spec")
+	}
+
+	if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
+		t.Fatalf("RequiredDuringSchedulingIgnoredDuringExecution not defined in spec")
+	}
+
+	if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms == nil {
+		t.Fatalf("NodeSelectorTerms not defined in spec")
+	}
+}
+
+func getPlatformsFromNodeAffinity(pod *k8sv1.Pod) []v1.Platform {
+	var platforms []v1.Platform
+	nodeAffinityPlatforms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms
+
+	for _, np := range nodeAffinityPlatforms {
+		os, arch := "", ""
+		for _, me := range np.MatchExpressions {
+			if me.Key == "kubernetes.io/os" {
+				os = strings.Join(me.Values, "")
+			}
+
+			if me.Key == "kubernetes.io/arch" {
+				arch = strings.Join(me.Values, "")
+			}
+		}
+
+		platforms = append(platforms, v1.Platform{OS: os, Architecture: arch})
+	}
+
+	return platforms
+}
+
+func platformsCliValue(platforms []v1.Platform) string {
+	var platformsCliValue []string
+	for _, platform := range platforms {
+		platformsCliValue = append(platformsCliValue, fmt.Sprintf("%s/%s", platform.OS, platform.Architecture))
+	}
+
+	return strings.Join(platformsCliValue, ",")
+}
+
+func expectedPlatformsForRunningCluster(platforms []v1.Platform) []v1.Platform {
+	switch clusterName := os.Getenv("GKE_CLUSTER_NAME"); clusterName {
+	case hybridClusterName:
+		return platforms
+	case armClusterName:
+		return []v1.Platform{{OS: "linux", Architecture: "arm64"}}
+	default:
+		return []v1.Platform{{OS: "linux", Architecture: "amd64"}}
+	}
+}
+
+func checkIfAPlatformMatch(t *testing.T, platforms []v1.Platform, expectedPlatform v1.Platform) {
+	const expectedMatchedPlatforms = 1
+	matchedPlatforms := 0
+	nodeAffinityPlatformValue := expectedPlatform.OS + "/" + expectedPlatform.Architecture
+
+	for _, platform := range platforms {
+		expectedPlatformValue := platform.OS + "/" + platform.Architecture
+
+		if nodeAffinityPlatformValue == expectedPlatformValue {
+			matchedPlatforms++
+		}
+	}
+
+	if matchedPlatforms != expectedMatchedPlatforms {
+		t.Fatalf("Number of matched platforms should be %v", expectedMatchedPlatforms)
+	}
+}

integration/testdata/multi-config-pods/module1/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.18 as builder
+WORKDIR /code
+COPY main.go .
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -trimpath -o /app main.go
+
+FROM alpine:3
+# Define GOTRACEBACK to mark this container as using the Go language runtime
+# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
+ENV GOTRACEBACK=single
+CMD ["./app"]
+COPY --from=builder /app .

integration/testdata/multi-config-pods/module1/k8s/k8s-pod.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: module1
+spec:
+  containers:
+  - name: module1
+    image: multi-config-module1

integration/testdata/multi-config-pods/module1/main.go

@@ -0,0 +1,15 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+)
+
+func main() {
+	for {
+		fmt.Printf("Hello module-1! Running on %s/%s\n", runtime.GOOS, runtime.GOARCH)
+
+		time.Sleep(time.Second * 1)
+	}
+}

integration/testdata/multi-config-pods/module1/skaffold.yaml

@@ -0,0 +1,11 @@
+apiVersion: skaffold/v3alpha1
+kind: Config
+build:
+  artifacts:
+  - image: multi-config-module1
+    context: .
+manifests:
+  rawYaml:
+  - k8s/k8s-pod.yaml
+deploy:
+  kubectl: {}

integration/testdata/multi-config-pods/module2/Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.18 as builder
+WORKDIR /code
+COPY main.go .
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -trimpath -o /app main.go
+
+FROM alpine:3
+# Define GOTRACEBACK to mark this container as using the Go language runtime
+# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/).
+ENV GOTRACEBACK=single
+CMD ["./app"]
+COPY --from=builder /app .