diff --git a/integration/multiplatform_test.go b/integration/multiplatform_test.go new file mode 100644 index 00000000000..5d72171a2be --- /dev/null +++ b/integration/multiplatform_test.go @@ -0,0 +1,332 @@ +/* +Copyright 2022 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "fmt" + "os" + "strings" + "testing" + + v1 "github.com/opencontainers/image-spec/specs-go/v1" + k8sv1 "k8s.io/api/core/v1" + + "github.com/GoogleContainerTools/skaffold/integration/skaffold" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker" + "github.com/GoogleContainerTools/skaffold/testutil" +) + +const ( + defaultRepo = "gcr.io/k8s-skaffold" + hybridClusterName = "integration-tests-hybrid" + armClusterName = "integration-tests-arm" +) + +func TestMultiPlatformWithRun(t *testing.T) { + MarkIntegrationTest(t, NeedsGcp) + isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName + type image struct { + name string + pod string + } + + tests := []struct { + description string + dir string + images []image + tag string + expectedPlatforms []v1.Platform + }{ + { + description: "Run with multiplatform linux/arm64 and linux/amd64", + dir: "examples/cross-platform-builds", + images: []image{{name: "skaffold-example", pod: "getting-started"}}, + tag: "multiplatform-integration-test", + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + { + description: "Run with multiplatform linux/arm64 and linux/amd64 in a multi config project", + dir: "testdata/multi-config-pods", + images: []image{ + {name: "multi-config-module1", pod: "module1"}, + {name: "multi-config-module2", pod: "module2"}, + }, + tag: "multiplatform-integration-test", + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + platforms := platformsCliValue(test.expectedPlatforms) + ns, client := SetupNamespace(t) + args := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false"} + expectedPlatforms := expectedPlatformsForRunningCluster(test.expectedPlatforms) + + skaffold.Run(args...).InDir(test.dir).InNs(ns.Name).RunOrFail(t) + defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunOrFail(t) + + for _, image := range test.images { + checkRemoteImagePlatforms(t, fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag), expectedPlatforms) + + if isRunningInHybridCluster { + pod := client.GetPod(image.pod) + checkNodeAffinity(t, test.expectedPlatforms, pod) + } + } + }) + } +} + +func TestMultiplatformWithDevAndDebug(t *testing.T) { + MarkIntegrationTest(t, NeedsGcp) + const platformsExpectedInNodeAffinity = 1 + const platformsExpectedInCreatedImage = 1 + isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName + + type image struct { + name string + pod string + } + + tests := []struct { + description string + dir string + images []image + tag string + command func(args ...string) *skaffold.RunBuilder + expectedPlatforms []v1.Platform + }{ + { + description: "Debug with multiplatform linux/arm64 and linux/amd64", + dir: "examples/cross-platform-builds", + images: []image{{name: "skaffold-example", pod: "getting-started"}}, + tag: "multiplatform-integration-test", + command: skaffold.Debug, + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + { + description: "Debug with multiplatform linux/arm64 and linux/amd64 in a multi config project", + dir: "testdata/multi-config-pods", + images: []image{ + {name: "multi-config-module1", pod: "module1"}, + {name: "multi-config-module2", pod: "module2"}, + }, + tag: "multiplatform-integration-test", + command: skaffold.Debug, + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + { + description: "Dev with multiplatform linux/arm64 and linux/amd64", + dir: "examples/cross-platform-builds", + images: []image{{name: "skaffold-example", pod: "getting-started"}}, + tag: "multiplatform-integration-test", + command: skaffold.Dev, + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + { + description: "Dev with multiplatform linux/arm64 and linux/amd64 in a multi config project", + dir: "testdata/multi-config-pods", + images: []image{ + {name: "multi-config-module1", pod: "module1"}, + {name: "multi-config-module2", pod: "module2"}, + }, + tag: "multiplatform-integration-test", + command: skaffold.Dev, + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + platforms := platformsCliValue(test.expectedPlatforms) + ns, client := SetupNamespace(t) + args := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false"} + expectedPlatforms := expectedPlatformsForRunningCluster(test.expectedPlatforms) + + test.command(args...).InDir(test.dir).InNs(ns.Name).RunBackground(t) + defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunBackground(t) + + for _, image := range test.images { + client.WaitForPodsReady(image.pod) + createdImagePlatforms, err := docker.GetPlatforms(fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag)) + failNowIfError(t, err) + + if len(createdImagePlatforms) != platformsExpectedInCreatedImage { + t.Fatalf("there are more platforms in created Image than expected, found %v, expected %v", len(createdImagePlatforms), platformsExpectedInCreatedImage) + } + + checkIfAPlatformMatch(t, expectedPlatforms, createdImagePlatforms[0]) + + if isRunningInHybridCluster { + pod := client.GetPod(image.pod) + failIfNodeAffinityNotSet(t, pod) + nodeAffinityPlatforms := getPlatformsFromNodeAffinity(pod) + platformsInNodeAffinity := len(nodeAffinityPlatforms) + + if platformsInNodeAffinity != platformsExpectedInNodeAffinity { + t.Fatalf("there are more platforms in NodeAffinity than expected, found %v, expected %v", platformsInNodeAffinity, platformsExpectedInNodeAffinity) + } + + checkIfAPlatformMatch(t, expectedPlatforms, nodeAffinityPlatforms[0]) + } + } + }) + } +} + +func TestMultiplatformWithDeploy(t *testing.T) { + MarkIntegrationTest(t, NeedsGcp) + isRunningInHybridCluster := os.Getenv("GKE_CLUSTER_NAME") == hybridClusterName + type image struct { + name string + pod string + } + + tests := []struct { + description string + dir string + images []image + tag string + expectedPlatforms []v1.Platform + }{ + { + description: "Deploy with multiplatform linux/arm64 and linux/amd64", + dir: "examples/cross-platform-builds", + images: []image{{name: "skaffold-example", pod: "getting-started"}}, + tag: "multiplatform-integration-test", + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + { + description: "Deploy with multiplatform linux/arm64 and linux/amd64 in a multi config project", + dir: "testdata/multi-config-pods", + images: []image{ + {name: "multi-config-module1", pod: "module1"}, + {name: "multi-config-module2", pod: "module2"}, + }, + tag: "multiplatform-integration-test", + expectedPlatforms: []v1.Platform{{OS: "linux", Architecture: "arm64"}, {OS: "linux", Architecture: "amd64"}}, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + tmpfile := testutil.TempFile(t, "", []byte{}) + platforms := platformsCliValue(test.expectedPlatforms) + argsBuild := []string{"--platform", platforms, "--default-repo", defaultRepo, "--tag", test.tag, "--cache-artifacts=false", "--file-output", tmpfile} + argsDeploy := []string{"--build-artifacts", tmpfile, "--default-repo", defaultRepo, "--enable-platform-node-affinity=true"} + + skaffold.Build(argsBuild...).InDir(test.dir).RunOrFail(t) + ns, client := SetupNamespace(t) + skaffold.Deploy(argsDeploy...).InDir(test.dir).InNs(ns.Name).RunOrFail(t) + defer skaffold.Delete().InDir(test.dir).InNs(ns.Name).RunOrFail(t) + + for _, image := range test.images { + checkRemoteImagePlatforms(t, fmt.Sprintf("%s/%s:%s", defaultRepo, image.name, test.tag), test.expectedPlatforms) + + if isRunningInHybridCluster { + pod := client.GetPod(image.pod) + checkNodeAffinity(t, test.expectedPlatforms, pod) + } + } + }) + } +} + +func checkNodeAffinity(t *testing.T, expectedPlatforms []v1.Platform, pod *k8sv1.Pod) { + failIfNodeAffinityNotSet(t, pod) + nodeAffinityPlatforms := getPlatformsFromNodeAffinity(pod) + checkPlatformsEqual(t, nodeAffinityPlatforms, expectedPlatforms) +} + +func failIfNodeAffinityNotSet(t *testing.T, pod *k8sv1.Pod) { + if pod.Spec.Affinity == nil { + t.Fatalf("Affinity not defined in spec") + } + + if pod.Spec.Affinity.NodeAffinity == nil { + t.Fatalf("NodeAffinity not defined in spec") + } + + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + t.Fatalf("RequiredDuringSchedulingIgnoredDuringExecution not defined in spec") + } + + if pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms == nil { + t.Fatalf("NodeSelectorTerms not defined in spec") + } +} + +func getPlatformsFromNodeAffinity(pod *k8sv1.Pod) []v1.Platform { + var platforms []v1.Platform + nodeAffinityPlatforms := pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + + for _, np := range nodeAffinityPlatforms { + os, arch := "", "" + for _, me := range np.MatchExpressions { + if me.Key == "kubernetes.io/os" { + os = strings.Join(me.Values, "") + } + + if me.Key == "kubernetes.io/arch" { + arch = strings.Join(me.Values, "") + } + } + + platforms = append(platforms, v1.Platform{OS: os, Architecture: arch}) + } + + return platforms +} + +func platformsCliValue(platforms []v1.Platform) string { + var platformsCliValue []string + for _, platform := range platforms { + platformsCliValue = append(platformsCliValue, fmt.Sprintf("%s/%s", platform.OS, platform.Architecture)) + } + + return strings.Join(platformsCliValue, ",") +} + +func expectedPlatformsForRunningCluster(platforms []v1.Platform) []v1.Platform { + switch clusterName := os.Getenv("GKE_CLUSTER_NAME"); clusterName { + case hybridClusterName: + return platforms + case armClusterName: + return []v1.Platform{{OS: "linux", Architecture: "arm64"}} + default: + return []v1.Platform{{OS: "linux", Architecture: "amd64"}} + } +} + +func checkIfAPlatformMatch(t *testing.T, platforms []v1.Platform, expectedPlatform v1.Platform) { + const expectedMatchedPlatforms = 1 + matchedPlatforms := 0 + nodeAffinityPlatformValue := expectedPlatform.OS + "/" + expectedPlatform.Architecture + + for _, platform := range platforms { + expectedPlatformValue := platform.OS + "/" + platform.Architecture + + if nodeAffinityPlatformValue == expectedPlatformValue { + matchedPlatforms++ + } + } + + if matchedPlatforms != expectedMatchedPlatforms { + t.Fatalf("Number of matched platforms should be %v", expectedMatchedPlatforms) + } +} diff --git a/integration/testdata/multi-config-pods/module1/Dockerfile b/integration/testdata/multi-config-pods/module1/Dockerfile new file mode 100644 index 00000000000..f5884c7f419 --- /dev/null +++ b/integration/testdata/multi-config-pods/module1/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.18 as builder +WORKDIR /code +COPY main.go . +# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations +ARG SKAFFOLD_GO_GCFLAGS +RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -trimpath -o /app main.go + +FROM alpine:3 +# Define GOTRACEBACK to mark this container as using the Go language runtime +# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/). +ENV GOTRACEBACK=single +CMD ["./app"] +COPY --from=builder /app . diff --git a/integration/testdata/multi-config-pods/module1/k8s/k8s-pod.yaml b/integration/testdata/multi-config-pods/module1/k8s/k8s-pod.yaml new file mode 100644 index 00000000000..df37fb3d04c --- /dev/null +++ b/integration/testdata/multi-config-pods/module1/k8s/k8s-pod.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: module1 +spec: + containers: + - name: module1 + image: multi-config-module1 diff --git a/integration/testdata/multi-config-pods/module1/main.go b/integration/testdata/multi-config-pods/module1/main.go new file mode 100644 index 00000000000..ef038c2a33a --- /dev/null +++ b/integration/testdata/multi-config-pods/module1/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "runtime" + "time" +) + +func main() { + for { + fmt.Printf("Hello module-1! Running on %s/%s\n", runtime.GOOS, runtime.GOARCH) + + time.Sleep(time.Second * 1) + } +} diff --git a/integration/testdata/multi-config-pods/module1/skaffold.yaml b/integration/testdata/multi-config-pods/module1/skaffold.yaml new file mode 100644 index 00000000000..30988dd1e37 --- /dev/null +++ b/integration/testdata/multi-config-pods/module1/skaffold.yaml @@ -0,0 +1,11 @@ +apiVersion: skaffold/v3alpha1 +kind: Config +build: + artifacts: + - image: multi-config-module1 + context: . +manifests: + rawYaml: + - k8s/k8s-pod.yaml +deploy: + kubectl: {} \ No newline at end of file diff --git a/integration/testdata/multi-config-pods/module2/Dockerfile b/integration/testdata/multi-config-pods/module2/Dockerfile new file mode 100644 index 00000000000..f5884c7f419 --- /dev/null +++ b/integration/testdata/multi-config-pods/module2/Dockerfile @@ -0,0 +1,13 @@ +FROM golang:1.18 as builder +WORKDIR /code +COPY main.go . +# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations +ARG SKAFFOLD_GO_GCFLAGS +RUN go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -trimpath -o /app main.go + +FROM alpine:3 +# Define GOTRACEBACK to mark this container as using the Go language runtime +# for `skaffold debug` (https://skaffold.dev/docs/workflows/debug/). +ENV GOTRACEBACK=single +CMD ["./app"] +COPY --from=builder /app . diff --git a/integration/testdata/multi-config-pods/module2/k8s/k8s-pod.yaml b/integration/testdata/multi-config-pods/module2/k8s/k8s-pod.yaml new file mode 100644 index 00000000000..49dce1f7c8c --- /dev/null +++ b/integration/testdata/multi-config-pods/module2/k8s/k8s-pod.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: module2 +spec: + containers: + - name: module2 + image: multi-config-module2 diff --git a/integration/testdata/multi-config-pods/module2/main.go b/integration/testdata/multi-config-pods/module2/main.go new file mode 100644 index 00000000000..d947ee8f695 --- /dev/null +++ b/integration/testdata/multi-config-pods/module2/main.go @@ -0,0 +1,15 @@ +package main + +import ( + "fmt" + "runtime" + "time" +) + +func main() { + for { + fmt.Printf("Hello module-2! Running on %s/%s\n", runtime.GOOS, runtime.GOARCH) + + time.Sleep(time.Second * 1) + } +} diff --git a/integration/testdata/multi-config-pods/module2/skaffold.yaml b/integration/testdata/multi-config-pods/module2/skaffold.yaml new file mode 100644 index 00000000000..6dbcd7aaa48 --- /dev/null +++ b/integration/testdata/multi-config-pods/module2/skaffold.yaml @@ -0,0 +1,11 @@ +apiVersion: skaffold/v3alpha1 +kind: Config +build: + artifacts: + - image: multi-config-module2 + context: . +manifests: + rawYaml: + - k8s/k8s-pod.yaml +deploy: + kubectl: {} \ No newline at end of file diff --git a/integration/testdata/multi-config-pods/skaffold.yaml b/integration/testdata/multi-config-pods/skaffold.yaml new file mode 100644 index 00000000000..0e7e960c2f4 --- /dev/null +++ b/integration/testdata/multi-config-pods/skaffold.yaml @@ -0,0 +1,5 @@ +apiVersion: skaffold/v3alpha1 +kind: Config +requires: + - path: ./module1 + - path: ./module2 \ No newline at end of file