Add Docker airgapped test (#4907)

cmd/eksctl-anywhere/cmd/listimages.go

@@ -11,7 +11,7 @@ import (
 )
 
 type listImagesOptions struct {
-	fileName string
+	fileName        string
 	bundlesOverride string
 }
 

cmd/eksctl-anywhere/cmd/listovas.go

@@ -19,7 +19,7 @@ import (
 )
 
 type listOvasOptions struct {
-	fileName string
+	fileName        string
 	bundlesOverride string
 }
 

cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml

@@ -20,6 +20,8 @@ env:
     T_REGISTRY_MIRROR_USERNAME: "harbor-registry-data:username"
     T_REGISTRY_MIRROR_PASSWORD: "harbor-registry-data:password"
     T_REGISTRY_MIRROR_CA_CERT: "harbor-registry-data:caCert"
+    T_REGISTRY_MIRROR_DEFAULT_SECURITY_GROUP: "harbor-registry-data:default_sg_id"
+    T_REGISTRY_MIRROR_AIRGAPPED_SECURITY_GROUP: "harbor-registry-data:airgapped_sg_id"
     T_AWS_IAM_ROLE_ARN: "aws-iam-auth-role:ec2_role_arn"
 phases:
   pre_build:

internal/aws-sdk-go-v2/internal/endpoints/v2/endpoints.go

@@ -5,9 +5,8 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/aws/smithy-go/logging"
-
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/smithy-go/logging"
 )
 
 // DefaultKey is a compound map key of a variant and other values.

internal/test/cleanup/cleanup.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws/session"
-
 	prismgoclient "github.com/nutanix-cloud-native/prism-go-client"
 	v3 "github.com/nutanix-cloud-native/prism-go-client/v3"
 

internal/test/e2e/registryMirror.go

@@ -43,6 +43,18 @@ func (e *E2ESession) setupRegistryMirrorEnv(testRegex string) error {
 		return e.mountRegistryCert(caCert, net.JoinHostPort(endpoint, port))
 	}
 
+	re = regexp.MustCompile(`^.*Docker.*Airgapped.*$`)
+	if re.MatchString(testRegex) {
+		err := os.Setenv("DEFAULT_SECURITY_GROUP", e.testEnvVars[e2etests.RegistryMirrorDefaultSecurityGroup])
+		if err != nil {
+			return fmt.Errorf("unable to set DEFAULT_SECURITY_GROUP: %v", err)
+		}
+		err = os.Setenv("AIRGAPPED_SECURITY_GROUP", e.testEnvVars[e2etests.RegistryMirrorAirgappedSecurityGroup])
+		if err != nil {
+			return fmt.Errorf("unable to set AIRGAPPED_SECURITY_GROUP: %v", err)
+		}
+	}
+
 	return nil
 }
 

internal/test/e2e/setup.go

@@ -177,7 +177,7 @@ func (e *E2ESession) updateFSInotifyResources() error {
 	if err := ssm.Run(e.session, logr.Discard(), e.instanceId, command); err != nil {
 		return fmt.Errorf("updating fs inotify resources: %v", err)
 	}
-	e.logger.V(1).Info("Successfully updates the fs inotify user watches and instances")
+	e.logger.V(1).Info("Successfully updated the fs inotify user watches and instances")
 
 	return nil
 }

pkg/providers/docker/config/template-cp.yaml

@@ -138,6 +138,31 @@ spec:
 {{ .auditPolicy | indent 8 }}
       owner: root:root
       path: /etc/kubernetes/audit-policy.yaml
+{{- if .registryCACert }}
+    - content: |
+{{ .registryCACert | indent 8 }}
+      owner: root:root
+      path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+    - content: |
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+          {{- range $orig, $mirror := .registryMirrorMap }}
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+            endpoint = ["https://{{ $mirror }}"]
+          {{- end }}
+          {{- if .registryCACert }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+            ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+          {{- end }}
+          {{- if .registryAuth }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+            username = "{{.registryUsername}}"
+            password = "{{.registryPassword}}"
+          {{- end }}
+      owner: root:root
+      path: "/etc/containerd/config_append.toml"
+{{- end }}
 {{- if .awsIamAuth}}
     - content: |
         # clusters refers to the remote service.
@@ -216,6 +241,12 @@ spec:
             timeAdded: {{ .TimeAdded }}
 {{- end }}
         {{- end }}
+{{- end }}
+{{- if .registryMirrorMap }}
+    preKubeadmCommands:
+    - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+    - systemctl daemon-reload
+    - systemctl restart containerd
 {{- end }}
   replicas: {{.control_plane_replicas}}
   version: {{.kubernetesVersion}}
@@ -234,6 +265,14 @@ spec:
       version: {{.externalEtcdVersion}}
 {{- if .etcdCipherSuites }}
     cipherSuites: {{.etcdCipherSuites}}
+{{- end }}
+{{- if .registryMirrorMap }}
+    registryMirror:
+      endpoint: {{ .publicMirror }}
+      {{- if .registryCACert }}
+      caCert: |
+{{ .registryCACert | indent 8 }}
+      {{- end }}
 {{- end }}
   infrastructureTemplate:
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

pkg/providers/docker/config/template-md.yaml

@@ -26,6 +26,38 @@ spec:
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 12 }}
 {{- end }}
+{{- if .registryMirrorMap }}
+      files:
+{{- end }}
+{{- if .registryCACert }}
+      - content: |
+{{ .registryCACert | indent 10 }}
+        owner: root:root
+        path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+      - content: |
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+            {{- range $orig, $mirror := .registryMirrorMap }}
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+              endpoint = ["https://{{ $mirror }}"]
+            {{- end }}
+            {{- if .registryCACert }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+              ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+            {{- end }}
+            {{- if .registryAuth }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+              username = "{{.registryUsername}}"
+              password = "{{.registryPassword}}"
+            {{- end }}
+        owner: root:root
+        path: "/etc/containerd/config_append.toml"
+      preKubeadmCommands:
+      - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+      - systemctl daemon-reload
+      - systemctl restart containerd
+{{- end }}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment

pkg/providers/docker/docker.go

@@ -15,12 +15,15 @@ import (
 	"github.com/aws/eks-anywhere/pkg/bootstrapper"
 	"github.com/aws/eks-anywhere/pkg/cluster"
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
+	"github.com/aws/eks-anywhere/pkg/config"
 	"github.com/aws/eks-anywhere/pkg/constants"
 	"github.com/aws/eks-anywhere/pkg/crypto"
 	"github.com/aws/eks-anywhere/pkg/executables"
 	"github.com/aws/eks-anywhere/pkg/logger"
 	"github.com/aws/eks-anywhere/pkg/providers"
 	"github.com/aws/eks-anywhere/pkg/providers/common"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
+	"github.com/aws/eks-anywhere/pkg/registrymirror/containerd"
 	"github.com/aws/eks-anywhere/pkg/semver"
 	"github.com/aws/eks-anywhere/pkg/templater"
 	"github.com/aws/eks-anywhere/pkg/types"
@@ -292,6 +295,10 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 	}
 	values["auditPolicy"] = auditPolicy
 
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		values = populateRegistryMirrorValues(clusterSpec, values)
+	}
+
 	return values, nil
 }
 
@@ -320,6 +327,10 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupConfiguration
 		"autoscalingConfig":     workerNodeGroupConfiguration.AutoScalingConfiguration,
 	}
 
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		values = populateRegistryMirrorValues(clusterSpec, values)
+	}
+
 	return values, nil
 }
 
@@ -612,3 +623,24 @@ func (p *provider) PreCoreComponentsUpgrade(
 ) error {
 	return nil
 }
+
+func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]interface{}) map[string]interface{} {
+	registryMirror := registrymirror.FromCluster(clusterSpec.Cluster)
+	values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap)
+	values["mirrorBase"] = registryMirror.BaseRegistry
+	values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+	if len(registryMirror.CACertContent) > 0 {
+		values["registryCACert"] = registryMirror.CACertContent
+	}
+
+	if registryMirror.Auth {
+		values["registryAuth"] = registryMirror.Auth
+		username, password, err := config.ReadCredentials()
+		if err != nil {
+			return values
+		}
+		values["registryUsername"] = username
+		values["registryPassword"] = password
+	}
+	return values
+}

pkg/providers/docker/docker_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	_ "embed"
 	"fmt"
+	"path"
 	"testing"
 	"time"
 
@@ -28,6 +29,8 @@ import (
 	releasev1alpha1 "github.com/aws/eks-anywhere/release/api/v1alpha1"
 )
 
+const testdataDir = "testdata"
+
 type dockerTest struct {
 	*WithT
 	dockerClient *dockerMocks.MockProviderClient
@@ -47,6 +50,10 @@ func newTest(t *testing.T) *dockerTest {
 	}
 }
 
+func givenClusterSpec(t *testing.T, fileName string) *cluster.Spec {
+	return test.NewFullClusterSpec(t, path.Join(testdataDir, fileName))
+}
+
 func TestProviderUpdateKubeConfig(t *testing.T) {
 	input := []byte(`
 apiVersion: v1
@@ -818,3 +825,77 @@ func TestInvalidDockerTemplateWithControlplaneEndpoint(t *testing.T) {
 		t.Fatalf("err %v, wantErr %v", err, wantErr)
 	}
 }
+
+func TestDockerGenerateDeploymentFileWithMirrorConfig(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	ctx := context.Background()
+	client := dockerMocks.NewMockProviderClient(mockCtrl)
+	kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl)
+	provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow)
+	clusterObj := &types.Cluster{Name: "test"}
+	clusterSpec := givenClusterSpec(t, "cluster_mirror_config.yaml")
+
+	if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil {
+		t.Fatalf("failed to setup and validate: %v", err)
+	}
+
+	cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec)
+	if err != nil {
+		t.Fatalf("failed to generate cluster api spec contents: %v", err)
+	}
+
+	test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_config_cp.yaml")
+	test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_config_md.yaml")
+}
+
+func TestDockerGenerateDeploymentFileWithMirrorAndCertConfig(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	ctx := context.Background()
+	client := dockerMocks.NewMockProviderClient(mockCtrl)
+	kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl)
+	provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow)
+	clusterObj := &types.Cluster{Name: "test"}
+	clusterSpec := givenClusterSpec(t, "cluster_mirror_with_cert_config.yaml")
+
+	if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil {
+		t.Fatalf("failed to setup and validate: %v", err)
+	}
+
+	cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec)
+	if err != nil {
+		t.Fatalf("failed to generate cluster api spec contents: %v", err)
+	}
+
+	fmt.Println("CP template starts")
+	fmt.Println(string(cp))
+	fmt.Println("CP template ends")
+	fmt.Println("MD template starts")
+	fmt.Println(string(md))
+	fmt.Println("MDtemplate ends")
+	test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_with_cert_config_cp.yaml")
+	test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_with_cert_config_md.yaml")
+}
+
+func TestDockerGenerateDeploymentFileWithMirrorAndAuthConfig(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	t.Setenv("REGISTRY_USERNAME", "username")
+	t.Setenv("REGISTRY_PASSWORD", "password")
+	ctx := context.Background()
+	client := dockerMocks.NewMockProviderClient(mockCtrl)
+	kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl)
+	provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow)
+	clusterObj := &types.Cluster{Name: "test"}
+	clusterSpec := givenClusterSpec(t, "cluster_mirror_with_auth_config.yaml")
+
+	if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil {
+		t.Fatalf("failed to setup and validate: %v", err)
+	}
+
+	cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec)
+	if err != nil {
+		t.Fatalf("failed to generate cluster api spec contents: %v", err)
+	}
+
+	test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_with_auth_config_cp.yaml")
+	test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_with_auth_config_md.yaml")
+}