Add Docker airgapped test

cmd/eksctl-anywhere/cmd/listimages.go

@@ -11,7 +11,7 @@ import (
 )
 
 type listImagesOptions struct {
-	fileName string
+	fileName        string
 	bundlesOverride string
 }
 

cmd/eksctl-anywhere/cmd/listovas.go

@@ -19,7 +19,7 @@ import (
 )
 
 type listOvasOptions struct {
-	fileName string
+	fileName        string
 	bundlesOverride string
 }
 

cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml

@@ -29,6 +29,10 @@ phases:
       - ${CODEBUILD_SRC_DIR}/cmd/integration_test/build/script/start_docker.sh
   build:
     commands:
+      - make build-integration-test-binary
+      - make build-eks-a-for-e2e
+      - make e2e-tests-binary E2E_TAGS="e2e docker" E2E_OUTPUT_FILE=bin/docker/e2e.test
+      -  curl https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/bundle-release.yaml -o bin/local-bundle-release.yaml
       - export JOB_ID=$CODEBUILD_BUILD_ID
       - BUNDLES_OVERRIDE=false
       - |
@@ -45,7 +49,7 @@ phases:
         -i ${INTEGRATION_TEST_INSTANCE_PROFILE}
         -m ${INTEGRATION_TEST_MAX_EC2_COUNT}
         -p ${INTEGRATION_TEST_MAX_CONCURRENT_TEST_COUNT}
-        -r 'Test'
+        -r 'TestDockerKubernetes125RegistryMirrorAndCert'
         -v 4
         --skip ${SKIPPED_TESTS}
         --bundles-override=${BUNDLES_OVERRIDE}

pkg/providers/docker/config/template-cp.yaml

@@ -138,6 +138,31 @@ spec:
 {{ .auditPolicy | indent 8 }}
       owner: root:root
       path: /etc/kubernetes/audit-policy.yaml
+{{- if .registryCACert }}
+    - content: |
+{{ .registryCACert | indent 8 }}
+      owner: root:root
+      path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+    - content: |
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+          {{- range $orig, $mirror := .registryMirrorMap }}
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+            endpoint = ["https://{{ $mirror }}"]
+          {{- end }}
+          {{- if .registryCACert }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+            ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+          {{- end }}
+          {{- if .registryAuth }}
+          [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+            username = "{{.registryUsername}}"
+            password = "{{.registryPassword}}"
+          {{- end }}
+      owner: root:root
+      path: "/etc/containerd/config_append.toml"
+{{- end }}
 {{- if .awsIamAuth}}
     - content: |
         # clusters refers to the remote service.
@@ -216,6 +241,12 @@ spec:
             timeAdded: {{ .TimeAdded }}
 {{- end }}
         {{- end }}
+{{- end }}
+{{- if .registryMirrorMap }}
+    preKubeadmCommands:
+    - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+    - systemctl daemon-reload
+    - systemctl restart containerd
 {{- end }}
   replicas: {{.control_plane_replicas}}
   version: {{.kubernetesVersion}}
@@ -234,6 +265,14 @@ spec:
       version: {{.externalEtcdVersion}}
 {{- if .etcdCipherSuites }}
     cipherSuites: {{.etcdCipherSuites}}
+{{- end }}
+{{- if .registryMirrorMap }}
+    registryMirror:
+      endpoint: {{ .publicMirror }}
+      {{- if .registryCACert }}
+      caCert: |
+{{ .registryCACert | indent 8 }}
+      {{- end }}
 {{- end }}
   infrastructureTemplate:
     apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

pkg/providers/docker/config/template-md.yaml

@@ -26,6 +26,38 @@ spec:
 {{- if .kubeletExtraArgs }}
 {{ .kubeletExtraArgs.ToYaml | indent 12 }}
 {{- end }}
+{{- if .registryMirrorMap }}
+      files:
+{{- end }}
+{{- if .registryCACert }}
+      - content: |
+{{ .registryCACert | indent 10 }}
+        owner: root:root
+        path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+{{- end }}
+{{- if .registryMirrorMap }}
+      - content: |
+          [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+            {{- range $orig, $mirror := .registryMirrorMap }}
+            [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"]
+              endpoint = ["https://{{ $mirror }}"]
+            {{- end }}
+            {{- if .registryCACert }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls]
+              ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt"
+            {{- end }}
+            {{- if .registryAuth }}
+            [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth]
+              username = "{{.registryUsername}}"
+              password = "{{.registryPassword}}"
+            {{- end }}
+        owner: root:root
+        path: "/etc/containerd/config_append.toml"
+      preKubeadmCommands:
+      - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml
+      - systemctl daemon-reload
+      - systemctl restart containerd
+{{- end }}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment

pkg/providers/docker/docker.go

@@ -15,12 +15,15 @@ import (
 	"github.com/aws/eks-anywhere/pkg/bootstrapper"
 	"github.com/aws/eks-anywhere/pkg/cluster"
 	"github.com/aws/eks-anywhere/pkg/clusterapi"
+	"github.com/aws/eks-anywhere/pkg/config"
 	"github.com/aws/eks-anywhere/pkg/constants"
 	"github.com/aws/eks-anywhere/pkg/crypto"
 	"github.com/aws/eks-anywhere/pkg/executables"
 	"github.com/aws/eks-anywhere/pkg/logger"
 	"github.com/aws/eks-anywhere/pkg/providers"
 	"github.com/aws/eks-anywhere/pkg/providers/common"
+	"github.com/aws/eks-anywhere/pkg/registrymirror"
+	"github.com/aws/eks-anywhere/pkg/registrymirror/containerd"
 	"github.com/aws/eks-anywhere/pkg/semver"
 	"github.com/aws/eks-anywhere/pkg/templater"
 	"github.com/aws/eks-anywhere/pkg/types"
@@ -292,6 +295,10 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 	}
 	values["auditPolicy"] = auditPolicy
 
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		values = populateRegistryMirrorValues(clusterSpec, values)
+	}
+
 	return values, nil
 }
 
@@ -320,6 +327,10 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupConfiguration
 		"autoscalingConfig":     workerNodeGroupConfiguration.AutoScalingConfiguration,
 	}
 
+	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {
+		values = populateRegistryMirrorValues(clusterSpec, values)
+	}
+
 	return values, nil
 }
 
@@ -612,3 +623,24 @@ func (p *provider) PreCoreComponentsUpgrade(
 ) error {
 	return nil
 }
+
+func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]interface{}) map[string]interface{} {
+	registryMirror := registrymirror.FromCluster(clusterSpec.Cluster)
+	values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap)
+	values["mirrorBase"] = registryMirror.BaseRegistry
+	values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror())
+	if len(registryMirror.CACertContent) > 0 {
+		values["registryCACert"] = registryMirror.CACertContent
+	}
+
+	if registryMirror.Auth {
+		values["registryAuth"] = registryMirror.Auth
+		username, password, err := config.ReadCredentials()
+		if err != nil {
+			return values
+		}
+		values["registryUsername"] = username
+		values["registryPassword"] = password
+	}
+	return values
+}

pkg/providers/nutanix/client.go

@@ -9,12 +9,9 @@ import (
 type Client interface {
 	GetSubnet(ctx context.Context, uuid string) (*v3.SubnetIntentResponse, error)
 	ListSubnet(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.SubnetListIntentResponse, error)
-
 	GetImage(ctx context.Context, uuid string) (*v3.ImageIntentResponse, error)
 	ListImage(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ImageListIntentResponse, error)
-
 	GetCluster(ctx context.Context, uuid string) (*v3.ClusterIntentResponse, error)
 	ListCluster(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ClusterListIntentResponse, error)
-
 	GetCurrentLoggedInUser(ctx context.Context) (*v3.UserIntentResponse, error)
 }

pkg/providers/tinkerbell/tinkerbell.go

@@ -87,10 +87,8 @@ type ProviderKubectlClient interface {
 	WaitForRufioMachines(ctx context.Context, cluster *types.Cluster, timeout string, condition string, namespace string) error
 	SearchTinkerbellMachineConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellMachineConfig, error)
 	SearchTinkerbellDatacenterConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellDatacenterConfig, error)
-
 	AllTinkerbellHardware(ctx context.Context, kuebconfig string) ([]tinkv1alpha1.Hardware, error)
 	AllBaseboardManagements(ctx context.Context, kubeconfig string) ([]rufiounreleased.BaseboardManagement, error)
-
 	HasCRD(ctx context.Context, kubeconfig, crd string) (bool, error)
 	DeleteCRD(ctx context.Context, kubeconfig, crd string) error
 }

test/e2e/airgap.go

@@ -22,3 +22,15 @@ func runVSphereAirgapConfigFlow(test *framework.ClusterE2ETest, localCIDRs strin
 	)
 	test.DeleteCluster()
 }
+
+func runDockerAirgapConfigFlow(test *framework.ClusterE2ETest) {
+	test.GenerateClusterConfig()
+	test.DownloadArtifacts()
+	test.ExtractDownloadedArtifacts()
+	test.DownloadImages()
+	test.ImportImages()
+	test.ChangeInstanceSecurityGroup("sg-0f3b5c1368db406ee")
+	test.CreateCluster(framework.WithBundlesOverride("./eks-anywhere-downloads/bundle-release.yaml"))
+	test.DeleteCluster()
+	test.ChangeInstanceSecurityGroup("sg-ff4bbacb")
+}

test/e2e/autoscaler.go

@@ -15,4 +15,4 @@ func runAutoscalerWitMetricsServerSimpleFlow(test *framework.ClusterE2ETest) {
 		test.InstallAutoScalerWithMetricServer(targetNamespace)
 		test.CombinedAutoScalerMetricServerTest(autoscalerName, metricServerName, targetNamespace, withMgmtCluster(test))
 	})
-}
+}

test/e2e/constants.go

@@ -1,4 +1,4 @@
-//nolint
+// nolint
 package e2e
 
 import (
@@ -11,25 +11,25 @@ const (
 	EksaPackageControllerHelmURI       = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages"
 	EksaPackageControllerHelmVersion   = "0.2.20-eks-a-v0.0.0-dev-build.4894"
 	EksaPackageBundleURI               = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages-bundles"
-	EksaPackagesNamespace = "eksa-packages"
+	EksaPackagesNamespace              = "eksa-packages"
 
 	clusterNamespace = "test-namespace"
 
-	key1    = framework.LabelPrefix + "/" + "key1"
-	key2    = framework.LabelPrefix + "/" + "key2"
-	cpKey1  = framework.LabelPrefix + "/" + "cp-key1"
-	val1    = "val1"
-	val2    = "val2"
-	cpVal1  = "cp-val1"
+	key1            = framework.LabelPrefix + "/" + "key1"
+	key2            = framework.LabelPrefix + "/" + "key2"
+	cpKey1          = framework.LabelPrefix + "/" + "cp-key1"
+	val1            = "val1"
+	val2            = "val2"
+	cpVal1          = "cp-val1"
 	nodeGroupLabel1 = "md-0"
 	nodeGroupLabel2 = "md-1"
-	worker0 = "worker-0"
-	worker1 = "worker-1"
-	worker2 = "worker-2"
+	worker0         = "worker-0"
+	worker1         = "worker-1"
+	worker2         = "worker-2"
 
-	fluxUserProvidedBranch             = "testbranch"
-	fluxUserProvidedNamespace          = "testns"
-	fluxUserProvidedPath               = "test/testerson"
+	fluxUserProvidedBranch    = "testbranch"
+	fluxUserProvidedNamespace = "testns"
+	fluxUserProvidedPath      = "test/testerson"
 
 	vsphereCpVmNumCpuUpdateVar          = 4
 	vsphereCpVmMemoryUpdate             = 16384
@@ -44,4 +44,4 @@ const (
 	vsphereResourcePoolVar              = "T_VSPHERE_RESOURCE_POOL"
 )
 
-var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"}
+var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"}

test/e2e/docker_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/eks-anywhere/internal/pkg/api"
 	"github.com/aws/eks-anywhere/pkg/api/v1alpha1"
+	"github.com/aws/eks-anywhere/pkg/constants"
 	"github.com/aws/eks-anywhere/test/framework"
 	"github.com/stretchr/testify/suite"
 	corev1 "k8s.io/api/core/v1"
@@ -369,6 +370,17 @@ func TestDockerKubernetes125OIDC(t *testing.T) {
 	runOIDCFlow(test)
 }
 
+func TestDockerKubernetes125RegistryMirrorAndCert(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewDocker(t),
+		framework.WithClusterFiller(api.WithExternalEtcdTopology(1)),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube125)),
+		framework.WithRegistryMirrorEndpointAndCert(constants.DockerProviderName),
+	)
+	runRegistryMirrorConfigFlow(test)
+}
+
 // Simple flow
 func TestDockerKubernetes121SimpleFlow(t *testing.T) {
 	test := framework.NewClusterE2ETest(

test/e2e/registrymirror.go

@@ -9,14 +9,19 @@ import (
 
 func runRegistryMirrorConfigFlow(test *framework.ClusterE2ETest) {
 	test.GenerateClusterConfig()
+	test.DownloadArtifacts()
+	test.ExtractDownloadedArtifacts()
+	test.DownloadImages()
 	test.ImportImages()
 	test.CreateCluster()
-	test.ImportImages()
 	test.DeleteCluster()
 }
 
 func runTinkerbellRegistryMirrorFlow(test *framework.ClusterE2ETest) {
 	test.GenerateClusterConfig()
+	test.DownloadArtifacts()
+	test.ExtractDownloadedArtifacts()
+	test.DownloadImages()
 	test.ImportImages()
 	test.GenerateHardwareConfig()
 	test.PowerOffHardware()