From c4a7ec5555377f9f8e1feaeab1e097d50c64207b Mon Sep 17 00:00:00 2001 From: Abhay Krishna Arunachalam Date: Fri, 10 Feb 2023 23:31:09 -0800 Subject: [PATCH] Add Docker airgapped test --- internal/pkg/ec2/modify.go | 26 +++++++++++++ .../tinkerbell/zz_generated.deepcopy.go | 2 - pkg/providers/docker/config/template-cp.yaml | 39 +++++++++++++++++++ pkg/providers/docker/config/template-md.yaml | 32 +++++++++++++++ pkg/providers/docker/docker.go | 32 +++++++++++++++ pkg/providers/nutanix/client.go | 3 -- pkg/providers/tinkerbell/tinkerbell.go | 2 - test/e2e/autoscaler.go | 2 +- test/e2e/constants.go | 28 ++++++------- test/e2e/docker_test.go | 12 ++++++ 10 files changed, 156 insertions(+), 22 deletions(-) create mode 100644 internal/pkg/ec2/modify.go diff --git a/internal/pkg/ec2/modify.go b/internal/pkg/ec2/modify.go new file mode 100644 index 0000000000000..c1bf708f5a41f --- /dev/null +++ b/internal/pkg/ec2/modify.go @@ -0,0 +1,26 @@ +package ec2 + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) + +func ModifyInstanceSecurityGroup(session *session.Session, instanceID, securityGroupID string) error { + service := ec2.New(session) + input := &ec2.ModifyInstanceAttributeInput{ + InstanceId: aws.String(instanceID), + Attribute: aws.String(ec2.InstanceAttributeNameGroupSet), + Groups: []*string{ + aws.String(securityGroupID), + }, + } + _, err := service.ModifyInstanceAttribute(input) + if err != nil { + return fmt.Errorf("changing security groups for instance %s: %v", instanceID, err) + } + + return nil +} diff --git a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go index 9e7676c51c7b6..2afae011c22e0 100644 --- a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go @@ -19,8 +19,6 @@ package tinkerbell -import () - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Action) DeepCopyInto(out *Action) { *out = *in diff --git a/pkg/providers/docker/config/template-cp.yaml b/pkg/providers/docker/config/template-cp.yaml index f53f9353702f2..151c2b7eb9052 100644 --- a/pkg/providers/docker/config/template-cp.yaml +++ b/pkg/providers/docker/config/template-cp.yaml @@ -138,6 +138,31 @@ spec: {{ .auditPolicy | indent 8 }} owner: root:root path: /etc/kubernetes/audit-policy.yaml +{{- if .registryCACert }} + - content: | +{{ .registryCACert | indent 8 }} + owner: root:root + path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" +{{- end }} +{{- if .registryMirrorMap }} + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + {{- range $orig, $mirror := .registryMirrorMap }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"] + endpoint = ["https://{{ $mirror }}"] + {{- end }} + {{- if .registryCACert }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls] + ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" + {{- end }} + {{- if .registryAuth }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth] + username = "{{.registryUsername}}" + password = "{{.registryPassword}}" + {{- end }} + owner: root:root + path: "/etc/containerd/config_append.toml" +{{- end }} {{- if .awsIamAuth}} - content: | # clusters refers to the remote service. @@ -216,6 +241,12 @@ spec: timeAdded: {{ .TimeAdded }} {{- end }} {{- end }} +{{- end }} +{{- if .registryMirrorMap }} + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd {{- end }} replicas: {{.control_plane_replicas}} version: {{.kubernetesVersion}} @@ -234,6 +265,14 @@ spec: version: {{.externalEtcdVersion}} {{- if .etcdCipherSuites }} cipherSuites: {{.etcdCipherSuites}} +{{- end }} +{{- if .registryMirrorMap }} + registryMirror: + endpoint: {{ .publicMirror }} + {{- if .registryCACert }} + caCert: | +{{ .registryCACert | indent 8 }} + {{- end }} {{- end }} infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/pkg/providers/docker/config/template-md.yaml b/pkg/providers/docker/config/template-md.yaml index 71d2536c88320..b49719e2d717e 100644 --- a/pkg/providers/docker/config/template-md.yaml +++ b/pkg/providers/docker/config/template-md.yaml @@ -26,6 +26,38 @@ spec: {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 12 }} {{- end }} +{{- if .registryMirrorMap }} + files: +{{- end }} +{{- if .registryCACert }} + - content: | +{{ .registryCACert | indent 10 }} + owner: root:root + path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" +{{- end }} +{{- if .registryMirrorMap }} + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + {{- range $orig, $mirror := .registryMirrorMap }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"] + endpoint = ["https://{{ $mirror }}"] + {{- end }} + {{- if .registryCACert }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls] + ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" + {{- end }} + {{- if .registryAuth }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth] + username = "{{.registryUsername}}" + password = "{{.registryPassword}}" + {{- end }} + owner: root:root + path: "/etc/containerd/config_append.toml" + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd +{{- end }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment diff --git a/pkg/providers/docker/docker.go b/pkg/providers/docker/docker.go index aebb7f44ff273..c852b898eb5a1 100644 --- a/pkg/providers/docker/docker.go +++ b/pkg/providers/docker/docker.go @@ -15,12 +15,15 @@ import ( "github.com/aws/eks-anywhere/pkg/bootstrapper" "github.com/aws/eks-anywhere/pkg/cluster" "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/config" "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/crypto" "github.com/aws/eks-anywhere/pkg/executables" "github.com/aws/eks-anywhere/pkg/logger" "github.com/aws/eks-anywhere/pkg/providers" "github.com/aws/eks-anywhere/pkg/providers/common" + "github.com/aws/eks-anywhere/pkg/registrymirror" + "github.com/aws/eks-anywhere/pkg/registrymirror/containerd" "github.com/aws/eks-anywhere/pkg/semver" "github.com/aws/eks-anywhere/pkg/templater" "github.com/aws/eks-anywhere/pkg/types" @@ -292,6 +295,10 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro } values["auditPolicy"] = auditPolicy + if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil { + values = populateRegistryMirrorValues(clusterSpec, values) + } + return values, nil } @@ -320,6 +327,10 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupConfiguration "autoscalingConfig": workerNodeGroupConfiguration.AutoScalingConfiguration, } + if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil { + values = populateRegistryMirrorValues(clusterSpec, values) + } + return values, nil } @@ -612,3 +623,24 @@ func (p *provider) PreCoreComponentsUpgrade( ) error { return nil } + +func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]interface{}) map[string]interface{} { + registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) + values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) + values["mirrorBase"] = registryMirror.BaseRegistry + values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) + if len(registryMirror.CACertContent) > 0 { + values["registryCACert"] = registryMirror.CACertContent + } + + if registryMirror.Auth { + values["registryAuth"] = registryMirror.Auth + username, password, err := config.ReadCredentials() + if err != nil { + return values + } + values["registryUsername"] = username + values["registryPassword"] = password + } + return values +} diff --git a/pkg/providers/nutanix/client.go b/pkg/providers/nutanix/client.go index cb4c89249f731..bf98de9e89857 100644 --- a/pkg/providers/nutanix/client.go +++ b/pkg/providers/nutanix/client.go @@ -9,12 +9,9 @@ import ( type Client interface { GetSubnet(ctx context.Context, uuid string) (*v3.SubnetIntentResponse, error) ListSubnet(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.SubnetListIntentResponse, error) - GetImage(ctx context.Context, uuid string) (*v3.ImageIntentResponse, error) ListImage(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ImageListIntentResponse, error) - GetCluster(ctx context.Context, uuid string) (*v3.ClusterIntentResponse, error) ListCluster(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ClusterListIntentResponse, error) - GetCurrentLoggedInUser(ctx context.Context) (*v3.UserIntentResponse, error) } diff --git a/pkg/providers/tinkerbell/tinkerbell.go b/pkg/providers/tinkerbell/tinkerbell.go index c5f7403917dcd..811f5ac74916c 100644 --- a/pkg/providers/tinkerbell/tinkerbell.go +++ b/pkg/providers/tinkerbell/tinkerbell.go @@ -88,10 +88,8 @@ type ProviderKubectlClient interface { WaitForRufioMachines(ctx context.Context, cluster *types.Cluster, timeout string, condition string, namespace string) error SearchTinkerbellMachineConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellMachineConfig, error) SearchTinkerbellDatacenterConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellDatacenterConfig, error) - AllTinkerbellHardware(ctx context.Context, kuebconfig string) ([]tinkv1alpha1.Hardware, error) AllBaseboardManagements(ctx context.Context, kubeconfig string) ([]rufiounreleased.BaseboardManagement, error) - HasCRD(ctx context.Context, kubeconfig, crd string) (bool, error) DeleteCRD(ctx context.Context, kubeconfig, crd string) error } diff --git a/test/e2e/autoscaler.go b/test/e2e/autoscaler.go index 65519eb5ea596..6af7fe69ddedb 100644 --- a/test/e2e/autoscaler.go +++ b/test/e2e/autoscaler.go @@ -15,4 +15,4 @@ func runAutoscalerWitMetricsServerSimpleFlow(test *framework.ClusterE2ETest) { test.InstallAutoScalerWithMetricServer(targetNamespace) test.CombinedAutoScalerMetricServerTest(autoscalerName, metricServerName, targetNamespace, withMgmtCluster(test)) }) -} \ No newline at end of file +} diff --git a/test/e2e/constants.go b/test/e2e/constants.go index e2eedab6f45ce..649f9d68950f9 100644 --- a/test/e2e/constants.go +++ b/test/e2e/constants.go @@ -11,25 +11,25 @@ const ( EksaPackageControllerHelmURI = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages" EksaPackageControllerHelmVersion = "0.2.20-eks-a-v0.0.0-dev-build.4894" EksaPackageBundleURI = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages-bundles" - EksaPackagesNamespace = "eksa-packages" + EksaPackagesNamespace = "eksa-packages" clusterNamespace = "test-namespace" - key1 = framework.LabelPrefix + "/" + "key1" - key2 = framework.LabelPrefix + "/" + "key2" - cpKey1 = framework.LabelPrefix + "/" + "cp-key1" - val1 = "val1" - val2 = "val2" - cpVal1 = "cp-val1" + key1 = framework.LabelPrefix + "/" + "key1" + key2 = framework.LabelPrefix + "/" + "key2" + cpKey1 = framework.LabelPrefix + "/" + "cp-key1" + val1 = "val1" + val2 = "val2" + cpVal1 = "cp-val1" nodeGroupLabel1 = "md-0" nodeGroupLabel2 = "md-1" - worker0 = "worker-0" - worker1 = "worker-1" - worker2 = "worker-2" + worker0 = "worker-0" + worker1 = "worker-1" + worker2 = "worker-2" - fluxUserProvidedBranch = "testbranch" - fluxUserProvidedNamespace = "testns" - fluxUserProvidedPath = "test/testerson" + fluxUserProvidedBranch = "testbranch" + fluxUserProvidedNamespace = "testns" + fluxUserProvidedPath = "test/testerson" vsphereCpVmNumCpuUpdateVar = 4 vsphereCpVmMemoryUpdate = 16384 @@ -44,4 +44,4 @@ const ( vsphereResourcePoolVar = "T_VSPHERE_RESOURCE_POOL" ) -var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"} \ No newline at end of file +var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"} diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go index baaeff95685c3..932e1f0a198c4 100644 --- a/test/e2e/docker_test.go +++ b/test/e2e/docker_test.go @@ -9,6 +9,7 @@ import ( "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/test/framework" "github.com/stretchr/testify/suite" corev1 "k8s.io/api/core/v1" @@ -369,6 +370,17 @@ func TestDockerKubernetes125OIDC(t *testing.T) { runOIDCFlow(test) } +func TestDockerKubernetes125RegistryMirrorAndCert(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube125)), + framework.WithRegistryMirrorEndpointAndCert(constants.DockerProviderName), + ) + runRegistryMirrorConfigFlow(test) +} + // Simple flow func TestDockerKubernetes121SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest(