From e38730e1a4aa844350b280d2afaa93556f67cff5 Mon Sep 17 00:00:00 2001 From: Abhay Krishna Arunachalam Date: Tue, 21 Feb 2023 16:51:37 -0800 Subject: [PATCH] Add Docker airgapped test --- cmd/eksctl-anywhere/cmd/listimages.go | 2 +- cmd/eksctl-anywhere/cmd/listovas.go | 2 +- .../buildspecs/docker-test-eks-a-cli.yml | 2 + .../internal/endpoints/v2/endpoints.go | 3 +- internal/test/cleanup/cleanup.go | 1 - internal/test/e2e/registryMirror.go | 12 + internal/test/e2e/setup.go | 2 +- .../tinkerbell/zz_generated.deepcopy.go | 2 - pkg/api/v1alpha1/zz_generated.deepcopy.go | 3 +- pkg/providers/docker/config/template-cp.yaml | 39 ++ pkg/providers/docker/config/template-md.yaml | 32 ++ pkg/providers/docker/docker.go | 32 ++ pkg/providers/docker/docker_test.go | 81 ++++ .../testdata/cluster_mirror_config.yaml | 40 ++ .../cluster_mirror_with_auth_config.yaml | 59 +++ .../cluster_mirror_with_cert_config.yaml | 58 +++ .../expected_results_mirror_config_cp.yaml | 325 +++++++++++++++ .../expected_results_mirror_config_md.yaml | 68 ++++ ...ed_results_mirror_with_auth_config_cp.yaml | 370 ++++++++++++++++++ ...ed_results_mirror_with_auth_config_md.yaml | 94 +++++ ...ed_results_mirror_with_cert_config_cp.yaml | 367 +++++++++++++++++ ...ed_results_mirror_with_cert_config_md.yaml | 91 +++++ pkg/providers/nutanix/client.go | 3 - pkg/providers/tinkerbell/tinkerbell.go | 2 - test/e2e/airgap.go | 17 +- test/e2e/autoscaler.go | 2 +- test/e2e/constants.go | 30 +- test/e2e/docker_test.go | 29 +- test/e2e/metallb.go | 3 +- test/e2e/registrymirror.go | 7 +- test/e2e/snow_test.go | 3 +- test/e2e/tinkerbell_test.go | 3 +- test/e2e/vsphere_test.go | 3 +- test/framework/cluster.go | 89 +++-- test/framework/registryMirror.go | 30 +- 35 files changed, 1830 insertions(+), 76 deletions(-) create mode 100644 pkg/providers/docker/testdata/cluster_mirror_config.yaml create mode 100644 pkg/providers/docker/testdata/cluster_mirror_with_auth_config.yaml create mode 100644 pkg/providers/docker/testdata/cluster_mirror_with_cert_config.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_config_cp.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_config_md.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_cp.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_md.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_cp.yaml create mode 100644 pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_md.yaml diff --git a/cmd/eksctl-anywhere/cmd/listimages.go b/cmd/eksctl-anywhere/cmd/listimages.go index 872e54863222..7f31d8700129 100644 --- a/cmd/eksctl-anywhere/cmd/listimages.go +++ b/cmd/eksctl-anywhere/cmd/listimages.go @@ -11,7 +11,7 @@ import ( ) type listImagesOptions struct { - fileName string + fileName string bundlesOverride string } diff --git a/cmd/eksctl-anywhere/cmd/listovas.go b/cmd/eksctl-anywhere/cmd/listovas.go index 9b55b207f28c..2c0d9a6bdfa9 100644 --- a/cmd/eksctl-anywhere/cmd/listovas.go +++ b/cmd/eksctl-anywhere/cmd/listovas.go @@ -19,7 +19,7 @@ import ( ) type listOvasOptions struct { - fileName string + fileName string bundlesOverride string } diff --git a/cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml b/cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml index 9ecf0af3a94d..8dc0edd9582f 100644 --- a/cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml +++ b/cmd/integration_test/build/buildspecs/docker-test-eks-a-cli.yml @@ -20,6 +20,8 @@ env: T_REGISTRY_MIRROR_USERNAME: "harbor-registry-data:username" T_REGISTRY_MIRROR_PASSWORD: "harbor-registry-data:password" T_REGISTRY_MIRROR_CA_CERT: "harbor-registry-data:caCert" + T_REGISTRY_MIRROR_DEFAULT_SECURITY_GROUP: "harbor-registry-data:default_sg_id" + T_REGISTRY_MIRROR_AIRGAPPED_SECURITY_GROUP: "harbor-registry-data:airgapped_sg_id" T_AWS_IAM_ROLE_ARN: "aws-iam-auth-role:ec2_role_arn" phases: pre_build: diff --git a/internal/aws-sdk-go-v2/internal/endpoints/v2/endpoints.go b/internal/aws-sdk-go-v2/internal/endpoints/v2/endpoints.go index 120d044549bc..395b785eae82 100644 --- a/internal/aws-sdk-go-v2/internal/endpoints/v2/endpoints.go +++ b/internal/aws-sdk-go-v2/internal/endpoints/v2/endpoints.go @@ -5,9 +5,8 @@ import ( "regexp" "strings" - "github.com/aws/smithy-go/logging" - "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/smithy-go/logging" ) // DefaultKey is a compound map key of a variant and other values. diff --git a/internal/test/cleanup/cleanup.go b/internal/test/cleanup/cleanup.go index 56192d654246..fb2a42db833d 100644 --- a/internal/test/cleanup/cleanup.go +++ b/internal/test/cleanup/cleanup.go @@ -7,7 +7,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws/session" - prismgoclient "github.com/nutanix-cloud-native/prism-go-client" v3 "github.com/nutanix-cloud-native/prism-go-client/v3" diff --git a/internal/test/e2e/registryMirror.go b/internal/test/e2e/registryMirror.go index 44028f35c224..efb31d223d4c 100644 --- a/internal/test/e2e/registryMirror.go +++ b/internal/test/e2e/registryMirror.go @@ -43,6 +43,18 @@ func (e *E2ESession) setupRegistryMirrorEnv(testRegex string) error { return e.mountRegistryCert(caCert, net.JoinHostPort(endpoint, port)) } + re = regexp.MustCompile(`^.*Docker.*Airgapped.*$`) + if re.MatchString(testRegex) { + err := os.Setenv("DEFAULT_SECURITY_GROUP", e.testEnvVars[e2etests.RegistryMirrorDefaultSecurityGroup]) + if err != nil { + return fmt.Errorf("unable to set DEFAULT_SECURITY_GROUP: %v", err) + } + err = os.Setenv("AIRGAPPED_SECURITY_GROUP", e.testEnvVars[e2etests.RegistryMirrorAirgappedSecurityGroup]) + if err != nil { + return fmt.Errorf("unable to set AIRGAPPED_SECURITY_GROUP: %v", err) + } + } + return nil } diff --git a/internal/test/e2e/setup.go b/internal/test/e2e/setup.go index aa8f865af03d..658ca2e371d2 100644 --- a/internal/test/e2e/setup.go +++ b/internal/test/e2e/setup.go @@ -177,7 +177,7 @@ func (e *E2ESession) updateFSInotifyResources() error { if err := ssm.Run(e.session, logr.Discard(), e.instanceId, command); err != nil { return fmt.Errorf("updating fs inotify resources: %v", err) } - e.logger.V(1).Info("Successfully updates the fs inotify user watches and instances") + e.logger.V(1).Info("Successfully updated the fs inotify user watches and instances") return nil } diff --git a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go index 9e7676c51c7b..2afae011c22e 100644 --- a/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/thirdparty/tinkerbell/zz_generated.deepcopy.go @@ -19,8 +19,6 @@ package tinkerbell -import () - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Action) DeepCopyInto(out *Action) { *out = *in diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 4aa9e1bae674..23cb549f25dc 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -20,10 +20,11 @@ package v1alpha1 import ( - apiv1beta1 "github.com/aws/eks-anywhere/pkg/providers/snow/api/v1beta1" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" + + apiv1beta1 "github.com/aws/eks-anywhere/pkg/providers/snow/api/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. diff --git a/pkg/providers/docker/config/template-cp.yaml b/pkg/providers/docker/config/template-cp.yaml index f53f9353702f..151c2b7eb905 100644 --- a/pkg/providers/docker/config/template-cp.yaml +++ b/pkg/providers/docker/config/template-cp.yaml @@ -138,6 +138,31 @@ spec: {{ .auditPolicy | indent 8 }} owner: root:root path: /etc/kubernetes/audit-policy.yaml +{{- if .registryCACert }} + - content: | +{{ .registryCACert | indent 8 }} + owner: root:root + path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" +{{- end }} +{{- if .registryMirrorMap }} + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + {{- range $orig, $mirror := .registryMirrorMap }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"] + endpoint = ["https://{{ $mirror }}"] + {{- end }} + {{- if .registryCACert }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls] + ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" + {{- end }} + {{- if .registryAuth }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth] + username = "{{.registryUsername}}" + password = "{{.registryPassword}}" + {{- end }} + owner: root:root + path: "/etc/containerd/config_append.toml" +{{- end }} {{- if .awsIamAuth}} - content: | # clusters refers to the remote service. @@ -216,6 +241,12 @@ spec: timeAdded: {{ .TimeAdded }} {{- end }} {{- end }} +{{- end }} +{{- if .registryMirrorMap }} + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd {{- end }} replicas: {{.control_plane_replicas}} version: {{.kubernetesVersion}} @@ -234,6 +265,14 @@ spec: version: {{.externalEtcdVersion}} {{- if .etcdCipherSuites }} cipherSuites: {{.etcdCipherSuites}} +{{- end }} +{{- if .registryMirrorMap }} + registryMirror: + endpoint: {{ .publicMirror }} + {{- if .registryCACert }} + caCert: | +{{ .registryCACert | indent 8 }} + {{- end }} {{- end }} infrastructureTemplate: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 diff --git a/pkg/providers/docker/config/template-md.yaml b/pkg/providers/docker/config/template-md.yaml index 71d2536c8832..b49719e2d717 100644 --- a/pkg/providers/docker/config/template-md.yaml +++ b/pkg/providers/docker/config/template-md.yaml @@ -26,6 +26,38 @@ spec: {{- if .kubeletExtraArgs }} {{ .kubeletExtraArgs.ToYaml | indent 12 }} {{- end }} +{{- if .registryMirrorMap }} + files: +{{- end }} +{{- if .registryCACert }} + - content: | +{{ .registryCACert | indent 10 }} + owner: root:root + path: "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" +{{- end }} +{{- if .registryMirrorMap }} + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + {{- range $orig, $mirror := .registryMirrorMap }} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ $orig }}"] + endpoint = ["https://{{ $mirror }}"] + {{- end }} + {{- if .registryCACert }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".tls] + ca_file = "/etc/containerd/certs.d/{{ .mirrorBase }}/ca.crt" + {{- end }} + {{- if .registryAuth }} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ .mirrorBase }}".auth] + username = "{{.registryUsername}}" + password = "{{.registryPassword}}" + {{- end }} + owner: root:root + path: "/etc/containerd/config_append.toml" + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd +{{- end }} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment diff --git a/pkg/providers/docker/docker.go b/pkg/providers/docker/docker.go index aebb7f44ff27..c852b898eb5a 100644 --- a/pkg/providers/docker/docker.go +++ b/pkg/providers/docker/docker.go @@ -15,12 +15,15 @@ import ( "github.com/aws/eks-anywhere/pkg/bootstrapper" "github.com/aws/eks-anywhere/pkg/cluster" "github.com/aws/eks-anywhere/pkg/clusterapi" + "github.com/aws/eks-anywhere/pkg/config" "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/crypto" "github.com/aws/eks-anywhere/pkg/executables" "github.com/aws/eks-anywhere/pkg/logger" "github.com/aws/eks-anywhere/pkg/providers" "github.com/aws/eks-anywhere/pkg/providers/common" + "github.com/aws/eks-anywhere/pkg/registrymirror" + "github.com/aws/eks-anywhere/pkg/registrymirror/containerd" "github.com/aws/eks-anywhere/pkg/semver" "github.com/aws/eks-anywhere/pkg/templater" "github.com/aws/eks-anywhere/pkg/types" @@ -292,6 +295,10 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro } values["auditPolicy"] = auditPolicy + if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil { + values = populateRegistryMirrorValues(clusterSpec, values) + } + return values, nil } @@ -320,6 +327,10 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupConfiguration "autoscalingConfig": workerNodeGroupConfiguration.AutoScalingConfiguration, } + if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil { + values = populateRegistryMirrorValues(clusterSpec, values) + } + return values, nil } @@ -612,3 +623,24 @@ func (p *provider) PreCoreComponentsUpgrade( ) error { return nil } + +func populateRegistryMirrorValues(clusterSpec *cluster.Spec, values map[string]interface{}) map[string]interface{} { + registryMirror := registrymirror.FromCluster(clusterSpec.Cluster) + values["registryMirrorMap"] = containerd.ToAPIEndpoints(registryMirror.NamespacedRegistryMap) + values["mirrorBase"] = registryMirror.BaseRegistry + values["publicMirror"] = containerd.ToAPIEndpoint(registryMirror.CoreEKSAMirror()) + if len(registryMirror.CACertContent) > 0 { + values["registryCACert"] = registryMirror.CACertContent + } + + if registryMirror.Auth { + values["registryAuth"] = registryMirror.Auth + username, password, err := config.ReadCredentials() + if err != nil { + return values + } + values["registryUsername"] = username + values["registryPassword"] = password + } + return values +} diff --git a/pkg/providers/docker/docker_test.go b/pkg/providers/docker/docker_test.go index a3afb9b783a8..abe986c8381b 100644 --- a/pkg/providers/docker/docker_test.go +++ b/pkg/providers/docker/docker_test.go @@ -4,6 +4,7 @@ import ( "context" _ "embed" "fmt" + "path" "testing" "time" @@ -28,6 +29,8 @@ import ( releasev1alpha1 "github.com/aws/eks-anywhere/release/api/v1alpha1" ) +const testdataDir = "testdata" + type dockerTest struct { *WithT dockerClient *dockerMocks.MockProviderClient @@ -47,6 +50,10 @@ func newTest(t *testing.T) *dockerTest { } } +func givenClusterSpec(t *testing.T, fileName string) *cluster.Spec { + return test.NewFullClusterSpec(t, path.Join(testdataDir, fileName)) +} + func TestProviderUpdateKubeConfig(t *testing.T) { input := []byte(` apiVersion: v1 @@ -818,3 +825,77 @@ func TestInvalidDockerTemplateWithControlplaneEndpoint(t *testing.T) { t.Fatalf("err %v, wantErr %v", err, wantErr) } } + +func TestDockerGenerateDeploymentFileWithMirrorConfig(t *testing.T) { + mockCtrl := gomock.NewController(t) + ctx := context.Background() + client := dockerMocks.NewMockProviderClient(mockCtrl) + kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl) + provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow) + clusterObj := &types.Cluster{Name: "test"} + clusterSpec := givenClusterSpec(t, "cluster_mirror_config.yaml") + + if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil { + t.Fatalf("failed to setup and validate: %v", err) + } + + cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec) + if err != nil { + t.Fatalf("failed to generate cluster api spec contents: %v", err) + } + + test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_config_cp.yaml") + test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_config_md.yaml") +} + +func TestDockerGenerateDeploymentFileWithMirrorAndCertConfig(t *testing.T) { + mockCtrl := gomock.NewController(t) + ctx := context.Background() + client := dockerMocks.NewMockProviderClient(mockCtrl) + kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl) + provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow) + clusterObj := &types.Cluster{Name: "test"} + clusterSpec := givenClusterSpec(t, "cluster_mirror_with_cert_config.yaml") + + if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil { + t.Fatalf("failed to setup and validate: %v", err) + } + + cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec) + if err != nil { + t.Fatalf("failed to generate cluster api spec contents: %v", err) + } + + fmt.Println("CP template starts") + fmt.Println(string(cp)) + fmt.Println("CP template ends") + fmt.Println("MD template starts") + fmt.Println(string(md)) + fmt.Println("MDtemplate ends") + test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_with_cert_config_cp.yaml") + test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_with_cert_config_md.yaml") +} + +func TestDockerGenerateDeploymentFileWithMirrorAndAuthConfig(t *testing.T) { + mockCtrl := gomock.NewController(t) + t.Setenv("REGISTRY_USERNAME", "username") + t.Setenv("REGISTRY_PASSWORD", "password") + ctx := context.Background() + client := dockerMocks.NewMockProviderClient(mockCtrl) + kubectl := dockerMocks.NewMockProviderKubectlClient(mockCtrl) + provider := docker.NewProvider(&v1alpha1.DockerDatacenterConfig{}, client, kubectl, test.FakeNow) + clusterObj := &types.Cluster{Name: "test"} + clusterSpec := givenClusterSpec(t, "cluster_mirror_with_auth_config.yaml") + + if err := provider.SetupAndValidateCreateCluster(ctx, clusterSpec); err != nil { + t.Fatalf("failed to setup and validate: %v", err) + } + + cp, md, err := provider.GenerateCAPISpecForCreate(context.Background(), clusterObj, clusterSpec) + if err != nil { + t.Fatalf("failed to generate cluster api spec contents: %v", err) + } + + test.AssertContentToFile(t, string(cp), "testdata/expected_results_mirror_with_auth_config_cp.yaml") + test.AssertContentToFile(t, string(md), "testdata/expected_results_mirror_with_auth_config_md.yaml") +} diff --git a/pkg/providers/docker/testdata/cluster_mirror_config.yaml b/pkg/providers/docker/testdata/cluster_mirror_config.yaml new file mode 100644 index 000000000000..3c8b9f6d59d3 --- /dev/null +++ b/pkg/providers/docker/testdata/cluster_mirror_config.yaml @@ -0,0 +1,40 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test +spec: + clusterNetwork: + cniConfig: + cilium: {} + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 3 + datacenterRef: + kind: DockerDatacenterConfig + name: test + externalEtcdConfiguration: + count: 3 + kubernetesVersion: "1.21" + managementCluster: + name: test + workerNodeGroupConfigurations: + - count: 3 + name: md-0 + registryMirrorConfiguration: + endpoint: 1.2.3.4 + port: 1234 + ociNamespaces: + - registry: "public.ecr.aws" + namespace: "eks-anywhere" + +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: DockerDatacenterConfig +metadata: + name: test +spec: {} \ No newline at end of file diff --git a/pkg/providers/docker/testdata/cluster_mirror_with_auth_config.yaml b/pkg/providers/docker/testdata/cluster_mirror_with_auth_config.yaml new file mode 100644 index 000000000000..933225c00951 --- /dev/null +++ b/pkg/providers/docker/testdata/cluster_mirror_with_auth_config.yaml @@ -0,0 +1,59 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test +spec: + clusterNetwork: + cniConfig: + cilium: {} + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 3 + datacenterRef: + kind: DockerDatacenterConfig + name: test + externalEtcdConfiguration: + count: 3 + kubernetesVersion: "1.21" + managementCluster: + name: test + workerNodeGroupConfigurations: + - count: 3 + name: md-0 + registryMirrorConfiguration: + endpoint: 1.2.3.4 + port: 1234 + authenticate: true + ociNamespaces: + - registry: "public.ecr.aws" + namespace: "eks-anywhere" + caCertContent: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: DockerDatacenterConfig +metadata: + name: test +spec: {} \ No newline at end of file diff --git a/pkg/providers/docker/testdata/cluster_mirror_with_cert_config.yaml b/pkg/providers/docker/testdata/cluster_mirror_with_cert_config.yaml new file mode 100644 index 000000000000..01c714a5fe2d --- /dev/null +++ b/pkg/providers/docker/testdata/cluster_mirror_with_cert_config.yaml @@ -0,0 +1,58 @@ +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: Cluster +metadata: + name: test +spec: + clusterNetwork: + cniConfig: + cilium: {} + pods: + cidrBlocks: + - 192.168.0.0/16 + services: + cidrBlocks: + - 10.96.0.0/12 + controlPlaneConfiguration: + count: 3 + datacenterRef: + kind: DockerDatacenterConfig + name: test + externalEtcdConfiguration: + count: 3 + kubernetesVersion: "1.21" + managementCluster: + name: test + workerNodeGroupConfigurations: + - count: 3 + name: md-0 + registryMirrorConfiguration: + endpoint: 1.2.3.4 + port: 1234 + ociNamespaces: + - registry: "public.ecr.aws" + namespace: "eks-anywhere" + caCertContent: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + +--- +apiVersion: anywhere.eks.amazonaws.com/v1alpha1 +kind: DockerDatacenterConfig +metadata: + name: test +spec: {} \ No newline at end of file diff --git a/pkg/providers/docker/testdata/expected_results_mirror_config_cp.yaml b/pkg/providers/docker/testdata/expected_results_mirror_config_cp.yaml new file mode 100644 index 000000000000..ee257406056f --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_config_cp.yaml @@ -0,0 +1,325 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test + namespace: eksa-system +spec: + clusterNetwork: + pods: + cidrBlocks: [192.168.0.0/16] + serviceDomain: cluster.local + services: + cidrBlocks: [10.96.0.0/12] + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: test + namespace: eksa-system + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: test + namespace: eksa-system + managedExternalEtcdRef: + apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 + kind: EtcdadmCluster + name: test-etcd + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: test + namespace: eksa-system +spec: + loadBalancer: + imageRepository: + imageTag: +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-control-plane-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: test + namespace: eksa-system +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-control-plane-template-1234567890000 + namespace: eksa-system + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: public.ecr.aws/eks-distro/kubernetes + etcd: + external: + endpoints: [] + caFile: "/etc/kubernetes/pki/etcd/ca.crt" + certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key" + dns: + imageRepository: public.ecr.aws/eks-distro/coredns + imageTag: v1.8.3-eks-1-21-4 + apiServer: + certSANs: + - localhost + - 127.0.0.1 + extraArgs: + audit-policy-file: /etc/kubernetes/audit-policy.yaml + audit-log-path: /var/log/kubernetes/api-audit.log + audit-log-maxage: "30" + audit-log-maxbackup: "10" + audit-log-maxsize: "512" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + extraVolumes: + - hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + name: audit-policy + pathType: File + readOnly: true + - hostPath: /var/log/kubernetes + mountPath: /var/log/kubernetes + name: audit-log-dir + pathType: DirectoryOrCreate + readOnly: false + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + scheduler: + extraArgs: + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + apiVersion: audit.k8s.io/v1beta1 + kind: Policy + rules: + # Log aws-auth configmap changes + - level: RequestResponse + namespaces: ["kube-system"] + verbs: ["update", "patch", "delete"] + resources: + - group: "" # core + resources: ["configmaps"] + resourceNames: ["aws-auth"] + omitStages: + - "RequestReceived" + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + - level: Request + resources: + - group: "" + resources: ["serviceaccounts/token"] + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" + owner: root:root + path: /etc/kubernetes/audit-policy.yaml + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + owner: root:root + path: "/etc/containerd/config_append.toml" + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd + replicas: 3 + version: v1.21.2-eks-1-21-4 +--- +kind: EtcdadmCluster +apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 +metadata: + name: test-etcd + namespace: eksa-system +spec: + replicas: 3 + etcdadmConfigSpec: + etcdadmBuiltin: true + cloudInitConfig: + version: 3.4.16 + cipherSuites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + registryMirror: + endpoint: 1.2.3.4:1234/v2/eks-anywhere + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-etcd-template-1234567890000 + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-etcd-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 diff --git a/pkg/providers/docker/testdata/expected_results_mirror_config_md.yaml b/pkg/providers/docker/testdata/expected_results_mirror_config_md.yaml new file mode 100644 index 000000000000..9200737f484a --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_config_md.yaml @@ -0,0 +1,68 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: test-md-0-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + taints: [] + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + owner: root:root + path: "/etc/containerd/config_append.toml" + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: test-md-0 + namespace: eksa-system +spec: + clusterName: test + replicas: 3 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: test-md-0-template-1234567890000 + namespace: eksa-system + clusterName: test + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-md-0-1234567890000 + namespace: eksa-system + version: v1.21.2-eks-1-21-4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-md-0-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 + +--- diff --git a/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_cp.yaml b/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_cp.yaml new file mode 100644 index 000000000000..aaea088579fe --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_cp.yaml @@ -0,0 +1,370 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test + namespace: eksa-system +spec: + clusterNetwork: + pods: + cidrBlocks: [192.168.0.0/16] + serviceDomain: cluster.local + services: + cidrBlocks: [10.96.0.0/12] + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: test + namespace: eksa-system + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: test + namespace: eksa-system + managedExternalEtcdRef: + apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 + kind: EtcdadmCluster + name: test-etcd + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: test + namespace: eksa-system +spec: + loadBalancer: + imageRepository: + imageTag: +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-control-plane-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: test + namespace: eksa-system +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-control-plane-template-1234567890000 + namespace: eksa-system + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: public.ecr.aws/eks-distro/kubernetes + etcd: + external: + endpoints: [] + caFile: "/etc/kubernetes/pki/etcd/ca.crt" + certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key" + dns: + imageRepository: public.ecr.aws/eks-distro/coredns + imageTag: v1.8.3-eks-1-21-4 + apiServer: + certSANs: + - localhost + - 127.0.0.1 + extraArgs: + audit-policy-file: /etc/kubernetes/audit-policy.yaml + audit-log-path: /var/log/kubernetes/api-audit.log + audit-log-maxage: "30" + audit-log-maxbackup: "10" + audit-log-maxsize: "512" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + extraVolumes: + - hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + name: audit-policy + pathType: File + readOnly: true + - hostPath: /var/log/kubernetes + mountPath: /var/log/kubernetes + name: audit-log-dir + pathType: DirectoryOrCreate + readOnly: false + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + scheduler: + extraArgs: + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + apiVersion: audit.k8s.io/v1beta1 + kind: Policy + rules: + # Log aws-auth configmap changes + - level: RequestResponse + namespaces: ["kube-system"] + verbs: ["update", "patch", "delete"] + resources: + - group: "" # core + resources: ["configmaps"] + resourceNames: ["aws-auth"] + omitStages: + - "RequestReceived" + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + - level: Request + resources: + - group: "" + resources: ["serviceaccounts/token"] + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" + owner: root:root + path: /etc/kubernetes/audit-policy.yaml + - content: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + owner: root:root + path: "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".tls] + ca_file = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".auth] + username = "username" + password = "password" + owner: root:root + path: "/etc/containerd/config_append.toml" + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd + replicas: 3 + version: v1.21.2-eks-1-21-4 +--- +kind: EtcdadmCluster +apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 +metadata: + name: test-etcd + namespace: eksa-system +spec: + replicas: 3 + etcdadmConfigSpec: + etcdadmBuiltin: true + cloudInitConfig: + version: 3.4.16 + cipherSuites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + registryMirror: + endpoint: 1.2.3.4:1234/v2/eks-anywhere + caCert: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-etcd-template-1234567890000 + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-etcd-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 diff --git a/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_md.yaml b/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_md.yaml new file mode 100644 index 000000000000..d3d57e5502cb --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_with_auth_config_md.yaml @@ -0,0 +1,94 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: test-md-0-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + taints: [] + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + owner: root:root + path: "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".tls] + ca_file = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".auth] + username = "username" + password = "password" + owner: root:root + path: "/etc/containerd/config_append.toml" + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: test-md-0 + namespace: eksa-system +spec: + clusterName: test + replicas: 3 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: test-md-0-template-1234567890000 + namespace: eksa-system + clusterName: test + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-md-0-1234567890000 + namespace: eksa-system + version: v1.21.2-eks-1-21-4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-md-0-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 + +--- diff --git a/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_cp.yaml b/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_cp.yaml new file mode 100644 index 000000000000..5d281bb3a923 --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_cp.yaml @@ -0,0 +1,367 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: test + namespace: eksa-system +spec: + clusterNetwork: + pods: + cidrBlocks: [192.168.0.0/16] + serviceDomain: cluster.local + services: + cidrBlocks: [10.96.0.0/12] + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: test + namespace: eksa-system + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerCluster + name: test + namespace: eksa-system + managedExternalEtcdRef: + apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 + kind: EtcdadmCluster + name: test-etcd + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerCluster +metadata: + name: test + namespace: eksa-system +spec: + loadBalancer: + imageRepository: + imageTag: +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-control-plane-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: test + namespace: eksa-system +spec: + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-control-plane-template-1234567890000 + namespace: eksa-system + kubeadmConfigSpec: + clusterConfiguration: + imageRepository: public.ecr.aws/eks-distro/kubernetes + etcd: + external: + endpoints: [] + caFile: "/etc/kubernetes/pki/etcd/ca.crt" + certFile: "/etc/kubernetes/pki/apiserver-etcd-client.crt" + keyFile: "/etc/kubernetes/pki/apiserver-etcd-client.key" + dns: + imageRepository: public.ecr.aws/eks-distro/coredns + imageTag: v1.8.3-eks-1-21-4 + apiServer: + certSANs: + - localhost + - 127.0.0.1 + extraArgs: + audit-policy-file: /etc/kubernetes/audit-policy.yaml + audit-log-path: /var/log/kubernetes/api-audit.log + audit-log-maxage: "30" + audit-log-maxbackup: "10" + audit-log-maxsize: "512" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + extraVolumes: + - hostPath: /etc/kubernetes/audit-policy.yaml + mountPath: /etc/kubernetes/audit-policy.yaml + name: audit-policy + pathType: File + readOnly: true + - hostPath: /var/log/kubernetes + mountPath: /var/log/kubernetes + name: audit-log-dir + pathType: DirectoryOrCreate + readOnly: false + controllerManager: + extraArgs: + enable-hostpath-provisioner: "true" + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + scheduler: + extraArgs: + profiling: "false" + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + apiVersion: audit.k8s.io/v1beta1 + kind: Policy + rules: + # Log aws-auth configmap changes + - level: RequestResponse + namespaces: ["kube-system"] + verbs: ["update", "patch", "delete"] + resources: + - group: "" # core + resources: ["configmaps"] + resourceNames: ["aws-auth"] + omitStages: + - "RequestReceived" + # The following requests were manually identified as high-volume and low-risk, + # so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, + # so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + - level: Request + resources: + - group: "" + resources: ["serviceaccounts/token"] + # Get repsonses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" + owner: root:root + path: /etc/kubernetes/audit-policy.yaml + - content: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + owner: root:root + path: "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".tls] + ca_file = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + owner: root:root + path: "/etc/containerd/config_append.toml" + initConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd + replicas: 3 + version: v1.21.2-eks-1-21-4 +--- +kind: EtcdadmCluster +apiVersion: etcdcluster.cluster.x-k8s.io/v1beta1 +metadata: + name: test-etcd + namespace: eksa-system +spec: + replicas: 3 + etcdadmConfigSpec: + etcdadmBuiltin: true + cloudInitConfig: + version: 3.4.16 + cipherSuites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + registryMirror: + endpoint: 1.2.3.4:1234/v2/eks-anywhere + caCert: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + infrastructureTemplate: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-etcd-template-1234567890000 + namespace: eksa-system +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-etcd-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 diff --git a/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_md.yaml b/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_md.yaml new file mode 100644 index 000000000000..f5f2edc3a3cf --- /dev/null +++ b/pkg/providers/docker/testdata/expected_results_mirror_with_cert_config_md.yaml @@ -0,0 +1,91 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: test-md-0-template-1234567890000 + namespace: eksa-system +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + criSocket: /var/run/containerd/containerd.sock + taints: [] + kubeletExtraArgs: + eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0% + cgroup-driver: cgroupfs + tls-cipher-suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + files: + - content: | + -----BEGIN CERTIFICATE----- + MIICxjCCAa6gAwIBAgIJAInAeEdpH2uNMA0GCSqGSIb3DQEBBQUAMBUxEzARBgNV + BAMTCnRlc3QubG9jYWwwHhcNMjEwOTIzMjAxOTEyWhcNMzEwOTIxMjAxOTEyWjAV + MRMwEQYDVQQDEwp0ZXN0LmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB + CgKCAQEAwDHozKwX0kAGICTaV1XoMdJ+t+8LQsAGmzIKYhrSh+WdEcx/xc1SDJcp + EBFeUmVuFwI5DYX2BTvJ0AApSBuViNZn669yn1dBV7PHM27NV37/dDCFkjiqBtax + lOXchrL6IoZirmMgMnI/PfASdI/PCR75DNCIQFGZbwWAbEBxxLHgWPEFJ5TWP6fD + 2s95gbc9gykI09ta/H5ITKCd3EVtiAlcQ86Ax9EZRmvJYGw5NFmPnJ0X/OmXmLXx + o0ggkjHTeyG8sZQpDTs6oQrX/XLfLOvrJi3suiiJXz0pNAXZoFaLu8Z0Ci+EoquM + cFh4NhfSAD5BJADxwf7iv7KXCWtQTwIDAQABoxkwFzAVBgNVHREEDjAMggp0ZXN0 + LmxvY2FsMA0GCSqGSIb3DQEBBQUAA4IBAQBr4qDklaG/ZLcrkc0PBo9ylj3rtt1M + ar1nv+Nv8zXByTsYs9muEQYBKpzvk9SJZ4OfYVcx6qETbG7z7kdgZtDktQULw5fQ + hsiy0flLv+JkdD4M30rtjhDIiuNH2ew6+2JB80QaSznW7Z3Fd18BmDaE1qqLYQFX + iCau7fRD2aQyVluuJ0OeDOuk33jY3Vn3gyKGfnjPAnb4DxCg7v1IeazGSVK18urL + zkYl4nSFENRLV5sL/wox2ohjMLff2lv6gyqkMFrLNSeHSQLGu8diat4UVDk8MMza + 9n5t2E4AHPen+YrGeLY1qEn9WMv0XRGWrgJyLW9VSX8T3SlWO2w3okcw + -----END CERTIFICATE----- + + owner: root:root + path: "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + - content: | + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."public.ecr.aws"] + endpoint = ["https://1.2.3.4:1234/v2/eks-anywhere"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."1.2.3.4:1234".tls] + ca_file = "/etc/containerd/certs.d/1.2.3.4:1234/ca.crt" + owner: root:root + path: "/etc/containerd/config_append.toml" + preKubeadmCommands: + - cat /etc/containerd/config_append.toml >> /etc/containerd/config.toml + - systemctl daemon-reload + - systemctl restart containerd +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: test-md-0 + namespace: eksa-system +spec: + clusterName: test + replicas: 3 + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: test-md-0-template-1234567890000 + namespace: eksa-system + clusterName: test + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: test-md-0-1234567890000 + namespace: eksa-system + version: v1.21.2-eks-1-21-4 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: test-md-0-1234567890000 + namespace: eksa-system +spec: + template: + spec: + extraMounts: + - containerPath: /var/run/docker.sock + hostPath: /var/run/docker.sock + customImage: public.ecr.aws/l0g8r8j6/kubernetes-sigs/kind/node:v1.21.2-eks-d-1-21-4-eks-a-v0.0.0-dev-build.158 + +--- diff --git a/pkg/providers/nutanix/client.go b/pkg/providers/nutanix/client.go index cb4c89249f73..bf98de9e8985 100644 --- a/pkg/providers/nutanix/client.go +++ b/pkg/providers/nutanix/client.go @@ -9,12 +9,9 @@ import ( type Client interface { GetSubnet(ctx context.Context, uuid string) (*v3.SubnetIntentResponse, error) ListSubnet(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.SubnetListIntentResponse, error) - GetImage(ctx context.Context, uuid string) (*v3.ImageIntentResponse, error) ListImage(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ImageListIntentResponse, error) - GetCluster(ctx context.Context, uuid string) (*v3.ClusterIntentResponse, error) ListCluster(ctx context.Context, getEntitiesRequest *v3.DSMetadata) (*v3.ClusterListIntentResponse, error) - GetCurrentLoggedInUser(ctx context.Context) (*v3.UserIntentResponse, error) } diff --git a/pkg/providers/tinkerbell/tinkerbell.go b/pkg/providers/tinkerbell/tinkerbell.go index 88e971866bcd..86577ffffc0d 100644 --- a/pkg/providers/tinkerbell/tinkerbell.go +++ b/pkg/providers/tinkerbell/tinkerbell.go @@ -87,10 +87,8 @@ type ProviderKubectlClient interface { WaitForRufioMachines(ctx context.Context, cluster *types.Cluster, timeout string, condition string, namespace string) error SearchTinkerbellMachineConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellMachineConfig, error) SearchTinkerbellDatacenterConfig(ctx context.Context, name string, kubeconfigFile string, namespace string) ([]*v1alpha1.TinkerbellDatacenterConfig, error) - AllTinkerbellHardware(ctx context.Context, kuebconfig string) ([]tinkv1alpha1.Hardware, error) AllBaseboardManagements(ctx context.Context, kubeconfig string) ([]rufiounreleased.BaseboardManagement, error) - HasCRD(ctx context.Context, kubeconfig, crd string) (bool, error) DeleteCRD(ctx context.Context, kubeconfig, crd string) error } diff --git a/test/e2e/airgap.go b/test/e2e/airgap.go index d4badd1529d3..b55bcc49362a 100644 --- a/test/e2e/airgap.go +++ b/test/e2e/airgap.go @@ -4,6 +4,8 @@ package e2e import ( + "os" + "github.com/aws/eks-anywhere/test/framework" ) @@ -11,8 +13,9 @@ import ( func runVSphereAirgapConfigFlow(test *framework.ClusterE2ETest, localCIDRs string) { test.GenerateClusterConfig() test.DownloadArtifacts() - test.ImportImages() test.ExtractDownloadedArtifacts() + test.DownloadImages() + test.ImportImages() test.AirgapDockerContainers(localCIDRs) test.CreateAirgappedUser(localCIDRs) test.AssertAirgappedNetwork() @@ -22,3 +25,15 @@ func runVSphereAirgapConfigFlow(test *framework.ClusterE2ETest, localCIDRs strin ) test.DeleteCluster() } + +func runDockerAirgapConfigFlow(test *framework.ClusterE2ETest) { + test.GenerateClusterConfig() + test.DownloadArtifacts() + test.ExtractDownloadedArtifacts() + test.DownloadImages() + test.ChangeInstanceSecurityGroup(os.Getenv(framework.RegistryMirrorAirgappedSecurityGroup)) + test.ImportImages() + test.CreateCluster(framework.WithBundlesOverride("./eks-anywhere-downloads/bundle-release.yaml")) + test.DeleteCluster(framework.WithBundlesOverride("./eks-anywhere-downloads/bundle-release.yaml")) + test.ChangeInstanceSecurityGroup(os.Getenv(framework.RegistryMirrorDefaultSecurityGroup)) +} diff --git a/test/e2e/autoscaler.go b/test/e2e/autoscaler.go index 65519eb5ea59..6af7fe69dded 100644 --- a/test/e2e/autoscaler.go +++ b/test/e2e/autoscaler.go @@ -15,4 +15,4 @@ func runAutoscalerWitMetricsServerSimpleFlow(test *framework.ClusterE2ETest) { test.InstallAutoScalerWithMetricServer(targetNamespace) test.CombinedAutoScalerMetricServerTest(autoscalerName, metricServerName, targetNamespace, withMgmtCluster(test)) }) -} \ No newline at end of file +} diff --git a/test/e2e/constants.go b/test/e2e/constants.go index e2eedab6f45c..744cf3e37da6 100644 --- a/test/e2e/constants.go +++ b/test/e2e/constants.go @@ -1,4 +1,4 @@ -//nolint +// nolint package e2e import ( @@ -11,25 +11,25 @@ const ( EksaPackageControllerHelmURI = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages" EksaPackageControllerHelmVersion = "0.2.20-eks-a-v0.0.0-dev-build.4894" EksaPackageBundleURI = "oci://" + EksaPackagesSourceRegistry + "/eks-anywhere-packages-bundles" - EksaPackagesNamespace = "eksa-packages" + EksaPackagesNamespace = "eksa-packages" clusterNamespace = "test-namespace" - key1 = framework.LabelPrefix + "/" + "key1" - key2 = framework.LabelPrefix + "/" + "key2" - cpKey1 = framework.LabelPrefix + "/" + "cp-key1" - val1 = "val1" - val2 = "val2" - cpVal1 = "cp-val1" + key1 = framework.LabelPrefix + "/" + "key1" + key2 = framework.LabelPrefix + "/" + "key2" + cpKey1 = framework.LabelPrefix + "/" + "cp-key1" + val1 = "val1" + val2 = "val2" + cpVal1 = "cp-val1" nodeGroupLabel1 = "md-0" nodeGroupLabel2 = "md-1" - worker0 = "worker-0" - worker1 = "worker-1" - worker2 = "worker-2" + worker0 = "worker-0" + worker1 = "worker-1" + worker2 = "worker-2" - fluxUserProvidedBranch = "testbranch" - fluxUserProvidedNamespace = "testns" - fluxUserProvidedPath = "test/testerson" + fluxUserProvidedBranch = "testbranch" + fluxUserProvidedNamespace = "testns" + fluxUserProvidedPath = "test/testerson" vsphereCpVmNumCpuUpdateVar = 4 vsphereCpVmMemoryUpdate = 16384 @@ -44,4 +44,4 @@ const ( vsphereResourcePoolVar = "T_VSPHERE_RESOURCE_POOL" ) -var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"} \ No newline at end of file +var EksaPackageControllerHelmValues = []string{"sourceRegistry=public.ecr.aws/l0g8r8j6"} diff --git a/test/e2e/docker_test.go b/test/e2e/docker_test.go index 9770df2c99d1..dfe11a413024 100644 --- a/test/e2e/docker_test.go +++ b/test/e2e/docker_test.go @@ -7,11 +7,13 @@ package e2e import ( "testing" + "github.com/stretchr/testify/suite" + corev1 "k8s.io/api/core/v1" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" + "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/test/framework" - "github.com/stretchr/testify/suite" - corev1 "k8s.io/api/core/v1" ) // Labels @@ -336,6 +338,29 @@ func TestDockerKubernetes125OIDC(t *testing.T) { runOIDCFlow(test) } +// RegistryMirror +func TestDockerKubernetes125RegistryMirrorAndCert(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube125)), + framework.WithRegistryMirrorEndpointAndCert(constants.DockerProviderName), + ) + runRegistryMirrorConfigFlow(test) +} + +func TestDockerKubernetes125AirgappedRegistryMirrorAndCert(t *testing.T) { + test := framework.NewClusterE2ETest( + t, + framework.NewDocker(t), + framework.WithClusterFiller(api.WithExternalEtcdTopology(1)), + framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube125)), + framework.WithRegistryMirrorEndpointAndCert(constants.DockerProviderName), + ) + runDockerAirgapConfigFlow(test) +} + // Simple flow func TestDockerKubernetes122SimpleFlow(t *testing.T) { test := framework.NewClusterE2ETest( diff --git a/test/e2e/metallb.go b/test/e2e/metallb.go index 3b9e42241a34..382037b37547 100644 --- a/test/e2e/metallb.go +++ b/test/e2e/metallb.go @@ -11,11 +11,12 @@ import ( "testing" "time" + "github.com/stretchr/testify/suite" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/kubeconfig" "github.com/aws/eks-anywhere/test/framework" - "github.com/stretchr/testify/suite" ) type Suite struct { diff --git a/test/e2e/registrymirror.go b/test/e2e/registrymirror.go index d8bad1c7d60c..2d5a8fe821a7 100644 --- a/test/e2e/registrymirror.go +++ b/test/e2e/registrymirror.go @@ -9,14 +9,19 @@ import ( func runRegistryMirrorConfigFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() + test.DownloadArtifacts() + test.ExtractDownloadedArtifacts() + test.DownloadImages() test.ImportImages() test.CreateCluster() - test.ImportImages() test.DeleteCluster() } func runTinkerbellRegistryMirrorFlow(test *framework.ClusterE2ETest) { test.GenerateClusterConfig() + test.DownloadArtifacts() + test.ExtractDownloadedArtifacts() + test.DownloadImages() test.ImportImages() test.GenerateHardwareConfig() test.PowerOffHardware() diff --git a/test/e2e/snow_test.go b/test/e2e/snow_test.go index 5695081617c8..b8d201684e34 100644 --- a/test/e2e/snow_test.go +++ b/test/e2e/snow_test.go @@ -7,10 +7,11 @@ package e2e import ( "testing" + corev1 "k8s.io/api/core/v1" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/test/framework" - corev1 "k8s.io/api/core/v1" ) // AWS IAM Auth diff --git a/test/e2e/tinkerbell_test.go b/test/e2e/tinkerbell_test.go index a895a4d20397..9bbd97271243 100644 --- a/test/e2e/tinkerbell_test.go +++ b/test/e2e/tinkerbell_test.go @@ -7,11 +7,12 @@ package e2e import ( "testing" + corev1 "k8s.io/api/core/v1" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/test/framework" - corev1 "k8s.io/api/core/v1" ) // AWS IAM Auth diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go index 7838f55702a4..9b78d89499c1 100644 --- a/test/e2e/vsphere_test.go +++ b/test/e2e/vsphere_test.go @@ -8,12 +8,13 @@ import ( "os" "testing" + corev1 "k8s.io/api/core/v1" + "github.com/aws/eks-anywhere/internal/pkg/api" "github.com/aws/eks-anywhere/pkg/api/v1alpha1" "github.com/aws/eks-anywhere/pkg/constants" "github.com/aws/eks-anywhere/pkg/features" "github.com/aws/eks-anywhere/test/framework" - corev1 "k8s.io/api/core/v1" ) // Autoimport diff --git a/test/framework/cluster.go b/test/framework/cluster.go index a3243db4cfb2..490fc62c1717 100644 --- a/test/framework/cluster.go +++ b/test/framework/cluster.go @@ -8,6 +8,7 @@ import ( _ "embed" "fmt" "io" + "net" "os" "os/exec" "path/filepath" @@ -39,20 +40,22 @@ import ( ) const ( - defaultClusterConfigFile = "cluster.yaml" - defaultBundleReleaseManifestFile = "bin/local-bundle-release.yaml" - defaultEksaBinaryLocation = "eksctl anywhere" - defaultClusterName = "eksa-test" - eksctlVersionEnvVar = "EKSCTL_VERSION" - eksctlVersionEnvVarDummyVal = "ham sandwich" - ClusterPrefixVar = "T_CLUSTER_PREFIX" - JobIdVar = "T_JOB_ID" - BundlesOverrideVar = "T_BUNDLES_OVERRIDE" - ClusterIPPoolEnvVar = "T_CLUSTER_IP_POOL" - CleanupVmsVar = "T_CLEANUP_VMS" - hardwareYamlPath = "hardware.yaml" - hardwareCsvPath = "hardware.csv" - EksaPackagesInstallation = "eks-anywhere-packages" + defaultClusterConfigFile = "cluster.yaml" + defaultBundleReleaseManifestFile = "bin/local-bundle-release.yaml" + defaultEksaBinaryLocation = "eksctl anywhere" + defaultClusterName = "eksa-test" + defaultDownloadArtifactsOutputLocation = "eks-anywhere-downloads.tar.gz" + defaultDownloadImagesOutputLocation = "images.tar" + eksctlVersionEnvVar = "EKSCTL_VERSION" + eksctlVersionEnvVarDummyVal = "ham sandwich" + ClusterPrefixVar = "T_CLUSTER_PREFIX" + JobIdVar = "T_JOB_ID" + BundlesOverrideVar = "T_BUNDLES_OVERRIDE" + ClusterIPPoolEnvVar = "T_CLUSTER_IP_POOL" + CleanupVmsVar = "T_CLEANUP_VMS" + hardwareYamlPath = "hardware.yaml" + hardwareCsvPath = "hardware.csv" + EksaPackagesInstallation = "eks-anywhere-packages" ) //go:embed testdata/oidc-roles.yaml @@ -583,29 +586,65 @@ func (e *ClusterE2ETest) WithClusterConfig(fillers ...api.ClusterConfigFiller) * return e } -func (e *ClusterE2ETest) ImportImages(opts ...CommandOpt) { - importImagesArgs := []string{"import-images", "-f", e.ClusterConfigLocation} - e.RunEKSA(importImagesArgs, opts...) -} - +// DownloadArtifacts runs the EKS-A `download artifacts` command with appropriate args. func (e *ClusterE2ETest) DownloadArtifacts(opts ...CommandOpt) { downloadArtifactsArgs := []string{"download", "artifacts", "-f", e.ClusterConfigLocation} + if getBundlesOverride() == "true" { + downloadArtifactsArgs = append(downloadArtifactsArgs, "--bundles-override", defaultBundleReleaseManifestFile) + } e.RunEKSA(downloadArtifactsArgs, opts...) - if _, err := os.Stat("eks-anywhere-downloads.tar.gz"); err != nil { + if _, err := os.Stat(defaultDownloadArtifactsOutputLocation); err != nil { e.T.Fatal(err) } else { - e.T.Log("Downloaded artifacts saved at eks-anywhere-downloads.tar.gz") + e.T.Logf("Downloaded artifacts tarball saved at %s", defaultDownloadArtifactsOutputLocation) } } -// ExtractDownloadedArtifacts extract the downloaded artifacts. +// ExtractDownloadedArtifacts extracts the downloaded artifacts. func (e *ClusterE2ETest) ExtractDownloadedArtifacts(opts ...CommandOpt) { - if _, err := os.Stat("eks-anywhere-downloads.tar.gz"); err != nil { + e.T.Log("Extracting downloaded artifacts") + e.Run("tar", "-xf", defaultDownloadArtifactsOutputLocation) +} + +// DownloadImages runs the EKS-A `download images` command with appropriate args. +func (e *ClusterE2ETest) DownloadImages(opts ...CommandOpt) { + downloadImagesArgs := []string{"download", "images", "-o", defaultDownloadImagesOutputLocation} + if getBundlesOverride() == "true" { + var bundleManifestLocation string + if _, err := os.Stat(defaultDownloadArtifactsOutputLocation); err == nil { + bundleManifestLocation = "eks-anywhere-downloads/bundle-release.yaml" + } else { + bundleManifestLocation = defaultBundleReleaseManifestFile + } + downloadImagesArgs = append(downloadImagesArgs, "--bundles-override", bundleManifestLocation) + } + e.RunEKSA(downloadImagesArgs, opts...) + if _, err := os.Stat(defaultDownloadImagesOutputLocation); err != nil { e.T.Fatal(err) + } else { + e.T.Logf("Downloaded images archive saved at %s", defaultDownloadImagesOutputLocation) + } +} + +// ImportImages runs the EKS-A `import images` command with appropriate args. +func (e *ClusterE2ETest) ImportImages(opts ...CommandOpt) { + clusterConfig := e.ClusterConfig.Cluster + registyMirrorEndpoint, registryMirrorPort := clusterConfig.Spec.RegistryMirrorConfiguration.Endpoint, clusterConfig.Spec.RegistryMirrorConfiguration.Port + registryMirrorHost := net.JoinHostPort(registyMirrorEndpoint, registryMirrorPort) + var bundleManifestLocation string + if _, err := os.Stat(defaultDownloadArtifactsOutputLocation); err == nil { + bundleManifestLocation = "eks-anywhere-downloads/bundle-release.yaml" + } else { + bundleManifestLocation = defaultBundleReleaseManifestFile } + importImagesArgs := []string{"import images", "--input", defaultDownloadImagesOutputLocation, "--bundles", bundleManifestLocation, "--registry", registryMirrorHost, "--insecure"} + e.RunEKSA(importImagesArgs, opts...) +} - e.T.Logf("Extract downloaded artifacts ") - e.Run("tar", "-xf", "eks-anywhere-downloads.tar.gz") +// ChangeInstanceSecurityGroup modifies the security group of the instance to the provided value. +func (e *ClusterE2ETest) ChangeInstanceSecurityGroup(securityGroup string) { + e.T.Logf("Changing instance security group to %s", securityGroup) + e.Run(fmt.Sprintf("INSTANCE_ID=$(ec2-metadata -i | awk '{print $2}') && aws ec2 modify-instance-attribute --instance-id $INSTANCE_ID --groups %s", securityGroup)) } func (e *ClusterE2ETest) CreateCluster(opts ...CommandOpt) { diff --git a/test/framework/registryMirror.go b/test/framework/registryMirror.go index c57f3f927de0..cea97b90d3e3 100644 --- a/test/framework/registryMirror.go +++ b/test/framework/registryMirror.go @@ -11,21 +11,24 @@ import ( ) const ( - RegistryEndpointVar = "T_REGISTRY_MIRROR_ENDPOINT" - RegistryPortVar = "T_REGISTRY_MIRROR_PORT" - RegistryUsernameVar = "T_REGISTRY_MIRROR_USERNAME" - RegistryPasswordVar = "T_REGISTRY_MIRROR_PASSWORD" - RegistryCACertVar = "T_REGISTRY_MIRROR_CA_CERT" - RegistryEndpointTinkerbellVar = "T_REGISTRY_MIRROR_ENDPOINT_TINKERBELL" - RegistryPortTinkerbellVar = "T_REGISTRY_MIRROR_PORT_TINKERBELL" - RegistryUsernameTinkerbellVar = "T_REGISTRY_MIRROR_USERNAME_TINKERBELL" - RegistryPasswordTinkerbellVar = "T_REGISTRY_MIRROR_PASSWORD_TINKERBELL" - RegistryCACertTinkerbellVar = "T_REGISTRY_MIRROR_CA_CERT_TINKERBELL" + RegistryEndpointVar = "T_REGISTRY_MIRROR_ENDPOINT" + RegistryPortVar = "T_REGISTRY_MIRROR_PORT" + RegistryUsernameVar = "T_REGISTRY_MIRROR_USERNAME" + RegistryPasswordVar = "T_REGISTRY_MIRROR_PASSWORD" + RegistryCACertVar = "T_REGISTRY_MIRROR_CA_CERT" + RegistryEndpointTinkerbellVar = "T_REGISTRY_MIRROR_ENDPOINT_TINKERBELL" + RegistryPortTinkerbellVar = "T_REGISTRY_MIRROR_PORT_TINKERBELL" + RegistryUsernameTinkerbellVar = "T_REGISTRY_MIRROR_USERNAME_TINKERBELL" + RegistryPasswordTinkerbellVar = "T_REGISTRY_MIRROR_PASSWORD_TINKERBELL" + RegistryCACertTinkerbellVar = "T_REGISTRY_MIRROR_CA_CERT_TINKERBELL" + RegistryMirrorDefaultSecurityGroup = "T_REGISTRY_MIRROR_DEFAULT_SECURITY_GROUP" + RegistryMirrorAirgappedSecurityGroup = "T_REGISTRY_MIRROR_AIRGAPPED_SECURITY_GROUP" ) var ( - registryMirrorRequiredEnvVars = []string{RegistryEndpointVar, RegistryPortVar, RegistryUsernameVar, RegistryPasswordVar, RegistryCACertVar} - registryMirrorTinkerbellRequiredEnvVars = []string{RegistryEndpointTinkerbellVar, RegistryPortTinkerbellVar, RegistryUsernameTinkerbellVar, RegistryPasswordTinkerbellVar, RegistryCACertTinkerbellVar} + registryMirrorRequiredEnvVars = []string{RegistryEndpointVar, RegistryPortVar, RegistryUsernameVar, RegistryPasswordVar, RegistryCACertVar} + registryMirrorTinkerbellRequiredEnvVars = []string{RegistryEndpointTinkerbellVar, RegistryPortTinkerbellVar, RegistryUsernameTinkerbellVar, RegistryPasswordTinkerbellVar, RegistryCACertTinkerbellVar} + registryMirrorDockerAirgappedRequiredEnvVars = []string{RegistryMirrorDefaultSecurityGroup, RegistryMirrorAirgappedSecurityGroup} ) func WithRegistryMirrorEndpointAndCert(providerName string) ClusterE2ETestOpt { @@ -79,5 +82,6 @@ func WithRegistryMirrorEndpointAndCert(providerName string) ClusterE2ETestOpt { } func RequiredRegistryMirrorEnvVars() []string { - return append(registryMirrorRequiredEnvVars, registryMirrorTinkerbellRequiredEnvVars...) + registryMirrorRequiredEnvVars = append(registryMirrorRequiredEnvVars, registryMirrorTinkerbellRequiredEnvVars...) + return append(registryMirrorRequiredEnvVars, registryMirrorDockerAirgappedRequiredEnvVars...) }