From 393876f1c72aa475f01d9ab99b2c714142184792 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Tue, 24 Sep 2024 11:17:22 -0700 Subject: [PATCH] TEST ONLY --- Makefile | 2 +- test/e2e/e2e_test.go | 22 +++-- test/managedcluster/aws/aws.go | 84 ++++++++++++------- test/managedcluster/azure/azure.go | 22 ++--- .../resources/aws-hosted-cp.yaml.tpl | 3 + .../resources/aws-standalone-cp.yaml.tpl | 3 + 6 files changed, 91 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index 07465a91..330f4bc0 100644 --- a/Makefile +++ b/Makefile @@ -307,7 +307,7 @@ dev-release: .PHONY: dev-aws-creds dev-aws-creds: envsubst - @NAMESPACE=$(NAMESPACE) $(ENVSUBST) -no-unset -i config/dev/aws-credentials.yaml | $(KUBECTL) apply -f - + @NAMESPACE=$(NAMESPACE) $(ENVSUBST) -i config/dev/aws-credentials.yaml | $(KUBECTL) apply -f - .PHONY: dev-azure-creds dev-azure-creds: envsubst diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 7cb70561..c5394342 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -34,6 +34,7 @@ import ( "github.com/Mirantis/hmc/test/kubeclient" "github.com/Mirantis/hmc/test/managedcluster" "github.com/Mirantis/hmc/test/managedcluster/aws" + "github.com/Mirantis/hmc/test/managedcluster/azure" "github.com/Mirantis/hmc/test/managedcluster/vsphere" "github.com/Mirantis/hmc/test/utils" ) @@ -48,6 +49,7 @@ var _ = Describe("controller", Ordered, func() { cmd := exec.Command("make", "dev-apply") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + }) AfterAll(func() { @@ -62,7 +64,6 @@ var _ = Describe("controller", Ordered, func() { Context("Operator", func() { It("should run successfully", func() { kc := kubeclient.NewFromLocal(namespace) - aws.CreateCredentialSecret(context.Background(), kc) By("validating that the hmc-controller and capi provider controllers are running") Eventually(func() error { @@ -73,6 +74,11 @@ var _ = Describe("controller", Ordered, func() { } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + GinkgoT().Setenv("NAMESPACE", namespace) + cmd := exec.Command("make", "DEV_PROVIDER=aws", "dev-creds-apply") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + // aws.CreateCredentialSecret(context.Background(), kc) }) }) @@ -89,7 +95,11 @@ var _ = Describe("controller", Ordered, func() { BeforeAll(func() { By("ensuring AWS credentials are set") kc = kubeclient.NewFromLocal(namespace) - aws.CreateCredentialSecret(context.Background(), kc) + // aws.CreateCredentialSecret(context.Background(), kc) + GinkgoT().Setenv("NAMESPACE", namespace) + cmd := exec.Command("make", "DEV_PROVIDER=aws", "dev-creds-apply") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) }) AfterEach(func() { @@ -159,11 +169,15 @@ var _ = Describe("controller", Ordered, func() { cmd = exec.Command("make", "dev-templates") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) + GinkgoT().Setenv("NAMESPACE", namespace) + cmd = exec.Command("make", "DEV_PROVIDER=aws", "dev-creds-apply") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) // Ensure AWS credentials are set in the standalone cluster. standaloneClient = kc.NewFromCluster(context.Background(), namespace, clusterName) - aws.CreateCredentialSecret(context.Background(), standaloneClient) + // aws.CreateCredentialSecret(context.Background(), standaloneClient) templateBy(managedcluster.TemplateAWSHostedCP, "validating that the controller is ready") Eventually(func() error { @@ -291,7 +305,6 @@ var _ = Describe("controller", Ordered, func() { err = deleteFunc() Expect(err).NotTo(HaveOccurred()) } - }) It("should deploy standalone managed cluster", func() { @@ -455,7 +468,6 @@ var _ = Describe("controller", Ordered, func() { Eventually(func() error { return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - }) }) }) diff --git a/test/managedcluster/aws/aws.go b/test/managedcluster/aws/aws.go index 221f7a22..0bdccafa 100644 --- a/test/managedcluster/aws/aws.go +++ b/test/managedcluster/aws/aws.go @@ -17,53 +17,81 @@ package aws import ( + "bufio" + "bytes" "context" "encoding/json" - "os/exec" - - corev1 "k8s.io/api/core/v1" + "errors" + "io" + "os" + "github.com/a8m/envsubst" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/types" + yamlutil "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" "github.com/Mirantis/hmc/test/kubeclient" "github.com/Mirantis/hmc/test/managedcluster" - "github.com/Mirantis/hmc/test/utils" ) -// CreateCredentialSecret uses clusterawsadm to encode existing AWS -// credentials and create a secret in the given namespace if one does not -// already exist. func CreateCredentialSecret(ctx context.Context, kc *kubeclient.KubeClient) { GinkgoHelper() - - _, err := kc.Client.CoreV1().Secrets(kc.Namespace). - Get(ctx, managedcluster.AWSCredentialsSecretName, metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - Expect(err).NotTo(HaveOccurred(), "failed to get AWS credentials secret") - return + serializer := yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + yamlFile, err := os.ReadFile("config/dev/aws-credentials.yaml") + Expect(err).NotTo(HaveOccurred()) + + yamlFile, err = envsubst.Bytes(yamlFile) + Expect(err).NotTo(HaveOccurred()) + + c := discovery.NewDiscoveryClientForConfigOrDie(kc.Config) + groupResources, err := restmapper.GetAPIGroupResources(c) + Expect(err).NotTo(HaveOccurred()) + + yamlReader := yamlutil.NewYAMLReader(bufio.NewReader(bytes.NewReader(yamlFile))) + for { + yamlDoc, err := yamlReader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + Expect(err).NotTo(HaveOccurred(), "failed to read yaml file") + } + + credentialResource := &unstructured.Unstructured{} + _, _, err = serializer.Decode(yamlDoc, nil, credentialResource) + Expect(err).NotTo(HaveOccurred(), "failed to parse credential resource") + + mapper := restmapper.NewDiscoveryRESTMapper(groupResources) + mapping, err := mapper.RESTMapping(credentialResource.GroupVersionKind().GroupKind()) + Expect(err).NotTo(HaveOccurred(), "failed to get rest mapping") + + dc := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: credentialResource.GroupVersionKind().Group, + Version: credentialResource.GroupVersionKind().Version, + Resource: mapping.Resource.Resource, + }) + + exists, err := dc.Get(ctx, credentialResource.GetName(), metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "failed to get azure credential secret") + } + + if exists == nil { + if _, err := dc.Create(ctx, credentialResource, metav1.CreateOptions{}); err != nil { + Expect(err).NotTo(HaveOccurred(), "failed to create azure credential secret") + } + } } - - cmd := exec.Command("./bin/clusterawsadm", "bootstrap", "credentials", "encode-as-profile") - output, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "failed to encode AWS credentials with clusterawsadm") - - _, err = kc.Client.CoreV1().Secrets(kc.Namespace).Create(ctx, &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: managedcluster.AWSCredentialsSecretName, - }, - Data: map[string][]byte{ - "AWS_B64ENCODED_CREDENTIALS": output, - }, - Type: corev1.SecretTypeOpaque, - }, metav1.CreateOptions{}) - Expect(err).NotTo(HaveOccurred(), "failed to create AWS credentials secret") } // PopulateHostedTemplateVars populates the environment variables required for diff --git a/test/managedcluster/azure/azure.go b/test/managedcluster/azure/azure.go index 80dff329..fcf26208 100644 --- a/test/managedcluster/azure/azure.go +++ b/test/managedcluster/azure/azure.go @@ -19,6 +19,7 @@ import ( "bytes" "context" "errors" + "fmt" "io" "os" @@ -39,15 +40,15 @@ import ( "github.com/Mirantis/hmc/test/kubeclient" ) -func getAzureInfo(ctx context.Context, name string, kc *kubeclient.KubeClient) map[string]interface{} { +func getAzureInfo(ctx context.Context, name string, kc *kubeclient.KubeClient) map[string]any { GinkgoHelper() - resourceId := schema.GroupVersionResource{ + resourceID := schema.GroupVersionResource{ Group: "infrastructure.cluster.x-k8s.io", Version: "v1beta1", Resource: "azureclusters", } - dc := kc.GetDynamicClient(resourceId) + dc := kc.GetDynamicClient(resourceID) list, err := dc.List(ctx, metav1.ListOptions{ LabelSelector: labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: name}).String(), }) @@ -81,23 +82,23 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) Expect(found).To(BeTrue()) resourceGroup := spec["resourceGroup"] - GinkgoT().Setenv("AZURE_RESOURCE_GROUP", resourceGroup.(string)) - subnetMap, ok := subnets[0].(map[string]interface{}) + GinkgoT().Setenv("AZURE_RESOURCE_GROUP", fmt.Sprintf("%s", resourceGroup)) + subnetMap, ok := subnets[0].(map[string]any) Expect(ok).To(BeTrue()) subnetName := subnetMap["name"] - GinkgoT().Setenv("AZURE_NODE_SUBNET", subnetName.(string)) + GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) securityGroup, found, err := unstructured.NestedMap(subnetMap, "securityGroup") Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) securityGroupName := securityGroup["name"] - GinkgoT().Setenv("AZURE_SECURITY_GROUP", securityGroupName.(string)) + GinkgoT().Setenv("AZURE_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) routeTable, found, err := unstructured.NestedMap(subnetMap, "routeTable") Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) routeTableName := routeTable["name"] - GinkgoT().Setenv("AZURE_ROUTE_TABLE", routeTableName.(string)) + GinkgoT().Setenv("AZURE_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) } func CreateCredentialSecret(ctx context.Context, kc *kubeclient.KubeClient) { @@ -116,7 +117,6 @@ func CreateCredentialSecret(ctx context.Context, kc *kubeclient.KubeClient) { yamlReader := yamlutil.NewYAMLReader(bufio.NewReader(bytes.NewReader(yamlFile))) for { yamlDoc, err := yamlReader.Read() - if err != nil { if errors.Is(err, io.EOF) { break @@ -144,8 +144,8 @@ func CreateCredentialSecret(ctx context.Context, kc *kubeclient.KubeClient) { } if exists == nil { - if _, err = dc.Create(ctx, credentialResource, metav1.CreateOptions{}); err != nil { - Expect(err).NotTo(HaveOccurred(), "failed to create azure credential secret") + if _, createErr := dc.Create(ctx, credentialResource, metav1.CreateOptions{}); err != nil { + Expect(createErr).NotTo(HaveOccurred(), "failed to create azure credential secret") } } } diff --git a/test/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/managedcluster/resources/aws-hosted-cp.yaml.tpl index 06a4cf4c..64e46a0b 100644 --- a/test/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -5,6 +5,9 @@ metadata: spec: template: aws-hosted-cp config: + clusterIdentity: + name: aws-cluster-identity + namespace: ${NAMESPACE} vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: diff --git a/test/managedcluster/resources/aws-standalone-cp.yaml.tpl b/test/managedcluster/resources/aws-standalone-cp.yaml.tpl index 8b8943c2..e5ae8848 100644 --- a/test/managedcluster/resources/aws-standalone-cp.yaml.tpl +++ b/test/managedcluster/resources/aws-standalone-cp.yaml.tpl @@ -5,6 +5,9 @@ metadata: spec: template: aws-standalone-cp config: + clusterIdentity: + name: aws-cluster-identity + namespace: ${NAMESPACE} region: ${AWS_REGION} publicIP: ${AWS_PUBLIC_IP:=true} controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1}