From 3c91b1690c3237f2ccd7dfe10f236dba134ab4e6 Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:31:08 +0000 Subject: [PATCH 1/8] fix: destroy elb Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/aws/aws.go | 70 ++++++++++++++++++++++++++++++++++++++ internal/k8s/kubernetes.go | 15 ++++++++ 2 files changed, 85 insertions(+) diff --git a/internal/aws/aws.go b/internal/aws/aws.go index 8743de3b9..17808bd0a 100644 --- a/internal/aws/aws.go +++ b/internal/aws/aws.go @@ -993,6 +993,68 @@ func ProfileInjection(envs *map[string]string) { } } +func DestroyLoadBalancerByName(elbName string) error { + awsConfig, err := NewAws() + if err != nil { + log.Printf("Failed to load config: %v", err) + return err + } + loadBalancerNameString := string(elbName) + + if len(loadBalancerNameString) > 0 { + loadBalancerClient := elasticloadbalancing.NewFromConfig(awsConfig) + + loadBalancerInput := elasticloadbalancing.DeleteLoadBalancerInput{ + LoadBalancerName: &loadBalancerNameString, + } + + log.Printf("trying to delete load balancer %s\n", loadBalancerNameString) + + _, err = loadBalancerClient.DeleteLoadBalancer(context.Background(), &loadBalancerInput) + + if err != nil { + return err + } + + log.Printf("deleted load balancer %s\n", loadBalancerNameString) + } + + return nil +} + +func DestroySecurityGroupNyName(securityGroupName string) error { + // todo: use method approach to avoid new AWS client initializations + awsRegion := viper.GetString("aws.region") + awsProfile := viper.GetString("aws.profile") + awsConfig, err := config.LoadDefaultConfig( + context.Background(), + config.WithRegion(awsRegion), + config.WithSharedConfigProfile(awsProfile), + ) + if err != nil { + log.Println("error: ", err) + } + + if len(securityGroupName) > 0 { + securityGroupClient := ec2.NewFromConfig(awsConfig) + + securityGroupInput := ec2.DeleteSecurityGroupInput{ + + GroupName: &securityGroupName, + } + + log.Printf("trying to delete security group %s\n", securityGroupName) + + _, err = securityGroupClient.DeleteSecurityGroup(context.Background(), &securityGroupInput) + if err != nil { + return err + } + + log.Printf("deleted security group %s\n", securityGroupName) + } + + return nil +} func DestroyLoadBalancer(clusterName string) error { // todo: use method approach to avoid new AWS client initializations awsConfig, err := NewAws() @@ -1070,3 +1132,11 @@ func DestroySecurityGroup(clusterName string) error { return nil } + +func GetELBDetails(ingressHost string) (string, string, error) { + elb := strings.Split(ingressHost, "-")[0] + securityGroup := "k8s-elb-" + elb + + return elb, securityGroup, nil + +} diff --git a/internal/k8s/kubernetes.go b/internal/k8s/kubernetes.go index 5563d522b..f055a9b32 100644 --- a/internal/k8s/kubernetes.go +++ b/internal/k8s/kubernetes.go @@ -458,3 +458,18 @@ func SetArgocdCreds(dryRun bool) { viper.Set("argocd.admin.username", "admin") viper.WriteConfig() } + +func GetIngressHost(k8sClient *kubernetes.Clientset, namespace string, name string) string { + + ingress, err := k8sClient.NetworkingV1().Ingresses(namespace).Get(context.TODO(), name, metaV1.GetOptions{}) + if err != nil { + log.Println(fmt.Sprintf("error getting key: %s from ingress: %s", namespace, name), err) + } + + if ingress != nil { + ingressLB := ingress.Status.LoadBalancer.Ingress[0] + return ingressLB.Hostname + + } + return "" +} From f05fb02fe06150ba3c76e46bbd9e57d24407a394 Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:37:36 +0000 Subject: [PATCH 2/8] update terraform Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/terraform/terraform.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index f1e79fbff..2cf88c6bf 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -12,6 +12,7 @@ import ( "github.com/kubefirst/kubefirst/configs" "github.com/kubefirst/kubefirst/internal/aws" + "github.com/kubefirst/kubefirst/internal/k8s" "github.com/kubefirst/kubefirst/pkg" "github.com/spf13/viper" ) @@ -196,7 +197,18 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { envs["TF_VAR_instance_type"] = "t4g.medium" } - err = aws.DestroyLoadBalancer(viper.GetString("cluster-name")) + clientset, err := k8s.GetClientSet(false) + if err != nil { + log.Panic(err.Error()) + } + host := k8s.GetIngressHost(clientset, "argocd", "argocd-server") + elb, security_group, _ := aws.GetELBDetails(host) + + err = aws.DestroyLoadBalancerByName(elb) + if err != nil { + log.Panicf("Failed to destroy load balancer: %v", err) + } + err = aws.DestroySecurityGroupNyName(security_group) if err != nil { log.Panicf("Failed to destroy load balancer: %v", err) } From eecc6e7b383aa2ddf7740fbb24e4b908147b4b15 Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:39:29 +0000 Subject: [PATCH 3/8] update terraform Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/terraform/terraform.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index 2cf88c6bf..3eff50936 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -210,7 +210,7 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { } err = aws.DestroySecurityGroupNyName(security_group) if err != nil { - log.Panicf("Failed to destroy load balancer: %v", err) + log.Panicf("Failed to destroy load balancer security group: %v", err) } time.Sleep(45 * time.Second) From f61b305074371ae0015d882bd812004217ebbb55 Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:43:21 +0000 Subject: [PATCH 4/8] update terraform Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/terraform/terraform.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index 3eff50936..cc70e1ca7 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -197,9 +197,9 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { envs["TF_VAR_instance_type"] = "t4g.medium" } - clientset, err := k8s.GetClientSet(false) + clientset, err := k8s.GetClientSet(skipBaseTerraform) if err != nil { - log.Panic(err.Error()) + log.Panicf("Failed to get kubectl client: %v", err) } host := k8s.GetIngressHost(clientset, "argocd", "argocd-server") elb, security_group, _ := aws.GetELBDetails(host) From b35c04387ade24b3bc1096c348b861a89d741a9c Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 19:46:27 +0000 Subject: [PATCH 5/8] update terraform Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/k8s/kubernetes.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/k8s/kubernetes.go b/internal/k8s/kubernetes.go index f055a9b32..5281b3f01 100644 --- a/internal/k8s/kubernetes.go +++ b/internal/k8s/kubernetes.go @@ -467,9 +467,10 @@ func GetIngressHost(k8sClient *kubernetes.Clientset, namespace string, name stri } if ingress != nil { - ingressLB := ingress.Status.LoadBalancer.Ingress[0] - return ingressLB.Hostname - + if len(ingress.Status.LoadBalancer.Ingress) > 0 { + ingressLB := ingress.Status.LoadBalancer.Ingress[0] + return ingressLB.Hostname + } } return "" } From abc3a98ddf657bed8fc0965a9eda7d0c1467470a Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 20:03:16 +0000 Subject: [PATCH 6/8] revert sg delete mode Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/terraform/terraform.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index cc70e1ca7..31eb00da8 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -208,10 +208,13 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { if err != nil { log.Panicf("Failed to destroy load balancer: %v", err) } - err = aws.DestroySecurityGroupNyName(security_group) - if err != nil { - log.Panicf("Failed to destroy load balancer security group: %v", err) - } + /* + Removed for now, to be fixed later. + err = aws.DestroySecurityGroupNyName(security_group) + if err != nil { + log.Panicf("Failed to destroy load balancer security group: %v", err) + } + */ time.Sleep(45 * time.Second) err = pkg.ExecShellWithVars(envs, config.TerraformClientPath, "init") From 47ed13590d641dcdb075a7cb27b57d0e865f707f Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Wed, 30 Nov 2022 20:29:18 +0000 Subject: [PATCH 7/8] trying to get vpc Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- internal/aws/aws.go | 37 +++++++++++++++++++++++++++++++++ internal/terraform/terraform.go | 2 ++ 2 files changed, 39 insertions(+) diff --git a/internal/aws/aws.go b/internal/aws/aws.go index 17808bd0a..b67b33f7d 100644 --- a/internal/aws/aws.go +++ b/internal/aws/aws.go @@ -19,6 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2Types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancing" "github.com/aws/aws-sdk-go-v2/service/route53" route53Types "github.com/aws/aws-sdk-go-v2/service/route53/types" @@ -1140,3 +1141,39 @@ func GetELBDetails(ingressHost string) (string, string, error) { return elb, securityGroup, nil } + +func GetVPC(clusterName string) string { + awsConfig, err := NewAws() + if err != nil { + log.Printf("Failed to load config: %v", err) + } + + ec2Client := ec2.NewFromConfig(awsConfig) + + filterType := "tag:ClusterName" + vpcData, err := ec2Client.DescribeVpcs(context.Background(), &ec2.DescribeVpcsInput{ + Filters: []ec2Types.Filter{ + { + Name: &filterType, + Values: []string{clusterName}, + }, + }, + }) + if err != nil { + log.Printf("%v", err) + } + + if len(vpcData.Vpcs) > 0 { + log.Printf("there is no VPC for the cluster %q", clusterName) + } + + for _, v := range vpcData.Vpcs { + log.Printf("%s", v) + log.Print("vpc:", &v.VpcId) + log.Print("vpc:", v.State) + if v.State == "available" { + log.Printf("there is a VPC for the %q cluster, but the status is not available", clusterName) + } + } + return "" +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index 31eb00da8..503dd947d 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -203,6 +203,8 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { } host := k8s.GetIngressHost(clientset, "argocd", "argocd-server") elb, security_group, _ := aws.GetELBDetails(host) + log.Println("ELB in use:", elb) + log.Println("Security Group in use:", security_group) err = aws.DestroyLoadBalancerByName(elb) if err != nil { From 456616bd77b1345b045c8fe6f3b36b1b9347b6e9 Mon Sep 17 00:00:00 2001 From: 6za <53096417+6za@users.noreply.github.com> Date: Thu, 1 Dec 2022 14:20:36 +0000 Subject: [PATCH 8/8] improve destroy aws Signed-off-by: 6za <53096417+6za@users.noreply.github.com> --- cmd/create.go | 8 +++ internal/aws/aws.go | 89 ++++++++++++++++++++++++++++++--- internal/terraform/terraform.go | 29 +++-------- 3 files changed, 99 insertions(+), 27 deletions(-) diff --git a/cmd/create.go b/cmd/create.go index 0698188f1..95c5c1b7e 100644 --- a/cmd/create.go +++ b/cmd/create.go @@ -10,6 +10,7 @@ import ( "time" "github.com/kubefirst/kubefirst/configs" + "github.com/kubefirst/kubefirst/internal/aws" "github.com/kubefirst/kubefirst/internal/domain" "github.com/kubefirst/kubefirst/internal/handlers" "github.com/kubefirst/kubefirst/internal/services" @@ -202,6 +203,13 @@ cluster provisioning process spinning up the services, and validates the livenes } if viper.GetString("cloud") == flagset.CloudAws { + //POST-install aws cloud census + elbName, sg := aws.GetELBByClusterName(viper.GetString("cluster-name")) + viper.Set("aws.vpcid", aws.GetVPCIdByClusterName(viper.GetString("cluster-name"))) + viper.Set("aws.elb.name", elbName) + viper.Set("aws.elb.sg", sg) + viper.WriteConfig() + err = state.UploadKubefirstToStateStore(globalFlags.DryRun) if err != nil { log.Println(err) diff --git a/internal/aws/aws.go b/internal/aws/aws.go index b67b33f7d..9e9eb7195 100644 --- a/internal/aws/aws.go +++ b/internal/aws/aws.go @@ -1023,7 +1023,41 @@ func DestroyLoadBalancerByName(elbName string) error { return nil } -func DestroySecurityGroupNyName(securityGroupName string) error { +func DestroySecurityGroupById(securityGroupId string) error { + // todo: use method approach to avoid new AWS client initializations + awsRegion := viper.GetString("aws.region") + awsProfile := viper.GetString("aws.profile") + awsConfig, err := config.LoadDefaultConfig( + context.Background(), + config.WithRegion(awsRegion), + config.WithSharedConfigProfile(awsProfile), + ) + if err != nil { + log.Println("error: ", err) + } + + if len(securityGroupId) > 0 { + securityGroupClient := ec2.NewFromConfig(awsConfig) + + securityGroupInput := ec2.DeleteSecurityGroupInput{ + + GroupId: aws.String(securityGroupId), + } + + log.Printf("trying to delete security group %s\n", securityGroupId) + + _, err = securityGroupClient.DeleteSecurityGroup(context.Background(), &securityGroupInput) + if err != nil { + return err + } + + log.Printf("deleted security group %s\n", securityGroupId) + } + + return nil +} + +func DestroySecurityGroupByName(securityGroupName string) error { // todo: use method approach to avoid new AWS client initializations awsRegion := viper.GetString("aws.region") awsProfile := viper.GetString("aws.profile") @@ -1142,7 +1176,7 @@ func GetELBDetails(ingressHost string) (string, string, error) { } -func GetVPC(clusterName string) string { +func GetVPCIdByClusterName(clusterName string) string { awsConfig, err := NewAws() if err != nil { log.Printf("Failed to load config: %v", err) @@ -1168,12 +1202,55 @@ func GetVPC(clusterName string) string { } for _, v := range vpcData.Vpcs { - log.Printf("%s", v) - log.Print("vpc:", &v.VpcId) - log.Print("vpc:", v.State) + vpcId := aws.ToString(v.VpcId) if v.State == "available" { - log.Printf("there is a VPC for the %q cluster, but the status is not available", clusterName) + //it is only expected to have 1 vpc per cluster name + log.Printf("there is a VPC for the %q cluster, the vpcID is %s", clusterName, vpcId) + return vpcId } } return "" } + +// GetELBByClusterName return the elb name and its security groups +func GetELBByClusterName(clusterName string) (string, []string) { + awsConfig, err := NewAws() + if err != nil { + log.Printf("Failed to load config: %v", err) + } + + loadBalancerClient := elasticloadbalancing.NewFromConfig(awsConfig) + + elbs, err := loadBalancerClient.DescribeLoadBalancers(context.Background(), &elasticloadbalancing.DescribeLoadBalancersInput{}) + if err != nil { + log.Printf("%v", err) + } + if len(elbs.LoadBalancerDescriptions) > 0 { + log.Println("there is no ELB for the cluster ", clusterName) + } + + for _, elb := range elbs.LoadBalancerDescriptions { + elbName := aws.ToString(elb.LoadBalancerName) + tags, err := loadBalancerClient.DescribeTags(context.Background(), &elasticloadbalancing.DescribeTagsInput{ + LoadBalancerNames: []string{elbName}, + }) + if err != nil { + log.Printf("%v", err) + } + for _, tagDesc := range tags.TagDescriptions { + for _, tag := range tagDesc.Tags { + key := aws.ToString(tag.Key) + value := aws.ToString(tag.Value) + if value == "owned" && key == fmt.Sprintf("kubernetes.io/cluster/%s", clusterName) { + log.Println("Match tag:", key, value) + log.Println("Match ELB Name:", elbName) + log.Println("Match ELB SG:", elb.SecurityGroups) + //found the right ELB + return elbName, elb.SecurityGroups + } + + } + } + } + return "", []string{""} +} diff --git a/internal/terraform/terraform.go b/internal/terraform/terraform.go index 503dd947d..0b99bf3bc 100644 --- a/internal/terraform/terraform.go +++ b/internal/terraform/terraform.go @@ -12,7 +12,6 @@ import ( "github.com/kubefirst/kubefirst/configs" "github.com/kubefirst/kubefirst/internal/aws" - "github.com/kubefirst/kubefirst/internal/k8s" "github.com/kubefirst/kubefirst/pkg" "github.com/spf13/viper" ) @@ -197,26 +196,10 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { envs["TF_VAR_instance_type"] = "t4g.medium" } - clientset, err := k8s.GetClientSet(skipBaseTerraform) - if err != nil { - log.Panicf("Failed to get kubectl client: %v", err) - } - host := k8s.GetIngressHost(clientset, "argocd", "argocd-server") - elb, security_group, _ := aws.GetELBDetails(host) - log.Println("ELB in use:", elb) - log.Println("Security Group in use:", security_group) - - err = aws.DestroyLoadBalancerByName(elb) + err = aws.DestroyLoadBalancerByName(viper.GetString("aws.elb.name")) if err != nil { log.Panicf("Failed to destroy load balancer: %v", err) } - /* - Removed for now, to be fixed later. - err = aws.DestroySecurityGroupNyName(security_group) - if err != nil { - log.Panicf("Failed to destroy load balancer security group: %v", err) - } - */ time.Sleep(45 * time.Second) err = pkg.ExecShellWithVars(envs, config.TerraformClientPath, "init") @@ -229,9 +212,13 @@ func DestroyBaseTerraform(skipBaseTerraform bool) { log.Printf("failed to terraform destroy base %v", err) } - err = aws.DestroySecurityGroup(viper.GetString("cluster-name")) - if err != nil { - log.Panicf("Failed to destroy security group: %v", err) + //destroy all found sg + for _, sg := range viper.GetStringSlice("aws.elb.sg") { + log.Println("Removing Security Group:", sg) + err = aws.DestroySecurityGroupById(sg) + if err != nil { + log.Panicf("Failed to destroy security group: %v", err) + } } err = pkg.ExecShellWithVars(envs, config.TerraformClientPath, "init")