Merge pull request #408 from zhengxiexie/clean_dlb_vpc_dev

Re-implemented DLB cleanup in nsx-operator

pkg/clean/clean.go

@@ -27,6 +27,7 @@ var log = logger.Log
 
 // Clean cleans up NSX resources,
 // including security policy, static route, subnet, subnet port, subnet set, vpc, ip pool, nsx service account
+// besides, it also cleans up DLB resources, which was previously implemented in nsx-ncp,
 // it is usually used when nsx-operator is uninstalled and remove all the resources created by nsx-operator
 // return error if any, return nil if no error
 // the error type include followings:
@@ -43,7 +44,7 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig) error {
 	if nsxClient == nil {
 		return nsxutil.GetNSXClientFailed
 	}
-	if cleanupService, err := InitializeCleanupService(cf); err != nil {
+	if cleanupService, err := InitializeCleanupService(cf, nsxClient); err != nil {
 		return errors.Join(nsxutil.InitCleanupServiceFailed, err)
 	} else if cleanupService.err != nil {
 		return errors.Join(nsxutil.InitCleanupServiceFailed, cleanupService.err)
@@ -54,6 +55,22 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig) error {
 			}
 		}
 	}
+	// delete DLB group -> delete virtual servers -> DLB services -> DLB pools -> persistent profiles for DLB
+	if err := retry.OnError(retry.DefaultRetry, func(err error) bool {
+		if err != nil {
+			log.Info("retrying to clean up DLB resources", "error", err)
+			return true
+		}
+		return false
+	}, func() error {
+		if err := CleanDLB(ctx, nsxClient.Cluster, cf); err != nil {
+			return fmt.Errorf("failed to clean up specific resource: %w", err)
+		}
+		return nil
+	}); err != nil {
+		return err
+	}
+
 	log.Info("cleanup NSX resources successfully")
 	return nil
 }
@@ -76,14 +93,9 @@ func wrapCleanFunc(ctx context.Context, clean cleanup) func() error {
 }
 
 // InitializeCleanupService initializes all the CR services
-func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, error) {
+func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Client) (*CleanupService, error) {
 	cleanupService := NewCleanupService()
 
-	nsxClient := nsx.GetClient(cf)
-	if nsxClient == nil {
-		return cleanupService, fmt.Errorf("failed to get nsx client")
-	}
-
 	var commonService = common.Service{
 		NSXClient: nsxClient,
 		NSXConfig: cf,

pkg/clean/clean_dlb.go

@@ -0,0 +1,94 @@
+package clean
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	neturl "net/url"
+	"strings"
+
+	"github.com/vmware-tanzu/nsx-operator/pkg/config"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
+)
+
+type (
+	mapInterface = map[string]interface{}
+)
+
+const TagDLB = "DLB"
+
+func appendIfNotExist(slice []string, s string) []string {
+	for _, item := range slice {
+		if item == s {
+			return slice
+		}
+	}
+	return append(slice, s)
+}
+
+func httpQueryDLBResources(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) {
+	queryParam := "resource_type:" + resource +
+		"&tags.scope:ncp\\/cluster" +
+		"&tags.tag:" + cf.Cluster +
+		"&tags.scope:ncp\\/created_for" +
+		"&tags.tag:" + TagDLB
+
+	pairs := strings.Split(queryParam, "&")
+	params := make(map[string]string)
+	for _, pair := range pairs {
+		kv := strings.Split(pair, ":")
+		if len(kv) == 2 {
+			params[kv[0]] = kv[1]
+		}
+	}
+	var encodedPairs []string
+	for key, value := range params {
+		encodedKey := neturl.QueryEscape(key)
+		encodedValue := neturl.QueryEscape(value)
+		encodedPairs = append(encodedPairs, fmt.Sprintf("%s:%s", encodedKey, encodedValue))
+	}
+
+	encodedQuery := strings.Join(encodedPairs, "%20AND%20")
+	url := "policy/api/v1/search/query?query=" + encodedQuery
+
+	resp, err := cluster.HttpGet(url)
+	if err != nil {
+		return nil, err
+	}
+	var resourcePath []string
+	for _, item := range resp["results"].([]interface{}) {
+		resourcePath = appendIfNotExist(resourcePath, item.(mapInterface)["path"].(string))
+	}
+	return resourcePath, nil
+}
+
+func CleanDLB(ctx context.Context, cluster *nsx.Cluster, cf *config.NSXOperatorConfig) error {
+	log.Info("Deleting DLB resources started")
+
+	resources := []string{"Group", "LBVirtualServer", "LBService", "LBPool", "LBCookiePersistenceProfile"}
+	var allPaths []string
+
+	for _, resource := range resources {
+		paths, err := httpQueryDLBResources(cluster, cf, resource)
+		if err != nil {
+			return err
+		}
+		log.Info(resource, "count", len(paths))
+		allPaths = append(allPaths, paths...)
+	}
+
+	log.Info("Deleting DLB resources", "paths", allPaths)
+	for _, path := range allPaths {
+		url := "policy/api/v1" + path
+		select {
+		case <-ctx.Done():
+			return errors.Join(nsxutil.TimeoutFailed, ctx.Err())
+		default:
+			if err := cluster.HttpDelete(url); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/nsx/client.go

@@ -49,6 +49,7 @@ var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_AC
 type Client struct {
 	NsxConfig     *config.NSXOperatorConfig
 	RestConnector *client.RestConnector
+	Cluster       *Cluster
 
 	QueryClient    search.QueryClient
 	GroupClient    domains.GroupsClient
@@ -174,14 +175,14 @@ func GetClient(cf *config.NSXOperatorConfig) *Client {
 	}
 
 	nsxClient := &Client{
-		NsxConfig:      cf,
-		RestConnector:  restConnector(cluster),
-		QueryClient:    queryClient,
-		GroupClient:    groupClient,
-		SecurityClient: securityClient,
-		RuleClient:     ruleClient,
-		InfraClient:    infraClient,
-
+		NsxConfig:                  cf,
+		RestConnector:              restConnector(cluster),
+		QueryClient:                queryClient,
+		GroupClient:                groupClient,
+		SecurityClient:             securityClient,
+		RuleClient:                 ruleClient,
+		InfraClient:                infraClient,
+		Cluster:                    cluster,
 		ClusterControlPlanesClient: clusterControlPlanesClient,
 		HostTransPortNodesClient:   hostTransportNodesClient,
 		RealizedEntitiesClient:     realizedEntitiesClient,

pkg/nsx/cluster.go

@@ -363,19 +363,13 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) {
 		return nil, err
 	}
 	log.V(1).Info("Get url", "url", req.URL)
-	err = ep.UpdateHttpRequestAuth(req)
-	if err != nil {
-		log.Error(err, "http get update auth error")
-		return nil, err
-	}
-	ep.UpdateCAforEnvoy(req)
 	resp, err := ep.client.Do(req)
 	if err != nil {
 		log.Error(err, "failed to do http GET operation")
 		return nil, err
 	}
-	var respJson map[string]interface{}
-	err, _ = util.HandleHTTPResponse(resp, respJson, true)
+	respJson := make(map[string]interface{})
+	err, _ = util.HandleHTTPResponse(resp, &respJson, true)
 	return respJson, err
 }
 
@@ -390,12 +384,6 @@ func (cluster *Cluster) HttpDelete(url string) error {
 		return err
 	}
 	log.V(1).Info("Delete url", "url", req.URL)
-	err = ep.UpdateHttpRequestAuth(req)
-	if err != nil {
-		log.Error(err, "http get update auth error")
-		return err
-	}
-	ep.UpdateCAforEnvoy(req)
 	_, err = ep.client.Do(req)
 	if err != nil {
 		log.Error(err, "failed to do http DELETE operation")