From d4c8dca26007089b9bd9cf3fe64b8bab15fdee8d Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Wed, 26 Oct 2022 15:48:20 +0800 Subject: [PATCH 01/12] Refactor compare.go Extract interface Comparable, and only need to implement methods in sub-resource. --- pkg/nsx/services/securitypolicy/compare_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/nsx/services/securitypolicy/compare_test.go b/pkg/nsx/services/securitypolicy/compare_test.go index 137d68e71..4311c0265 100644 --- a/pkg/nsx/services/securitypolicy/compare_test.go +++ b/pkg/nsx/services/securitypolicy/compare_test.go @@ -216,4 +216,4 @@ func TestSecurityPolicyEqual(t *testing.T) { }, ) } -} +} \ No newline at end of file From 83b7a75e5cf9db6baa20f9f55cf9ba8aa4d96968 Mon Sep 17 00:00:00 2001 From: Deng Yun Date: Mon, 7 Nov 2022 18:27:35 +0800 Subject: [PATCH 02/12] Sync main branch to vpc_dev Sync main branch to vpc_dev to keep both branches consistency. This is base point for future vpc_dev code rebase back to main branch. --- pkg/nsx/services/securitypolicy/compare_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/nsx/services/securitypolicy/compare_test.go b/pkg/nsx/services/securitypolicy/compare_test.go index 4311c0265..137d68e71 100644 --- a/pkg/nsx/services/securitypolicy/compare_test.go +++ b/pkg/nsx/services/securitypolicy/compare_test.go @@ -216,4 +216,4 @@ func TestSecurityPolicyEqual(t *testing.T) { }, ) } -} \ No newline at end of file +} From 733a3bbcf6ff25ffafd70f1f294035a523636bc4 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Mon, 20 Mar 2023 14:36:17 +0800 Subject: [PATCH 03/12] Add GetOrgProject method to ServiceMediator Create the common method used by other modules. When using gomonkey to test the method, encounter the problem that the mocked method doesn't take effect, fix it by adding -gcflags=all=-l option to prohibit inline optimization. --- pkg/nsx/services/mediator/mediator_test.go | 31 ++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 pkg/nsx/services/mediator/mediator_test.go diff --git a/pkg/nsx/services/mediator/mediator_test.go b/pkg/nsx/services/mediator/mediator_test.go new file mode 100644 index 000000000..5045cdd47 --- /dev/null +++ b/pkg/nsx/services/mediator/mediator_test.go @@ -0,0 +1,31 @@ +package mediator + +import ( + "reflect" + "testing" + + "github.com/agiledragon/gomonkey" + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" +) + +func TestServiceMediator_GetOrgProject(t *testing.T) { + vpcService := &vpc.VPCService{} + vs := &ServiceMediator{ + SecurityPolicyService: nil, + VPCService: vpcService, + } + + patches := gomonkey.ApplyMethod(reflect.TypeOf(vpcService), "GetVPCsByNamespace", func(_ *vpc.VPCService, ns string) []model.Vpc { + return []model.Vpc{{Path: common.String("/orgs/default/projects/project-1/vpcs/vpc-1")}} + }) + defer patches.Reset() + + got := vs.GetOrgProjectVPC("ns")[0] + want := common.OrgProjectVPC{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1"} + if !reflect.DeepEqual(got, want) { + t.Errorf("GetOrgProject() = %v, want %v", got, want) + } +} From f9c46dbb6e37a8c0f69e4f8f65e76b0c1f64def8 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Fri, 13 Jan 2023 14:59:50 +0800 Subject: [PATCH 04/12] Add VPC search store --- pkg/nsx/services/common/compare.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/nsx/services/common/compare.go b/pkg/nsx/services/common/compare.go index f8b259355..4d9166775 100644 --- a/pkg/nsx/services/common/compare.go +++ b/pkg/nsx/services/common/compare.go @@ -41,6 +41,8 @@ func CompareResources(existing []Comparable, expected []Comparable) (changed []C if existed_item, ok := existingMap[key]; ok { if isChanged := CompareResource(existed_item, expected_item); !isChanged { continue + } else { + log.V(1).Info("resource changed", "existing", existed_item, "expected", expected_item) } log.V(1).Info("resource changed", "existing", existed_item, "expected", expected_item) } From b2fa16acc95b1691d5d7531ba256916944d277db Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Mon, 29 May 2023 13:16:45 +0800 Subject: [PATCH 05/12] Replace OrgProjectVPC with VPCInfo Refactor the name convention. --- pkg/nsx/services/mediator/mediator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/nsx/services/mediator/mediator_test.go b/pkg/nsx/services/mediator/mediator_test.go index 5045cdd47..38d154a19 100644 --- a/pkg/nsx/services/mediator/mediator_test.go +++ b/pkg/nsx/services/mediator/mediator_test.go @@ -23,8 +23,8 @@ func TestServiceMediator_GetOrgProject(t *testing.T) { }) defer patches.Reset() - got := vs.GetOrgProjectVPC("ns")[0] - want := common.OrgProjectVPC{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1"} + got := vs.GetVPCInfo("ns")[0] + want := common.VPCInfo{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1"} if !reflect.DeepEqual(got, want) { t.Errorf("GetOrgProject() = %v, want %v", got, want) } From b639acc339fa91696724c28a87b92917a7d4a83e Mon Sep 17 00:00:00 2001 From: Qian Sun Date: Tue, 25 Jul 2023 09:40:12 +0000 Subject: [PATCH 06/12] Add a function to parse the path and obtain the org/project/vpc/parent ID --- pkg/nsx/services/mediator/mediator_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/nsx/services/mediator/mediator_test.go b/pkg/nsx/services/mediator/mediator_test.go index 38d154a19..69eae18d2 100644 --- a/pkg/nsx/services/mediator/mediator_test.go +++ b/pkg/nsx/services/mediator/mediator_test.go @@ -23,8 +23,8 @@ func TestServiceMediator_GetOrgProject(t *testing.T) { }) defer patches.Reset() - got := vs.GetVPCInfo("ns")[0] - want := common.VPCInfo{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1"} + got := vs.ListVPCInfo("ns")[0] + want := common.VPCResourceInfo{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1", ID: "vpc-1", ParentID: "project-1"} if !reflect.DeepEqual(got, want) { t.Errorf("GetOrgProject() = %v, want %v", got, want) } From 29cc5f5be44a8c9921e917876738237bfbc5bdeb Mon Sep 17 00:00:00 2001 From: jsui Date: Wed, 16 Nov 2022 12:56:37 +0800 Subject: [PATCH 07/12] Implement subnet/subnetset CRD Add subnet/subnetset controller to reconcile subnet/subnetset CRD. And add subnet service to perform NSX subnet CRUD operations. --- .../subnetset/subnetport_handler.go | 88 ++++++++++++++ pkg/controllers/subnetset/vpc_handler.go | 113 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 pkg/controllers/subnetset/subnetport_handler.go create mode 100644 pkg/controllers/subnetset/vpc_handler.go diff --git a/pkg/controllers/subnetset/subnetport_handler.go b/pkg/controllers/subnetset/subnetport_handler.go new file mode 100644 index 000000000..a0d545545 --- /dev/null +++ b/pkg/controllers/subnetset/subnetport_handler.go @@ -0,0 +1,88 @@ +package subnetset + +import ( + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +// SubnetPortHandler supports lazy-creation of Subnet, the first Subnet won't +// be created until there is a SubnetPort attached to it. +// - SubnetPort creation: get available Subnet for the SubnetPort, create new +// Subnet if necessary. +// - SubnetPort deletion: if recycling Subnet is required, delete Subnets without +// SubnetPort attached to it. + +type SubnetPortHandler struct { + Reconciler *SubnetSetReconciler +} + +//TODO Remove this handler when confirmed that SubnetPort could get allocated subnet via the interface +// subnetservice.GetAvailableSubnet + +// Create allocates Subnet for SubnetPort from SubnetSet. +func (h *SubnetPortHandler) Create(e event.CreateEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("SubnetPort generic event, do nothing") + //subnetPort := e.Object.(*v1alpha1.SubnetPort) + //if subnetPort.Spec.Subnet != "" { + // // Two possible scenarios: + // // - 1. User uses `.Spec.Subnet` directly instead of `.Spec.SubnetSet`. + // // - 2. Subnet has been allocated and `.Spec.Subnet` is rendered by SubnetPortHandler. + // return + //} + //subnetSet := &v1alpha1.SubnetSet{} + //key := types.NamespacedName{ + // Namespace: subnetPort.GetNamespace(), + // Name: subnetPort.Spec.SubnetSet, + //} + //if err := h.Reconciler.Client.Get(context.Background(), key, subnetSet); err != nil { + // log.Error(err, "failed to get SubnetSet", "ns", key.Namespace, "name", key.Name) + // return + //} + //log.Info("allocating Subnet for SubnetPort") + //vpcList := &v1alpha1.VPCList{} + //if err := h.Reconciler.Client.List(context.Background(), vpcList, client.InNamespace(subnetPort.GetNamespace())); err != nil { + // log.Error(err, fmt.Sprintf("failed to get VPC under namespace: %s.\n", subnetPort.GetNamespace())) + // return + //} + //vpcInfo, err := servicecommon.ParseVPCResourcePath(vpcList.Items[0].Status.NSXResourcePath) + //if err != nil { + // log.Error(err, "failed to resolve VPC info") + // return + //} + //_, err = h.Reconciler.getAvailableSubnet(subnetSet, &vpcInfo) + //if err != nil { + // log.Error(err, "failed to allocate Subnet") + //} + // TODO return subnetport id to caller. +} + +// Delete TODO Implement this method if required to recycle Subnet without SubnetPort attached. +func (h *SubnetPortHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("SubnetPort generic event, do nothing") +} + +func (h *SubnetPortHandler) Generic(_ event.GenericEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("SubnetPort generic event, do nothing") +} + +func (h *SubnetPortHandler) Update(_ event.UpdateEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("SubnetPort update event, do nothing") +} + +var SubnetPortPredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + // TODO When recycling Subnet is required, return true. + return false + }, + GenericFunc: func(genericEvent event.GenericEvent) bool { + return false + }, +} diff --git a/pkg/controllers/subnetset/vpc_handler.go b/pkg/controllers/subnetset/vpc_handler.go new file mode 100644 index 000000000..71f054989 --- /dev/null +++ b/pkg/controllers/subnetset/vpc_handler.go @@ -0,0 +1,113 @@ +package subnetset + +import ( + "context" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + + "sigs.k8s.io/controller-runtime/pkg/predicate" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" +) + +// VPCHandler handles VPC event for SubnetSet: +// - VPC creation: create default SubnetSet for the VPC. +// - VPC deletion: delete all SubnetSets under the VPC. + +var defaultSubnetSets = map[string]string{ + "default-vm-subnetset": common.LabelDefaultVMSubnet, + "default-pod-subnetset": common.LabelDefaultPodSubnetSet, +} + +type VPCHandler struct { + Client client.Client +} + +func (h *VPCHandler) Create(e event.CreateEvent, _ workqueue.RateLimitingInterface) { + ns := e.Object.GetNamespace() + log.Info("creating default Subnetset for VPC", "Namespace", ns, "Name", e.Object.GetName()) + for name, subnetSetType := range defaultSubnetSets { + if err := retry.OnError(retry.DefaultRetry, func(err error) bool { + return err != nil + }, func() error { + list := &v1alpha1.SubnetSetList{} + label := client.MatchingLabels{ + common.LabelDefaultSubnetSet: subnetSetType, + } + nsOption := client.InNamespace(ns) + if err := h.Client.List(context.Background(), list, label, nsOption); err != nil { + return err + } + if len(list.Items) > 0 { + // avoid creating when nsx-operator restarted if Subnetset exists. + log.Info("default subnetset already exists", common.LabelDefaultSubnetSet, subnetSetType) + return nil + } + obj := &v1alpha1.SubnetSet{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns, + Name: name, + Labels: map[string]string{ + common.LabelDefaultSubnetSet: subnetSetType, + }, + }, + Spec: v1alpha1.SubnetSetSpec{}, + } + if err := h.Client.Create(context.Background(), obj); err != nil { + return err + } + return nil + }); err != nil { + log.Error(err, "failed to create SubnetSet", "Namespace", ns, "Name", name) + } + } +} + +func (h *VPCHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) { + log.Info("cleaning default Subnetset for VPC", "Name", e.Object.GetName()) + for _, subnetSetType := range defaultSubnetSets { + if err := retry.OnError(retry.DefaultRetry, func(err error) bool { + return err != nil + }, func() error { + label := client.MatchingLabels{ + common.LabelDefaultSubnetSet: subnetSetType, + } + nsOption := client.InNamespace(e.Object.GetNamespace()) + obj := &v1alpha1.SubnetSet{} + if err := h.Client.DeleteAllOf(context.Background(), obj, label, nsOption); err != nil { + return client.IgnoreNotFound(err) + } + return nil + }); err != nil { + log.Error(err, "failed to delete SubnetSet", common.LabelDefaultSubnetSet, subnetSetType) + } + } +} + +func (h *VPCHandler) Generic(_ event.GenericEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("VPC generic event, do nothing") +} + +func (h *VPCHandler) Update(_ event.UpdateEvent, _ workqueue.RateLimitingInterface) { + log.V(4).Info("VPC update event, do nothing") +} + +var VPCPredicate = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return true + }, + GenericFunc: func(genericEvent event.GenericEvent) bool { + return false + }, +} From 2a1c62fc74ba4d6617b8a92d2b9c337fc6b01ebb Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Tue, 15 Aug 2023 14:48:04 +0800 Subject: [PATCH 08/12] Support passing user/pass/ca from parameter Default nsx-operator reloads user/name/ca from file. Support to pass user/pass/ca from parameter --- pkg/config/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 2c59b4cea..43473841e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -281,6 +281,12 @@ func (vcConfig *VCConfig) validate() error { configLog.Info("validate VcConfig failed VCUser %s VCPassword %s", vcConfig.VCUser, vcConfig.VCPassword) return err } + // VCPassword, VCUser should be both empty or valid + if !((len(vcConfig.VCPassword) > 0) == (len(vcConfig.VCUser) > 0)) { + err := errors.New("invalid field " + "VCUser, VCPassword") + log.Info("validate VcConfig failed", "VCUser", vcConfig.VCUser, "VCPassword", vcConfig.VCPassword) + return err + } return nil } From ba21425253a25e1c1cdcc71bc19f3427a21c9e9d Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Tue, 29 Aug 2023 16:06:49 +0800 Subject: [PATCH 09/12] Add cleanup for ippool and subnet Add cleanup for ippool and subnet as securitypolicy does. --- pkg/clean/types.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pkg/clean/types.go b/pkg/clean/types.go index 40c5e43f8..48264f855 100644 --- a/pkg/clean/types.go +++ b/pkg/clean/types.go @@ -31,3 +31,29 @@ func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService { c.cleans = append(c.cleans, clean) return c } + +type cleanupFunc func() (cleanup, error) + +type CleanupService struct { + cleans []cleanup + err error +} + +func NewCleanupService() *CleanupService { + return &CleanupService{} +} + +func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService { + var clean cleanup + if c.err != nil { + return c + } + + clean, c.err = f() + if c.err != nil { + return c + } + + c.cleans = append(c.cleans, clean) + return c +} From 1803f2fdb56ef4cd343a57729efe7940d2422bd6 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Mon, 7 Aug 2023 17:12:43 +0800 Subject: [PATCH 10/12] Refactor loglevel based on debug in configmap We hold these truths to be self-evident: log.info -> info, log.v(1).info -> debug, log.error-> error, log.v(2).info-> develop level log If config.Debug=true, then loglevel=1, if command is passed in loglevel and greater than debug loglevel, then the loglevel is overridden. Use another log in config.go. --- cmd_clean/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd_clean/main.go b/cmd_clean/main.go index cd3138515..ef68fef1a 100644 --- a/cmd_clean/main.go +++ b/cmd_clean/main.go @@ -63,6 +63,7 @@ func main() { flag.IntVar(&config.LogLevel, "log-level", 0, "Use zap-core log system.") flag.Parse() + logf.SetLogger(logger.ZapLogger()) cf = config.NewNSXOpertorConfig() cf.NsxApiManagers = []string{mgrIp} cf.VCUser = vcUser From 6ba6790ac346c964d6086691fbff2589e1342fe7 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Fri, 8 Sep 2023 15:47:23 +0800 Subject: [PATCH 11/12] Fix cleanup for subnet and ippool 1. Should delete subnet and subnetset, ListSubnetID should get them all. 2. Since ippool and staticroute rely on vpcService of serviceMediator, init vpc first, otherwise, it would report nil pointer when deleting ippool or staticroute. --- pkg/clean/clean.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index 49ed81537..12a98ec56 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -11,6 +11,7 @@ import ( "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/config" + commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" @@ -90,6 +91,9 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er } vpcService, vpcErr := vpc.InitializeVPC(commonService) + vpcService, vpcErr := vpc.InitializeVPC(commonService) + commonctl.ServiceMediator.VPCService = vpcService + // initialize all the CR services // Use Fluent Interface to escape error check hell From 24d80be5138aea1f34228aa4e4fe70003f399400 Mon Sep 17 00:00:00 2001 From: Xie Zheng Date: Fri, 17 Nov 2023 14:29:58 +0800 Subject: [PATCH 12/12] Re-implemented DLB cleanup in nsx-operator Delete DLB resources. The size of load balancer service can be, SMALL, MEDIUM, LARGE, XLARGE, or DLB. The first four sizes are realized on Edge node as a centralized load balancer. DLB is realized on each ESXi hypervisor as a distributed load balancer. Previously, this cleanup function was implemented in NCP nsx_policy_cleanup.py. Now, it is re-implemented in nsx-operator pkg/clean/. Signed-off-by: Xie Zheng --- cmd_clean/main.go | 1 - pkg/clean/clean.go | 30 +++-- pkg/clean/clean_dlb.go | 94 +++++++++++++++ pkg/clean/types.go | 26 ---- pkg/config/config.go | 6 - .../subnetset/subnetport_handler.go | 88 -------------- pkg/controllers/subnetset/vpc_handler.go | 113 ------------------ pkg/nsx/client.go | 17 +-- pkg/nsx/cluster.go | 16 +-- pkg/nsx/services/common/compare.go | 2 - pkg/nsx/services/mediator/mediator_test.go | 31 ----- 11 files changed, 124 insertions(+), 300 deletions(-) create mode 100644 pkg/clean/clean_dlb.go delete mode 100644 pkg/controllers/subnetset/subnetport_handler.go delete mode 100644 pkg/controllers/subnetset/vpc_handler.go delete mode 100644 pkg/nsx/services/mediator/mediator_test.go diff --git a/cmd_clean/main.go b/cmd_clean/main.go index ef68fef1a..cd3138515 100644 --- a/cmd_clean/main.go +++ b/cmd_clean/main.go @@ -63,7 +63,6 @@ func main() { flag.IntVar(&config.LogLevel, "log-level", 0, "Use zap-core log system.") flag.Parse() - logf.SetLogger(logger.ZapLogger()) cf = config.NewNSXOpertorConfig() cf.NsxApiManagers = []string{mgrIp} cf.VCUser = vcUser diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index 12a98ec56..a73007024 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -11,7 +11,6 @@ import ( "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/config" - commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" @@ -28,6 +27,7 @@ var log = logger.Log // Clean cleans up NSX resources, // including security policy, static route, subnet, subnet port, subnet set, vpc, ip pool, nsx service account +// besides, it also cleans up DLB resources, which was previously implemented in nsx-ncp, // it is usually used when nsx-operator is uninstalled and remove all the resources created by nsx-operator // return error if any, return nil if no error // the error type include followings: @@ -44,7 +44,7 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig) error { if nsxClient == nil { return nsxutil.GetNSXClientFailed } - if cleanupService, err := InitializeCleanupService(cf); err != nil { + if cleanupService, err := InitializeCleanupService(cf, nsxClient); err != nil { return errors.Join(nsxutil.InitCleanupServiceFailed, err) } else if cleanupService.err != nil { return errors.Join(nsxutil.InitCleanupServiceFailed, cleanupService.err) @@ -55,6 +55,22 @@ func Clean(ctx context.Context, cf *config.NSXOperatorConfig) error { } } } + // delete DLB group -> delete virtual servers -> DLB services -> DLB pools -> persistent profiles for DLB + if err := retry.OnError(retry.DefaultRetry, func(err error) bool { + if err != nil { + log.Info("retrying to clean up DLB resources", "error", err) + return true + } + return false + }, func() error { + if err := CleanDLB(ctx, nsxClient.Cluster, cf); err != nil { + return fmt.Errorf("failed to clean up specific resource: %w", err) + } + return nil + }); err != nil { + return err + } + log.Info("cleanup NSX resources successfully") return nil } @@ -77,23 +93,15 @@ func wrapCleanFunc(ctx context.Context, clean cleanup) func() error { } // InitializeCleanupService initializes all the CR services -func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, error) { +func InitializeCleanupService(cf *config.NSXOperatorConfig, nsxClient *nsx.Client) (*CleanupService, error) { cleanupService := NewCleanupService() - nsxClient := nsx.GetClient(cf) - if nsxClient == nil { - return cleanupService, fmt.Errorf("failed to get nsx client") - } - var commonService = common.Service{ NSXClient: nsxClient, NSXConfig: cf, } vpcService, vpcErr := vpc.InitializeVPC(commonService) - vpcService, vpcErr := vpc.InitializeVPC(commonService) - commonctl.ServiceMediator.VPCService = vpcService - // initialize all the CR services // Use Fluent Interface to escape error check hell diff --git a/pkg/clean/clean_dlb.go b/pkg/clean/clean_dlb.go new file mode 100644 index 000000000..a4f1b577c --- /dev/null +++ b/pkg/clean/clean_dlb.go @@ -0,0 +1,94 @@ +package clean + +import ( + "context" + "errors" + "fmt" + neturl "net/url" + "strings" + + "github.com/vmware-tanzu/nsx-operator/pkg/config" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" +) + +type ( + mapInterface = map[string]interface{} +) + +const TagDLB = "DLB" + +func appendIfNotExist(slice []string, s string) []string { + for _, item := range slice { + if item == s { + return slice + } + } + return append(slice, s) +} + +func httpQueryDLBResources(cluster *nsx.Cluster, cf *config.NSXOperatorConfig, resource string) ([]string, error) { + queryParam := "resource_type:" + resource + + "&tags.scope:ncp\\/cluster" + + "&tags.tag:" + cf.Cluster + + "&tags.scope:ncp\\/created_for" + + "&tags.tag:" + TagDLB + + pairs := strings.Split(queryParam, "&") + params := make(map[string]string) + for _, pair := range pairs { + kv := strings.Split(pair, ":") + if len(kv) == 2 { + params[kv[0]] = kv[1] + } + } + var encodedPairs []string + for key, value := range params { + encodedKey := neturl.QueryEscape(key) + encodedValue := neturl.QueryEscape(value) + encodedPairs = append(encodedPairs, fmt.Sprintf("%s:%s", encodedKey, encodedValue)) + } + + encodedQuery := strings.Join(encodedPairs, "%20AND%20") + url := "policy/api/v1/search/query?query=" + encodedQuery + + resp, err := cluster.HttpGet(url) + if err != nil { + return nil, err + } + var resourcePath []string + for _, item := range resp["results"].([]interface{}) { + resourcePath = appendIfNotExist(resourcePath, item.(mapInterface)["path"].(string)) + } + return resourcePath, nil +} + +func CleanDLB(ctx context.Context, cluster *nsx.Cluster, cf *config.NSXOperatorConfig) error { + log.Info("Deleting DLB resources started") + + resources := []string{"Group", "LBVirtualServer", "LBService", "LBPool", "LBCookiePersistenceProfile"} + var allPaths []string + + for _, resource := range resources { + paths, err := httpQueryDLBResources(cluster, cf, resource) + if err != nil { + return err + } + log.Info(resource, "count", len(paths)) + allPaths = append(allPaths, paths...) + } + + log.Info("Deleting DLB resources", "paths", allPaths) + for _, path := range allPaths { + url := "policy/api/v1" + path + select { + case <-ctx.Done(): + return errors.Join(nsxutil.TimeoutFailed, ctx.Err()) + default: + if err := cluster.HttpDelete(url); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/clean/types.go b/pkg/clean/types.go index 48264f855..40c5e43f8 100644 --- a/pkg/clean/types.go +++ b/pkg/clean/types.go @@ -31,29 +31,3 @@ func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService { c.cleans = append(c.cleans, clean) return c } - -type cleanupFunc func() (cleanup, error) - -type CleanupService struct { - cleans []cleanup - err error -} - -func NewCleanupService() *CleanupService { - return &CleanupService{} -} - -func (c *CleanupService) AddCleanupService(f cleanupFunc) *CleanupService { - var clean cleanup - if c.err != nil { - return c - } - - clean, c.err = f() - if c.err != nil { - return c - } - - c.cleans = append(c.cleans, clean) - return c -} diff --git a/pkg/config/config.go b/pkg/config/config.go index 43473841e..2c59b4cea 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -281,12 +281,6 @@ func (vcConfig *VCConfig) validate() error { configLog.Info("validate VcConfig failed VCUser %s VCPassword %s", vcConfig.VCUser, vcConfig.VCPassword) return err } - // VCPassword, VCUser should be both empty or valid - if !((len(vcConfig.VCPassword) > 0) == (len(vcConfig.VCUser) > 0)) { - err := errors.New("invalid field " + "VCUser, VCPassword") - log.Info("validate VcConfig failed", "VCUser", vcConfig.VCUser, "VCPassword", vcConfig.VCPassword) - return err - } return nil } diff --git a/pkg/controllers/subnetset/subnetport_handler.go b/pkg/controllers/subnetset/subnetport_handler.go deleted file mode 100644 index a0d545545..000000000 --- a/pkg/controllers/subnetset/subnetport_handler.go +++ /dev/null @@ -1,88 +0,0 @@ -package subnetset - -import ( - "sigs.k8s.io/controller-runtime/pkg/predicate" - - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/event" -) - -// SubnetPortHandler supports lazy-creation of Subnet, the first Subnet won't -// be created until there is a SubnetPort attached to it. -// - SubnetPort creation: get available Subnet for the SubnetPort, create new -// Subnet if necessary. -// - SubnetPort deletion: if recycling Subnet is required, delete Subnets without -// SubnetPort attached to it. - -type SubnetPortHandler struct { - Reconciler *SubnetSetReconciler -} - -//TODO Remove this handler when confirmed that SubnetPort could get allocated subnet via the interface -// subnetservice.GetAvailableSubnet - -// Create allocates Subnet for SubnetPort from SubnetSet. -func (h *SubnetPortHandler) Create(e event.CreateEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("SubnetPort generic event, do nothing") - //subnetPort := e.Object.(*v1alpha1.SubnetPort) - //if subnetPort.Spec.Subnet != "" { - // // Two possible scenarios: - // // - 1. User uses `.Spec.Subnet` directly instead of `.Spec.SubnetSet`. - // // - 2. Subnet has been allocated and `.Spec.Subnet` is rendered by SubnetPortHandler. - // return - //} - //subnetSet := &v1alpha1.SubnetSet{} - //key := types.NamespacedName{ - // Namespace: subnetPort.GetNamespace(), - // Name: subnetPort.Spec.SubnetSet, - //} - //if err := h.Reconciler.Client.Get(context.Background(), key, subnetSet); err != nil { - // log.Error(err, "failed to get SubnetSet", "ns", key.Namespace, "name", key.Name) - // return - //} - //log.Info("allocating Subnet for SubnetPort") - //vpcList := &v1alpha1.VPCList{} - //if err := h.Reconciler.Client.List(context.Background(), vpcList, client.InNamespace(subnetPort.GetNamespace())); err != nil { - // log.Error(err, fmt.Sprintf("failed to get VPC under namespace: %s.\n", subnetPort.GetNamespace())) - // return - //} - //vpcInfo, err := servicecommon.ParseVPCResourcePath(vpcList.Items[0].Status.NSXResourcePath) - //if err != nil { - // log.Error(err, "failed to resolve VPC info") - // return - //} - //_, err = h.Reconciler.getAvailableSubnet(subnetSet, &vpcInfo) - //if err != nil { - // log.Error(err, "failed to allocate Subnet") - //} - // TODO return subnetport id to caller. -} - -// Delete TODO Implement this method if required to recycle Subnet without SubnetPort attached. -func (h *SubnetPortHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("SubnetPort generic event, do nothing") -} - -func (h *SubnetPortHandler) Generic(_ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("SubnetPort generic event, do nothing") -} - -func (h *SubnetPortHandler) Update(_ event.UpdateEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("SubnetPort update event, do nothing") -} - -var SubnetPortPredicate = predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return false - }, - DeleteFunc: func(e event.DeleteEvent) bool { - // TODO When recycling Subnet is required, return true. - return false - }, - GenericFunc: func(genericEvent event.GenericEvent) bool { - return false - }, -} diff --git a/pkg/controllers/subnetset/vpc_handler.go b/pkg/controllers/subnetset/vpc_handler.go deleted file mode 100644 index 71f054989..000000000 --- a/pkg/controllers/subnetset/vpc_handler.go +++ /dev/null @@ -1,113 +0,0 @@ -package subnetset - -import ( - "context" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - - "sigs.k8s.io/controller-runtime/pkg/predicate" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/util/retry" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" - - "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" -) - -// VPCHandler handles VPC event for SubnetSet: -// - VPC creation: create default SubnetSet for the VPC. -// - VPC deletion: delete all SubnetSets under the VPC. - -var defaultSubnetSets = map[string]string{ - "default-vm-subnetset": common.LabelDefaultVMSubnet, - "default-pod-subnetset": common.LabelDefaultPodSubnetSet, -} - -type VPCHandler struct { - Client client.Client -} - -func (h *VPCHandler) Create(e event.CreateEvent, _ workqueue.RateLimitingInterface) { - ns := e.Object.GetNamespace() - log.Info("creating default Subnetset for VPC", "Namespace", ns, "Name", e.Object.GetName()) - for name, subnetSetType := range defaultSubnetSets { - if err := retry.OnError(retry.DefaultRetry, func(err error) bool { - return err != nil - }, func() error { - list := &v1alpha1.SubnetSetList{} - label := client.MatchingLabels{ - common.LabelDefaultSubnetSet: subnetSetType, - } - nsOption := client.InNamespace(ns) - if err := h.Client.List(context.Background(), list, label, nsOption); err != nil { - return err - } - if len(list.Items) > 0 { - // avoid creating when nsx-operator restarted if Subnetset exists. - log.Info("default subnetset already exists", common.LabelDefaultSubnetSet, subnetSetType) - return nil - } - obj := &v1alpha1.SubnetSet{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ns, - Name: name, - Labels: map[string]string{ - common.LabelDefaultSubnetSet: subnetSetType, - }, - }, - Spec: v1alpha1.SubnetSetSpec{}, - } - if err := h.Client.Create(context.Background(), obj); err != nil { - return err - } - return nil - }); err != nil { - log.Error(err, "failed to create SubnetSet", "Namespace", ns, "Name", name) - } - } -} - -func (h *VPCHandler) Delete(e event.DeleteEvent, _ workqueue.RateLimitingInterface) { - log.Info("cleaning default Subnetset for VPC", "Name", e.Object.GetName()) - for _, subnetSetType := range defaultSubnetSets { - if err := retry.OnError(retry.DefaultRetry, func(err error) bool { - return err != nil - }, func() error { - label := client.MatchingLabels{ - common.LabelDefaultSubnetSet: subnetSetType, - } - nsOption := client.InNamespace(e.Object.GetNamespace()) - obj := &v1alpha1.SubnetSet{} - if err := h.Client.DeleteAllOf(context.Background(), obj, label, nsOption); err != nil { - return client.IgnoreNotFound(err) - } - return nil - }); err != nil { - log.Error(err, "failed to delete SubnetSet", common.LabelDefaultSubnetSet, subnetSetType) - } - } -} - -func (h *VPCHandler) Generic(_ event.GenericEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("VPC generic event, do nothing") -} - -func (h *VPCHandler) Update(_ event.UpdateEvent, _ workqueue.RateLimitingInterface) { - log.V(4).Info("VPC update event, do nothing") -} - -var VPCPredicate = predicate.Funcs{ - CreateFunc: func(e event.CreateEvent) bool { - return true - }, - UpdateFunc: func(e event.UpdateEvent) bool { - return false - }, - DeleteFunc: func(e event.DeleteEvent) bool { - return true - }, - GenericFunc: func(genericEvent event.GenericEvent) bool { - return false - }, -} diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index bba1f19ad..5a5d13284 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -49,6 +49,7 @@ var FeaturesName = [AllFeatures]string{"VPC", "SECURITY_POLICY", "NSX_SERVICE_AC type Client struct { NsxConfig *config.NSXOperatorConfig RestConnector *client.RestConnector + Cluster *Cluster QueryClient search.QueryClient GroupClient domains.GroupsClient @@ -174,14 +175,14 @@ func GetClient(cf *config.NSXOperatorConfig) *Client { } nsxClient := &Client{ - NsxConfig: cf, - RestConnector: restConnector(cluster), - QueryClient: queryClient, - GroupClient: groupClient, - SecurityClient: securityClient, - RuleClient: ruleClient, - InfraClient: infraClient, - + NsxConfig: cf, + RestConnector: restConnector(cluster), + QueryClient: queryClient, + GroupClient: groupClient, + SecurityClient: securityClient, + RuleClient: ruleClient, + InfraClient: infraClient, + Cluster: cluster, ClusterControlPlanesClient: clusterControlPlanesClient, HostTransPortNodesClient: hostTransportNodesClient, RealizedEntitiesClient: realizedEntitiesClient, diff --git a/pkg/nsx/cluster.go b/pkg/nsx/cluster.go index b61f80e18..b4d1375e5 100644 --- a/pkg/nsx/cluster.go +++ b/pkg/nsx/cluster.go @@ -363,19 +363,13 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) { return nil, err } log.V(1).Info("Get url", "url", req.URL) - err = ep.UpdateHttpRequestAuth(req) - if err != nil { - log.Error(err, "http get update auth error") - return nil, err - } - ep.UpdateCAforEnvoy(req) resp, err := ep.client.Do(req) if err != nil { log.Error(err, "failed to do http GET operation") return nil, err } - var respJson map[string]interface{} - err, _ = util.HandleHTTPResponse(resp, respJson, true) + respJson := make(map[string]interface{}) + err, _ = util.HandleHTTPResponse(resp, &respJson, true) return respJson, err } @@ -390,12 +384,6 @@ func (cluster *Cluster) HttpDelete(url string) error { return err } log.V(1).Info("Delete url", "url", req.URL) - err = ep.UpdateHttpRequestAuth(req) - if err != nil { - log.Error(err, "http get update auth error") - return err - } - ep.UpdateCAforEnvoy(req) _, err = ep.client.Do(req) if err != nil { log.Error(err, "failed to do http DELETE operation") diff --git a/pkg/nsx/services/common/compare.go b/pkg/nsx/services/common/compare.go index 4d9166775..f8b259355 100644 --- a/pkg/nsx/services/common/compare.go +++ b/pkg/nsx/services/common/compare.go @@ -41,8 +41,6 @@ func CompareResources(existing []Comparable, expected []Comparable) (changed []C if existed_item, ok := existingMap[key]; ok { if isChanged := CompareResource(existed_item, expected_item); !isChanged { continue - } else { - log.V(1).Info("resource changed", "existing", existed_item, "expected", expected_item) } log.V(1).Info("resource changed", "existing", existed_item, "expected", expected_item) } diff --git a/pkg/nsx/services/mediator/mediator_test.go b/pkg/nsx/services/mediator/mediator_test.go deleted file mode 100644 index 69eae18d2..000000000 --- a/pkg/nsx/services/mediator/mediator_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package mediator - -import ( - "reflect" - "testing" - - "github.com/agiledragon/gomonkey" - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" - - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" -) - -func TestServiceMediator_GetOrgProject(t *testing.T) { - vpcService := &vpc.VPCService{} - vs := &ServiceMediator{ - SecurityPolicyService: nil, - VPCService: vpcService, - } - - patches := gomonkey.ApplyMethod(reflect.TypeOf(vpcService), "GetVPCsByNamespace", func(_ *vpc.VPCService, ns string) []model.Vpc { - return []model.Vpc{{Path: common.String("/orgs/default/projects/project-1/vpcs/vpc-1")}} - }) - defer patches.Reset() - - got := vs.ListVPCInfo("ns")[0] - want := common.VPCResourceInfo{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1", ID: "vpc-1", ParentID: "project-1"} - if !reflect.DeepEqual(got, want) { - t.Errorf("GetOrgProject() = %v, want %v", got, want) - } -}