diff --git a/cmd/main.go b/cmd/main.go index a7bb3f5d5..5ee89dd37 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -19,7 +19,6 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha2" "github.com/vmware-tanzu/nsx-operator/pkg/config" - commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" ippool2 "github.com/vmware-tanzu/nsx-operator/pkg/controllers/ippool" namespacecontroller "github.com/vmware-tanzu/nsx-operator/pkg/controllers/namespace" networkpolicycontroller "github.com/vmware-tanzu/nsx-operator/pkg/controllers/networkpolicy" @@ -41,6 +40,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/nsxserviceaccount" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/staticroute" subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" + subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) @@ -118,10 +118,7 @@ func StartVPCController(mgr ctrl.Manager, vpcService *vpc.VPCService) { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), } - vpcReconciler.Service = vpcService - commonctl.ServiceMediator.VPCService = vpcService - if err := vpcReconciler.Start(mgr); err != nil { log.Error(err, "failed to create vpc controller", "controller", "VPC") os.Exit(1) @@ -188,9 +185,12 @@ func main() { ipPoolService, err := ippool.InitializeIPPool(commonService, vpcService) if err != nil { log.Error(err, "failed to initialize ippool commonService", "controller", "IPPool") + } + subnetPortService, err := subnetportservice.InitializeSubnetPort(commonService) + if err != nil { + log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort") os.Exit(1) } - commonctl.ServiceMediator.VPCService = vpcService nodeService, err := nodeservice.InitializeNode(commonService) if err != nil { log.Error(err, "failed to initialize node commonService", "controller", "Node") @@ -205,17 +205,17 @@ func main() { StartVPCController(mgr, vpcService) StartNamespaceController(mgr, cf, vpcService) // Start subnet/subnetset controller. - if err := subnet.StartSubnetController(mgr, subnetService); err != nil { + if err := subnet.StartSubnetController(mgr, subnetService, subnetPortService, vpcService); err != nil { os.Exit(1) } - if err := subnetset.StartSubnetSetController(mgr, subnetService); err != nil { + if err := subnetset.StartSubnetSetController(mgr, subnetService, subnetPortService, vpcService); err != nil { os.Exit(1) } node.StartNodeController(mgr, nodeService) staticroutecontroller.StartStaticRouteController(mgr, staticRouteService) - subnetport.StartSubnetPortController(mgr, commonService) - pod.StartPodController(mgr, commonService, nodeService) + subnetport.StartSubnetPortController(mgr, subnetPortService, subnetService, vpcService) + pod.StartPodController(mgr, subnetPortService, subnetService, vpcService, nodeService) StartIPPoolController(mgr, ipPoolService, vpcService) networkpolicycontroller.StartNetworkPolicyController(mgr, commonService, vpcService) } diff --git a/pkg/clean/clean.go b/pkg/clean/clean.go index 2fabb58ec..5941ac4d2 100644 --- a/pkg/clean/clean.go +++ b/pkg/clean/clean.go @@ -10,7 +10,6 @@ import ( "k8s.io/client-go/util/retry" "github.com/vmware-tanzu/nsx-operator/pkg/config" - commonctl "github.com/vmware-tanzu/nsx-operator/pkg/controllers/common" "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" @@ -82,7 +81,6 @@ func InitializeCleanupService(cf *config.NSXOperatorConfig) (*CleanupService, er NSXConfig: cf, } vpcService, vpcErr := vpc.InitializeVPC(commonService) - commonctl.ServiceMediator.VPCService = vpcService // initialize all the CR services // Use Fluent Interface to escape error check hell diff --git a/pkg/controllers/common/types.go b/pkg/controllers/common/types.go index 64b57c6ed..6045281e8 100644 --- a/pkg/controllers/common/types.go +++ b/pkg/controllers/common/types.go @@ -4,8 +4,6 @@ import ( "time" ctrl "sigs.k8s.io/controller-runtime" - - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/mediator" ) const ( @@ -33,6 +31,4 @@ var ( ResultRequeueAfter10sec = ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second} // for unstable event, eg: failed to k8s resources when reconciling, may due to k8s unstable ResultRequeueAfter5mins = ctrl.Result{Requeue: true, RequeueAfter: 5 * time.Minute} - - ServiceMediator = mediator.ServiceMediator{} ) diff --git a/pkg/controllers/common/utils.go b/pkg/controllers/common/utils.go index e9ffe1453..46ab8cc69 100644 --- a/pkg/controllers/common/utils.go +++ b/pkg/controllers/common/utils.go @@ -15,6 +15,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" "github.com/vmware-tanzu/nsx-operator/pkg/logger" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/util" ) var ( @@ -22,16 +23,34 @@ var ( lock = &sync.Mutex{} ) -func AllocateSubnetFromSubnetSet(subnetSet *v1alpha1.SubnetSet) (string, error) { +func AllocateSubnetFromSubnetSet(subnetSet *v1alpha1.SubnetSet, vpcService servicecommon.VPCServiceProvider, subnetService servicecommon.SubnetServiceProvider, subnetPortService servicecommon.SubnetPortServiceProvider) (string, error) { // TODO: For now, this is a global lock. In the future, we need to narrow its scope down to improve the performance. lock.Lock() defer lock.Unlock() - subnetPath, err := ServiceMediator.GetAvailableSubnet(subnetSet) - if err != nil { + subnetList := subnetService.GetSubnetsByIndex(servicecommon.TagScopeSubnetSetCRUID, string(subnetSet.GetUID())) + for _, nsxSubnet := range subnetList { + portNums := len(subnetPortService.GetPortsOfSubnet(*nsxSubnet.Id)) + totalIP := int(*nsxSubnet.Ipv4SubnetSize) + if len(nsxSubnet.IpAddresses) > 0 { + // totalIP will be overrided if IpAddresses are specified. + totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses) + } + if portNums < totalIP-3 { + return *nsxSubnet.Path, nil + } + } + tags := subnetService.GenerateSubnetNSTags(subnetSet, subnetSet.Namespace) + if tags == nil { + return "", errors.New("failed to generate subnet tags") + } + log.Info("the existing subnets are not available, creating new subnet", "subnetList", subnetList, "subnetSet.Name", subnetSet.Name, "subnetSet.Namespace", subnetSet.Namespace) + vpcInfoList := vpcService.ListVPCInfo(subnetSet.Namespace) + if len(vpcInfoList) == 0 { + err := errors.New("no VPC found") log.Error(err, "failed to allocate Subnet") return "", err } - return subnetPath, nil + return subnetService.CreateOrUpdateSubnet(subnetSet, vpcInfoList[0], tags) } func getSharedNamespaceAndVpcForNamespace(client k8sclient.Client, ctx context.Context, namespaceName string) (string, string, error) { @@ -64,14 +83,14 @@ func GetDefaultSubnetSet(client k8sclient.Client, ctx context.Context, namespace log.Info("namespace doesn't have shared VPC, searching the default subnetset in the current namespace", "namespace", namespace) targetNamespace = namespace } - subnetSet, err := getDefaultSubnetSetByNamespace(client, ctx, targetNamespace, resourceType) + subnetSet, err := getDefaultSubnetSetByNamespace(client, targetNamespace, resourceType) if err != nil { return nil, err } return subnetSet, err } -func getDefaultSubnetSetByNamespace(client k8sclient.Client, ctx context.Context, namespace string, resourceType string) (*v1alpha1.SubnetSet, error) { +func getDefaultSubnetSetByNamespace(client k8sclient.Client, namespace string, resourceType string) (*v1alpha1.SubnetSet, error) { subnetSetList := &v1alpha1.SubnetSetList{} subnetSetSelector := &metav1.LabelSelector{ MatchLabels: map[string]string{ diff --git a/pkg/controllers/pod/pod_controller.go b/pkg/controllers/pod/pod_controller.go index ab0ad0122..52632f5c1 100644 --- a/pkg/controllers/pod/pod_controller.go +++ b/pkg/controllers/pod/pod_controller.go @@ -39,8 +39,11 @@ var ( // PodReconciler reconciles a Pod object type PodReconciler struct { client.Client - Scheme *apimachineryruntime.Scheme - Service *subnetport.SubnetPortService + Scheme *apimachineryruntime.Scheme + + SubnetPortService *subnetport.SubnetPortService + SubnetService servicecommon.SubnetServiceProvider + VPCService servicecommon.VPCServiceProvider NodeServiceReader servicecommon.NodeServiceReader } @@ -48,7 +51,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R pod := &v1.Pod{} log.Info("reconciling pod", "pod", req.NamespacedName) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypePod) if err := r.Client.Get(ctx, req.NamespacedName, pod); err != nil { log.Error(err, "unable to fetch pod", "req", req.NamespacedName) @@ -64,7 +67,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R } if pod.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypePod) if !controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { controllerutil.AddFinalizer(pod, servicecommon.PodFinalizerName) if err := r.Client.Update(ctx, pod); err != nil { @@ -88,7 +91,7 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R return common.ResultRequeue, err } contextID := *node.Id - nsxSubnetPortState, err := r.Service.CreateOrUpdateSubnetPort(pod, nsxSubnetPath, contextID, &pod.ObjectMeta.Labels) + nsxSubnetPortState, err := r.SubnetPortService.CreateOrUpdateSubnetPort(pod, nsxSubnetPath, contextID, &pod.ObjectMeta.Labels) if err != nil { log.Error(err, "failed to create or update NSX subnet port, would retry exponentially", "pod.Name", req.NamespacedName, "pod.UID", pod.UID) updateFail(r, &ctx, pod, &err) @@ -106,8 +109,8 @@ func (r *PodReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R updateSuccess(r, &ctx, pod) } else { if controllerutil.ContainsFinalizer(pod, servicecommon.PodFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - if err := r.Service.DeleteSubnetPort(pod.UID); err != nil { + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) + if err := r.SubnetPortService.DeleteSubnetPort(pod.UID); err != nil { log.Error(err, "deletion failed, would retry exponentially", "pod", req.NamespacedName) deleteFail(r, &ctx, pod, &err) return common.ResultRequeue, err @@ -161,20 +164,15 @@ func (r *PodReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func StartPodController(mgr ctrl.Manager, commonService servicecommon.Service, nodeServiceReader servicecommon.NodeServiceReader) { +func StartPodController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService servicecommon.SubnetServiceProvider, vpcService servicecommon.VPCServiceProvider, nodeService servicecommon.NodeServiceReader) { podPortReconciler := PodReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), - NodeServiceReader: nodeServiceReader, + SubnetService: subnetService, + SubnetPortService: subnetPortService, + VPCService: vpcService, + NodeServiceReader: nodeService, } - subnetPortService, err := subnetport.InitializeSubnetPort(commonService) - if err != nil { - log.Error(err, "failed to initialize subnetport commonService", "controller", "Pod") - os.Exit(1) - } - podPortReconciler.Service = subnetPortService - common.ServiceMediator.SubnetPortService = podPortReconciler.Service - if err := podPortReconciler.Start(mgr); err != nil { log.Error(err, "failed to create controller", "controller", "Pod") os.Exit(1) @@ -202,7 +200,7 @@ func (r *PodReconciler) GarbageCollector(cancel chan bool, timeout time.Duration return case <-time.After(timeout): } - nsxSubnetPortSet := r.Service.ListNSXSubnetPortIDForPod() + nsxSubnetPortSet := r.SubnetPortService.ListNSXSubnetPortIDForPod() if len(nsxSubnetPortSet) == 0 { continue } @@ -223,45 +221,45 @@ func (r *PodReconciler) GarbageCollector(cancel chan bool, timeout time.Duration continue } log.V(1).Info("GC collected Pod", "UID", elem) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) - err = r.Service.DeleteSubnetPort(types.UID(elem)) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypePod) + err = r.SubnetPortService.DeleteSubnetPort(types.UID(elem)) if err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypePod) } } } } func updateFail(r *PodReconciler, c *context.Context, o *v1.Pod, e *error) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypePod) } func deleteFail(r *PodReconciler, c *context.Context, o *v1.Pod, e *error) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypePod) } func updateSuccess(r *PodReconciler, c *context.Context, o *v1.Pod) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypePod) } func deleteSuccess(r *PodReconciler, _ *context.Context, _ *v1.Pod) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypePod) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypePod) } func (r *PodReconciler) GetSubnetPathForPod(ctx context.Context, pod *v1.Pod) (string, error) { - subnetPath := r.Service.GetSubnetPathForSubnetPortFromStore(string(pod.UID)) + subnetPath := r.SubnetPortService.GetSubnetPathForSubnetPortFromStore(string(pod.UID)) if len(subnetPath) > 0 { log.V(1).Info("NSX subnet port had been created, returning the existing NSX subnet path", "pod.UID", pod.UID, "subnetPath", subnetPath) return subnetPath, nil } - subnetSet, err := common.GetDefaultSubnetSet(r.Service.Client, ctx, pod.Namespace, servicecommon.LabelDefaultPodSubnetSet) + subnetSet, err := common.GetDefaultSubnetSet(r.SubnetPortService.Client, ctx, pod.Namespace, servicecommon.LabelDefaultPodSubnetSet) if err != nil { return "", err } log.Info("got default subnetset for pod, allocating the NSX subnet", "subnetSet.Name", subnetSet.Name, "subnetSet.UID", subnetSet.UID, "pod.Name", pod.Name, "pod.UID", pod.UID) - subnetPath, err = common.AllocateSubnetFromSubnetSet(subnetSet) + subnetPath, err = common.AllocateSubnetFromSubnetSet(subnetSet, r.VPCService, r.SubnetService, r.SubnetPortService) if err != nil { return subnetPath, err } diff --git a/pkg/controllers/subnet/subnet_controller.go b/pkg/controllers/subnet/subnet_controller.go index 6e0a644d2..4a63cf7a0 100644 --- a/pkg/controllers/subnet/subnet_controller.go +++ b/pkg/controllers/subnet/subnet_controller.go @@ -12,7 +12,6 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apimachineryruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -42,29 +41,25 @@ var ( // SubnetReconciler reconciles a SubnetSet object type SubnetReconciler struct { - Client client.Client - Scheme *apimachineryruntime.Scheme - Service *subnet.SubnetService + Client client.Client + Scheme *apimachineryruntime.Scheme + SubnetService *subnet.SubnetService + SubnetPortService servicecommon.SubnetPortServiceProvider + VPCService servicecommon.VPCServiceProvider } func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { obj := &v1alpha1.Subnet{} - nsObj := &v1.Namespace{} log.Info("reconciling subnet CR", "subnet", req.NamespacedName) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnet) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { log.Error(err, "unable to fetch Subnet CR", "req", req.NamespacedName) return ResultNormal, client.IgnoreNotFound(err) } - if err := r.Client.Get(ctx, client.ObjectKey{Name: obj.Namespace}, nsObj); err != nil { - err = fmt.Errorf("unable to fetch namespace %s", obj.Namespace) - log.Error(err, "") - return ResultRequeue, err - } if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnet) if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { controllerutil.AddFinalizer(obj, servicecommon.SubnetFinalizerName) if err := r.Client.Update(ctx, obj); err != nil { @@ -75,7 +70,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr log.V(1).Info("added finalizer on subnet CR", "subnet", req.NamespacedName) } if obj.Spec.AccessMode == "" || obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := common.ServiceMediator.GetVPCNetworkConfigByNamespace(obj.Namespace) + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) if vpcNetworkConfig == nil { err := fmt.Errorf("operate failed: cannot get configuration for Subnet CR") log.Error(nil, "failed to find VPCNetworkConfig for Subnet CR", "subnet", req.NamespacedName, "namespace %s", obj.Namespace) @@ -89,25 +84,15 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr obj.Spec.IPv4SubnetSize = vpcNetworkConfig.DefaultIPv4SubnetSize } } - - namespace := &v1.Namespace{} - namespacedName := types.NamespacedName{ - Name: req.Namespace, + tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) + if tags == nil { + return ResultRequeue, errors.New("failed to generate subnet tags") } - if err := r.Client.Get(context.Background(), namespacedName, namespace); err != nil { - log.Error(err, "unable to fetch namespace of Subnet CR", "req", req.NamespacedName) - updateFail(r, &ctx, obj, "") - return ResultRequeue, err - } - tags := r.Service.GenerateSubnetNSTags(obj, string(nsObj.UID)) - for k, v := range nsObj.Labels { - tags = append(tags, model.Tag{Scope: servicecommon.String(k), Tag: servicecommon.String(v)}) - } - vpcInfo, err := common.ServiceMediator.GetNamespaceVPCInfo(req.Namespace) - if err != nil { + vpcInfoList := r.VPCService.ListVPCInfo(req.Namespace) + if len(vpcInfoList) == 0 { return ResultRequeueAfter10sec, nil } - if _, err := r.Service.CreateOrUpdateSubnet(obj, *vpcInfo, tags); err != nil { + if _, err := r.SubnetService.CreateOrUpdateSubnet(obj, vpcInfoList[0], tags); err != nil { if errors.As(err, &util.ExceedTagsError{}) { log.Error(err, "exceed tags limit, would not retry", "subnet", req.NamespacedName) updateFail(r, &ctx, obj, err.Error()) @@ -125,7 +110,7 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr updateSuccess(r, &ctx, obj) } else { if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnet) if err := r.DeleteSubnet(*obj); err != nil { log.Error(err, "deletion failed, would retry exponentially", "subnet", req.NamespacedName) deleteFail(r, &ctx, obj, "") @@ -147,27 +132,27 @@ func (r *SubnetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *SubnetReconciler) DeleteSubnet(obj v1alpha1.Subnet) error { - nsxSubnets := r.Service.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, string(obj.GetUID())) + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetCRUID, string(obj.GetUID())) if len(nsxSubnets) == 0 { log.Info("no subnet found for subnet CR", "uid", string(obj.GetUID())) return nil } - portNums := len(common.ServiceMediator.GetPortsOfSubnet(*nsxSubnets[0].Id)) + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*nsxSubnets[0].Id)) if portNums > 0 { err := errors.New("subnet still attached by port") log.Error(err, "", "ID", *nsxSubnets[0].Id) return err } - return r.Service.DeleteSubnet(*nsxSubnets[0]) + return r.SubnetService.DeleteSubnet(*nsxSubnets[0]) } func (r *SubnetReconciler) updateSubnetStatus(obj *v1alpha1.Subnet) error { - nsxSubnet := r.Service.SubnetStore.GetByKey(r.Service.BuildSubnetID(obj)) + nsxSubnet := r.SubnetService.SubnetStore.GetByKey(r.SubnetService.BuildSubnetID(obj)) if nsxSubnet == nil { return errors.New("failed to get NSX Subnet from store") } obj.Status.IPAddresses = obj.Status.IPAddresses[:0] - statusList, err := r.Service.GetSubnetStatus(nsxSubnet) + statusList, err := r.SubnetService.GetSubnetStatus(nsxSubnet) if err != nil { return err } @@ -252,21 +237,21 @@ func getExistingConditionOfType(conditionType v1alpha1.ConditionType, existingCo func updateFail(r *SubnetReconciler, c *context.Context, o *v1alpha1.Subnet, m string) { r.setSubnetReadyStatusFalse(c, o, metav1.Now(), m) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnet) } func deleteFail(r *SubnetReconciler, c *context.Context, o *v1alpha1.Subnet, m string) { r.setSubnetReadyStatusFalse(c, o, metav1.Now(), m) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnet) } func updateSuccess(r *SubnetReconciler, c *context.Context, o *v1alpha1.Subnet) { r.setSubnetReadyStatusTrue(c, o, metav1.Now()) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnet) } func deleteSuccess(r *SubnetReconciler, _ *context.Context, _ *v1alpha1.Subnet) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnet) } func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { @@ -290,13 +275,14 @@ func (r *SubnetReconciler) setupWithManager(mgr ctrl.Manager) error { Complete(r) } -func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService) error { +func StartSubnetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider) error { subnetReconciler := &SubnetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SubnetService: subnetService, + SubnetPortService: subnetPortService, + VPCService: vpcService, } - subnetReconciler.Service = subnetService - common.ServiceMediator.SubnetService = subnetReconciler.Service if err := subnetReconciler.Start(mgr); err != nil { log.Error(err, "failed to create controller", "controller", "Subnet") return err @@ -331,7 +317,7 @@ func (r *SubnetReconciler) GarbageCollector(cancel chan bool, timeout time.Durat } var nsxSubnetList []*model.VpcSubnet for _, subnet := range crdSubnetList.Items { - nsxSubnetList = append(nsxSubnetList, r.Service.ListSubnetCreatedBySubnet(string(subnet.UID))...) + nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnet(string(subnet.UID))...) } if len(nsxSubnetList) == 0 { continue @@ -349,12 +335,12 @@ func (r *SubnetReconciler) GarbageCollector(cancel chan bool, timeout time.Durat } log.Info("GC collected Subnet CR", "UID", elem) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeSubnet) - err = r.Service.DeleteSubnet(*elem) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, common.MetricResTypeSubnet) + err = r.SubnetService.DeleteSubnet(*elem) if err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, common.MetricResTypeSubnet) } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeSubnet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, common.MetricResTypeSubnet) } } } diff --git a/pkg/controllers/subnet/subnet_controller_test.go b/pkg/controllers/subnet/subnet_controller_test.go index e19b2acb3..28c9d6062 100644 --- a/pkg/controllers/subnet/subnet_controller_test.go +++ b/pkg/controllers/subnet/subnet_controller_test.go @@ -50,9 +50,9 @@ func TestSubnetReconciler_GarbageCollector(t *testing.T) { k8sClient := mock_client.NewMockClient(mockCtl) r := &SubnetReconciler{ - Client: k8sClient, - Scheme: nil, - Service: service, + Client: k8sClient, + Scheme: nil, + SubnetService: service, } ctx := context.Background() srList := &v1alpha1.SubnetList{} diff --git a/pkg/controllers/subnetport/subnetport_controller.go b/pkg/controllers/subnetport/subnetport_controller.go index 03b1da994..fe433cf95 100644 --- a/pkg/controllers/subnetport/subnetport_controller.go +++ b/pkg/controllers/subnetport/subnetport_controller.go @@ -35,7 +35,9 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/logger" "github.com/vmware-tanzu/nsx-operator/pkg/metrics" servicecommon "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" ) var ( @@ -46,8 +48,10 @@ var ( // SubnetPortReconciler reconciles a SubnetPort object type SubnetPortReconciler struct { client.Client - Scheme *apimachineryruntime.Scheme - Service *subnetport.SubnetPortService + Scheme *apimachineryruntime.Scheme + SubnetPortService *subnetport.SubnetPortService + SubnetService servicecommon.SubnetServiceProvider + VPCService servicecommon.VPCServiceProvider } // +kubebuilder:rbac:groups=nsx.vmware.com,resources=subnetports,verbs=get;list;watch;create;update;patch;delete @@ -57,7 +61,7 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) subnetPort := &v1alpha1.SubnetPort{} log.Info("reconciling subnetport CR", "subnetport", req.NamespacedName) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetPort) if err := r.Client.Get(ctx, req.NamespacedName, subnetPort); err != nil { log.Error(err, "unable to fetch subnetport CR", "req", req.NamespacedName) @@ -71,7 +75,7 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if subnetPort.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetPort) if !controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { controllerutil.AddFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) if err := r.Client.Update(ctx, subnetPort); err != nil { @@ -94,7 +98,7 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) return common.ResultRequeue, err } log.Info("got labels from virtualmachine for subnetport", "subnetPort.UID", subnetPort.UID, "virtualmachine name", subnetPort.Spec.AttachmentRef.Name, "labels", labels) - nsxSubnetPortState, err := r.Service.CreateOrUpdateSubnetPort(subnetPort, nsxSubnetPath, "", labels) + nsxSubnetPortState, err := r.SubnetPortService.CreateOrUpdateSubnetPort(subnetPort, nsxSubnetPath, "", labels) if err != nil { log.Error(err, "failed to create or update NSX subnet port, would retry exponentially", "subnetport", req.NamespacedName) updateFail(r, &ctx, subnetPort, &err) @@ -119,8 +123,8 @@ func (r *SubnetPortReconciler) Reconcile(ctx context.Context, req ctrl.Request) updateSuccess(r, &ctx, subnetPort) } else { if controllerutil.ContainsFinalizer(subnetPort, servicecommon.SubnetPortFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - if err := r.Service.DeleteSubnetPort(subnetPort.UID); err != nil { + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) + if err := r.SubnetPortService.DeleteSubnetPort(subnetPort.UID); err != nil { log.Error(err, "deletion failed, would retry exponentially", "subnetport", req.NamespacedName) deleteFail(r, &ctx, subnetPort, &err) return common.ResultRequeue, err @@ -189,18 +193,14 @@ func (r *SubnetPortReconciler) vmMapFunc(_ context.Context, vm client.Object) [] return requests } -func StartSubnetPortController(mgr ctrl.Manager, commonService servicecommon.Service) { +func StartSubnetPortController(mgr ctrl.Manager, subnetPortService *subnetport.SubnetPortService, subnetService *subnet.SubnetService, vpcService *vpc.VPCService) { subnetPortReconciler := SubnetPortReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SubnetService: subnetService, + SubnetPortService: subnetPortService, + VPCService: vpcService, } - subnetPortService, err := subnetport.InitializeSubnetPort(commonService) - if err != nil { - log.Error(err, "failed to initialize subnetport commonService", "controller", "SubnetPort") - os.Exit(1) - } - subnetPortReconciler.Service = subnetPortService - common.ServiceMediator.SubnetPortService = subnetPortReconciler.Service if err := subnetPortReconciler.Start(mgr); err != nil { log.Error(err, "failed to create controller", "controller", "SubnetPort") os.Exit(1) @@ -228,7 +228,7 @@ func (r *SubnetPortReconciler) GarbageCollector(cancel chan bool, timeout time.D return case <-time.After(timeout): } - nsxSubnetPortSet := r.Service.ListNSXSubnetPortIDForCR() + nsxSubnetPortSet := r.SubnetPortService.ListNSXSubnetPortIDForCR() if len(nsxSubnetPortSet) == 0 { continue } @@ -249,12 +249,12 @@ func (r *SubnetPortReconciler) GarbageCollector(cancel chan bool, timeout time.D continue } log.V(1).Info("GC collected SubnetPort CR", "UID", elem) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) - err = r.Service.DeleteSubnetPort(types.UID(elem)) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetPort) + err = r.SubnetPortService.DeleteSubnetPort(types.UID(elem)) if err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort) } } } @@ -332,25 +332,25 @@ func getExistingConditionOfType(conditionType v1alpha1.ConditionType, existingCo func updateFail(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.SubnetPort, e *error) { r.setSubnetPortReadyStatusFalse(c, o, metav1.Now(), e) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnetPort) } func deleteFail(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.SubnetPort, e *error) { r.setSubnetPortReadyStatusFalse(c, o, metav1.Now(), e) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetPort) } func updateSuccess(r *SubnetPortReconciler, c *context.Context, o *v1alpha1.SubnetPort) { r.setSubnetPortReadyStatusTrue(c, o, metav1.Now()) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnetPort) } func deleteSuccess(r *SubnetPortReconciler, _ *context.Context, _ *v1alpha1.SubnetPort) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort) + metrics.CounterInc(r.SubnetPortService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetPort) } func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(ctx context.Context, subnetPort *v1alpha1.SubnetPort) (string, error) { - subnetPath := r.Service.GetSubnetPathForSubnetPortFromStore(string(subnetPort.UID)) + subnetPath := r.SubnetPortService.GetSubnetPathForSubnetPortFromStore(string(subnetPort.UID)) if len(subnetPath) > 0 { log.V(1).Info("NSX subnet port had been created, returning the existing NSX subnet path", "subnetPort.UID", subnetPort.UID, "subnetPath", subnetPath) return subnetPath, nil @@ -381,7 +381,7 @@ func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(ctx context.Context, s return subnetPath, err } log.Info("got subnetset for subnetport CR, allocating the NSX subnet", "subnetSet.Name", subnetSet.Name, "subnetSet.UID", subnetSet.UID, "subnetPort.Name", subnetPort.Name, "subnetPort.UID", subnetPort.UID) - subnetPath, err := common.AllocateSubnetFromSubnetSet(subnetSet) + subnetPath, err := common.AllocateSubnetFromSubnetSet(subnetSet, r.VPCService, r.SubnetService, r.SubnetPortService) log.Info("allocated Subnet for SubnetPort", "subnetPath", subnetPath, "subnetPort.Name", subnetPort.Name, "subnetPort.UID", subnetPort.UID) if err != nil { return subnetPath, err @@ -393,7 +393,7 @@ func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(ctx context.Context, s return "", err } log.Info("got default subnetset for subnetport CR, allocating the NSX subnet", "subnetSet.Name", subnetSet.Name, "subnetSet.UID", subnetSet.UID, "subnetPort.Name", subnetPort.Name, "subnetPort.UID", subnetPort.UID) - subnetPath, err := common.AllocateSubnetFromSubnetSet(subnetSet) + subnetPath, err := common.AllocateSubnetFromSubnetSet(subnetSet, r.VPCService, r.SubnetService, r.SubnetPortService) log.Info("allocated Subnet for SubnetPort", "subnetPath", subnetPath, "subnetPort.Name", subnetPort.Name, "subnetPort.UID", subnetPort.UID) if err != nil { return subnetPath, err @@ -404,7 +404,7 @@ func (r *SubnetPortReconciler) GetSubnetPathForSubnetPort(ctx context.Context, s } func (r *SubnetPortReconciler) updateSubnetStatusOnSubnetPort(subnetPort *v1alpha1.SubnetPort, nsxSubnetPath string) error { - gateway, netmask, err := r.Service.GetGatewayNetmaskForSubnetPort(subnetPort, nsxSubnetPath) + gateway, netmask, err := r.SubnetPortService.GetGatewayNetmaskForSubnetPort(subnetPort, nsxSubnetPath) if err != nil { return err } @@ -415,7 +415,7 @@ func (r *SubnetPortReconciler) updateSubnetStatusOnSubnetPort(subnetPort *v1alph // For now, we have an asumption that one subnetport only have one IP address subnetPort.Status.IPAddresses[0].Gateway = gateway subnetPort.Status.IPAddresses[0].Netmask = netmask - nsxSubnet := common.ServiceMediator.SubnetStore.GetByKey(subnetInfo.ID) + nsxSubnet := r.SubnetService.GetSubnetByKey(subnetInfo.ID) if nsxSubnet == nil { return errors.New("NSX subnet not found in store") } diff --git a/pkg/controllers/subnetport/subnetport_controller_test.go b/pkg/controllers/subnetport/subnetport_controller_test.go index 431710d85..3bd57374d 100644 --- a/pkg/controllers/subnetport/subnetport_controller_test.go +++ b/pkg/controllers/subnetport/subnetport_controller_test.go @@ -29,9 +29,9 @@ import ( func NewFakeSubnetPortReconciler() *SubnetPortReconciler { return &SubnetPortReconciler{ - Client: fake.NewClientBuilder().Build(), - Scheme: fake.NewClientBuilder().Build().Scheme(), - Service: nil, + Client: fake.NewClientBuilder().Build(), + Scheme: fake.NewClientBuilder().Build().Scheme(), + SubnetPortService: nil, } } @@ -49,9 +49,9 @@ func TestSubnetPortReconciler_Reconcile(t *testing.T) { }, } r := &SubnetPortReconciler{ - Client: k8sClient, - Scheme: nil, - Service: service, + Client: k8sClient, + Scheme: nil, + SubnetPortService: service, } ctx := context.Background() req := controllerruntime.Request{NamespacedName: types.NamespacedName{Namespace: "dummy", Name: "dummy"}} @@ -275,9 +275,9 @@ func TestSubnetPortReconciler_GarbageCollector(t *testing.T) { mockCtl := gomock.NewController(t) k8sClient := mock_client.NewMockClient(mockCtl) r := &SubnetPortReconciler{ - Client: k8sClient, - Scheme: nil, - Service: service, + Client: k8sClient, + Scheme: nil, + SubnetPortService: service, } subnetPortList := &v1alpha1.SubnetPortList{} k8sClient.EXPECT().List(gomock.Any(), subnetPortList).Return(nil).Do(func(_ context.Context, list client.ObjectList, _ ...client.ListOption) error { diff --git a/pkg/controllers/subnetset/subnetset_controller.go b/pkg/controllers/subnetset/subnetset_controller.go index 4d60ba33e..ac85cfcb8 100644 --- a/pkg/controllers/subnetset/subnetset_controller.go +++ b/pkg/controllers/subnetset/subnetset_controller.go @@ -37,15 +37,17 @@ var ( // SubnetSetReconciler reconciles a SubnetSet object type SubnetSetReconciler struct { - Client client.Client - Scheme *apimachineryruntime.Scheme - Service *subnet.SubnetService + Client client.Client + Scheme *apimachineryruntime.Scheme + SubnetService *subnet.SubnetService + SubnetPortService servicecommon.SubnetPortServiceProvider + VPCService servicecommon.VPCServiceProvider } func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { obj := &v1alpha1.SubnetSet{} log.Info("reconciling subnetset CR", "subnetset", req.NamespacedName) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerSyncTotal, MetricResTypeSubnetSet) if err := r.Client.Get(ctx, req.NamespacedName, obj); err != nil { log.Error(err, "unable to fetch subnetset CR", "req", req.NamespacedName) @@ -53,11 +55,11 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } if obj.ObjectMeta.DeletionTimestamp.IsZero() { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateTotal, MetricResTypeSubnetSet) if !controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { controllerutil.AddFinalizer(obj, servicecommon.SubnetSetFinalizerName) if obj.Spec.AccessMode == "" || obj.Spec.IPv4SubnetSize == 0 { - vpcNetworkConfig := common.ServiceMediator.GetVPCNetworkConfigByNamespace(obj.Namespace) + vpcNetworkConfig := r.VPCService.GetVPCNetworkConfigByNamespace(obj.Namespace) if vpcNetworkConfig == nil { err := fmt.Errorf("failed to find VPCNetworkConfig for namespace %s", obj.Namespace) log.Error(err, "operate failed, would retry exponentially", "subnet", req.NamespacedName) @@ -80,17 +82,11 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( } // update subnetset tags if labels of namespace changed - nsxSubnets := r.Service.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.UID)) + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.UID)) if len(nsxSubnets) > 0 { - nsObj := &v1.Namespace{} - if err := r.Client.Get(ctx, client.ObjectKey{Name: obj.Namespace}, nsObj); err != nil { - err = fmt.Errorf("unable to fetch namespace %s", obj.Namespace) - log.Error(err, "") - return ResultRequeue, err - } - tags := r.Service.GenerateSubnetNSTags(obj, string(nsObj.UID)) - for k, v := range nsObj.Labels { - tags = append(tags, model.Tag{Scope: servicecommon.String(k), Tag: servicecommon.String(v)}) + tags := r.SubnetService.GenerateSubnetNSTags(obj, obj.Namespace) + if tags == nil { + return ResultRequeue, errors.New("failed to generate subnet tags") } // tags cannot exceed maximum size 26 if len(tags) > servicecommon.TagsCountMax { @@ -99,14 +95,14 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( updateFail(r, &ctx, obj, errorMsg) return ResultNormal, nil } - if err := r.Service.UpdateSubnetSetTags(obj.Namespace, nsxSubnets, tags); err != nil { + if err := r.SubnetService.UpdateSubnetSetTags(obj.Namespace, nsxSubnets, tags); err != nil { log.Error(err, "failed to update subnetset tags") } } updateSuccess(r, &ctx, obj) } else { if controllerutil.ContainsFinalizer(obj, servicecommon.SubnetSetFinalizerName) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteTotal, MetricResTypeSubnetSet) if err := r.DeleteSubnetForSubnetSet(*obj, false); err != nil { log.Error(err, "deletion failed, would retry exponentially", "subnetset", req.NamespacedName) deleteFail(r, &ctx, obj, "") @@ -129,21 +125,21 @@ func (r *SubnetSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( func updateFail(r *SubnetSetReconciler, c *context.Context, o *v1alpha1.SubnetSet, m string) { r.setSubnetSetReadyStatusFalse(c, o, metav1.Now(), m) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateFailTotal, MetricResTypeSubnetSet) } func deleteFail(r *SubnetSetReconciler, c *context.Context, o *v1alpha1.SubnetSet, m string) { r.setSubnetSetReadyStatusFalse(c, o, metav1.Now(), m) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } func updateSuccess(r *SubnetSetReconciler, c *context.Context, o *v1alpha1.SubnetSet) { r.setSubnetSetReadyStatusTrue(c, o, metav1.Now()) - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerUpdateSuccessTotal, MetricResTypeSubnetSet) } func deleteSuccess(r *SubnetSetReconciler, _ *context.Context, _ *v1alpha1.SubnetSet) { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } func (r *SubnetSetReconciler) setSubnetSetReadyStatusTrue(ctx *context.Context, subnetset *v1alpha1.SubnetSet, transitionTime metav1.Time) { @@ -252,7 +248,7 @@ func (r *SubnetSetReconciler) GarbageCollector(cancel chan bool, timeout time.Du } var nsxSubnetList []*model.VpcSubnet for _, subnetSet := range subnetSetList.Items { - nsxSubnetList = append(nsxSubnetList, r.Service.ListSubnetCreatedBySubnetSet(string(subnetSet.UID))...) + nsxSubnetList = append(nsxSubnetList, r.SubnetService.ListSubnetCreatedBySubnetSet(string(subnetSet.UID))...) } if len(nsxSubnetList) == 0 { continue @@ -261,41 +257,41 @@ func (r *SubnetSetReconciler) GarbageCollector(cancel chan bool, timeout time.Du subnetSetIDs := sets.New[string]() for _, subnetSet := range subnetSetList.Items { if err := r.DeleteSubnetForSubnetSet(subnetSet, true); err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } subnetSetIDs.Insert(string(subnetSet.UID)) } for _, subnet := range nsxSubnetList { - if !r.Service.IsOrphanSubnet(*subnet, subnetSetIDs) { + if !r.SubnetService.IsOrphanSubnet(*subnet, subnetSetIDs) { continue } - if err := r.Service.DeleteSubnet(*subnet); err != nil { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) + if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteFailTotal, MetricResTypeSubnetSet) } else { - metrics.CounterInc(r.Service.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) + metrics.CounterInc(r.SubnetService.NSXConfig, metrics.ControllerDeleteSuccessTotal, MetricResTypeSubnetSet) } } } } func (r *SubnetSetReconciler) DeleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, updataStatus bool) error { - nsxSubnets := r.Service.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.GetUID())) + nsxSubnets := r.SubnetService.SubnetStore.GetByIndex(servicecommon.TagScopeSubnetSetCRUID, string(obj.GetUID())) hitError := false for _, subnet := range nsxSubnets { - portNums := len(common.ServiceMediator.GetPortsOfSubnet(*subnet.Id)) + portNums := len(r.SubnetPortService.GetPortsOfSubnet(*subnet.Id)) if portNums > 0 { continue } - if err := r.Service.DeleteSubnet(*subnet); err != nil { + if err := r.SubnetService.DeleteSubnet(*subnet); err != nil { log.Error(err, "fail to delete subnet from subnetset cr", "ID", *subnet.Id) hitError = true } } if updataStatus { - if err := r.Service.UpdateSubnetSetStatus(&obj); err != nil { + if err := r.SubnetService.UpdateSubnetSetStatus(&obj); err != nil { return err } } @@ -305,12 +301,14 @@ func (r *SubnetSetReconciler) DeleteSubnetForSubnetSet(obj v1alpha1.SubnetSet, u return nil } -func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService) error { +func StartSubnetSetController(mgr ctrl.Manager, subnetService *subnet.SubnetService, subnetPortService servicecommon.SubnetPortServiceProvider, vpcService servicecommon.VPCServiceProvider) error { subnetsetReconciler := &SubnetSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SubnetService: subnetService, + SubnetPortService: subnetPortService, + VPCService: vpcService, } - subnetsetReconciler.Service = subnetService if err := subnetsetReconciler.Start(mgr); err != nil { log.Error(err, "failed to create controller", "controller", "Subnet") return err diff --git a/pkg/controllers/vpc/vpc_controller.go b/pkg/controllers/vpc/vpc_controller.go index 61e365e4d..27f109667 100644 --- a/pkg/controllers/vpc/vpc_controller.go +++ b/pkg/controllers/vpc/vpc_controller.go @@ -5,7 +5,6 @@ package vpc import ( "context" - "os" "runtime" "time" @@ -214,20 +213,3 @@ func (r *VPCReconciler) GarbageCollector(cancel chan bool, timeout time.Duration } } } - -func StartVPCController(mgr ctrl.Manager, commonService commonservice.Service) { - vpcReconcile := VPCReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - } - vpcService, err := vpc.InitializeVPC(commonService) - if err != nil { - log.Error(err, "failed to initialize VPC commonService") - os.Exit(1) - } - vpcReconcile.Service = vpcService - if err := vpcReconcile.Start(mgr); err != nil { - log.Error(err, "failed to create VPC controller") - os.Exit(1) - } -} diff --git a/pkg/nsx/services/common/services.go b/pkg/nsx/services/common/services.go index 748d500c2..86df100ca 100644 --- a/pkg/nsx/services/common/services.go +++ b/pkg/nsx/services/common/services.go @@ -1,8 +1,12 @@ package common -import "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +import ( + "sigs.k8s.io/controller-runtime/pkg/client" -// The method in this interface can be provided to other controllers. + "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" +) + +// VPCServiceProvider provides to methods other controllers and services. // Using interface instead vpc service instance can prevent other service // calling method that should not be exposed. type VPCServiceProvider interface { @@ -15,6 +19,17 @@ type VPCServiceProvider interface { ListVPCInfo(ns string) []VPCResourceInfo } +type SubnetServiceProvider interface { + GetSubnetByKey(key string) *model.VpcSubnet + GetSubnetsByIndex(key, value string) []*model.VpcSubnet + CreateOrUpdateSubnet(obj client.Object, vpcInfo VPCResourceInfo, tags []model.Tag) (string, error) + GenerateSubnetNSTags(obj client.Object, nsUID string) []model.Tag +} + +type SubnetPortServiceProvider interface { + GetPortsOfSubnet(nsxSubnetID string) (ports []*model.VpcSubnetPort) +} + type NodeServiceReader interface { GetNodeByName(nodeName string) []*model.HostTransportNode } diff --git a/pkg/nsx/services/mediator/mediator.go b/pkg/nsx/services/mediator/mediator.go deleted file mode 100644 index 4315c9031..000000000 --- a/pkg/nsx/services/mediator/mediator.go +++ /dev/null @@ -1,88 +0,0 @@ -package mediator - -import ( - "context" - "fmt" - - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - - "github.com/vmware-tanzu/nsx-operator/pkg/apis/v1alpha1" - "github.com/vmware-tanzu/nsx-operator/pkg/logger" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" - "github.com/vmware-tanzu/nsx-operator/pkg/util" -) - -var log = logger.Log - -// ServiceMediator We use mediator pattern to wrap all the services, -// embed all the services in ServiceMediator, so that we can mediate all the methods of all the services -// transparently to the caller, for example, in other packages, we can use ServiceMediator.GetVPCsByNamespace directly. -// In startCRDController function, we register the CRDService to the ServiceMediator, since only one controller writes to -// its own store and other controllers read from the store, so we don't need lock here. -type ServiceMediator struct { - *vpc.VPCService - *subnet.SubnetService - *subnetport.SubnetPortService -} - -// This method is used for subnet service since vpc network config contains default subnet size -// and default subnet access mode. -func (m *ServiceMediator) GetVPCNetworkConfigByNamespace(ns string) *common.VPCNetworkConfigInfo { - return m.VPCService.GetVPCNetworkConfigByNamespace(ns) -} - -// GetAvailableSubnet returns available Subnet under SubnetSet, and creates Subnet if necessary. -func (serviceMediator *ServiceMediator) GetAvailableSubnet(subnetSet *v1alpha1.SubnetSet) (string, error) { - subnetList := serviceMediator.SubnetStore.GetByIndex(common.TagScopeSubnetSetCRUID, string(subnetSet.GetUID())) - for _, nsxSubnet := range subnetList { - portNums := len(serviceMediator.GetPortsOfSubnet(*nsxSubnet.Id)) - totalIP := int(*nsxSubnet.Ipv4SubnetSize) - if len(nsxSubnet.IpAddresses) > 0 { - // totalIP will be overrided if IpAddresses are specified. - totalIP, _ = util.CalculateIPFromCIDRs(nsxSubnet.IpAddresses) - } - if portNums < totalIP-3 { - return *nsxSubnet.Path, nil - } - } - namespace := &corev1.Namespace{} - namespacedName := types.NamespacedName{ - Name: subnetSet.Namespace, - } - if err := serviceMediator.SubnetService.Client.Get(context.Background(), namespacedName, namespace); err != nil { - return "", err - } - tags := serviceMediator.SubnetService.GenerateSubnetNSTags(subnetSet, string(namespace.UID)) - for k, v := range namespace.Labels { - tags = append(tags, model.Tag{Scope: common.String(k), Tag: common.String(v)}) - } - log.Info("the existing subnets are not available, creating new subnet", "subnetList", subnetList, "subnetSet.Name", subnetSet.Name, "subnetSet.Namespace", subnetSet.Namespace) - vpcInfo, err := serviceMediator.GetNamespaceVPCInfo(subnetSet.Namespace) - if err != nil { - return "", err - } - return serviceMediator.CreateOrUpdateSubnet(subnetSet, *vpcInfo, tags) -} - -func (serviceMediator *ServiceMediator) GetNamespaceVPCInfo(ns string) (*common.VPCResourceInfo, error) { - vpcList := serviceMediator.GetVPCsByNamespace(ns) - if len(vpcList) == 0 { - return nil, fmt.Errorf("no vpc found for ns %s", ns) - } - vpcInfo, err := common.ParseVPCResourcePath(*vpcList[0].Path) - if err != nil { - err := fmt.Errorf("failed to parse NSX VPC path for VPC %s: %s", *vpcList[0].Id, err) - return nil, err - } - return &vpcInfo, nil -} - -func (serviceMediator *ServiceMediator) GetPortsOfSubnet(nsxSubnetID string) (ports []*model.VpcSubnetPort) { - subnetPortList := serviceMediator.SubnetPortStore.GetByIndex(common.IndexKeySubnetID, nsxSubnetID) - return subnetPortList -} diff --git a/pkg/nsx/services/mediator/mediator_test.go b/pkg/nsx/services/mediator/mediator_test.go deleted file mode 100644 index 28f7b21db..000000000 --- a/pkg/nsx/services/mediator/mediator_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package mediator - -import ( - "reflect" - "testing" - - "github.com/agiledragon/gomonkey/v2" - "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" - - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" - "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" -) - -func TestServiceMediator_GetOrgProject(t *testing.T) { - vpcService := &vpc.VPCService{} - vs := &ServiceMediator{ - VPCService: vpcService, - } - - patches := gomonkey.ApplyMethod(reflect.TypeOf(vpcService), "GetVPCsByNamespace", func(_ *vpc.VPCService, ns string) []*model.Vpc { - return []*model.Vpc{{Path: common.String("/orgs/default/projects/project-1/vpcs/vpc-1")}} - }) - defer patches.Reset() - - got := vs.ListVPCInfo("ns")[0] - want := common.VPCResourceInfo{OrgID: "default", ProjectID: "project-1", VPCID: "vpc-1", ID: "vpc-1", ParentID: "project-1"} - if !reflect.DeepEqual(got, want) { - t.Errorf("GetOrgProject() = %v, want %v", got, want) - } -} diff --git a/pkg/nsx/services/subnet/subnet.go b/pkg/nsx/services/subnet/subnet.go index e171b093b..30a9d33b3 100644 --- a/pkg/nsx/services/subnet/subnet.go +++ b/pkg/nsx/services/subnet/subnet.go @@ -8,6 +8,7 @@ import ( "time" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -273,6 +274,10 @@ func (service *SubnetService) UpdateSubnetSetStatus(obj *v1alpha1.SubnetSet) err return nil } +func (service *SubnetService) GetSubnetByKey(key string) *model.VpcSubnet { + return service.SubnetStore.GetByKey(key) +} + func (service *SubnetService) ListSubnetID() sets.Set[string] { subnets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetCRUID) subnetSets := service.SubnetStore.ListIndexFuncValues(common.TagScopeSubnetSetCRUID) @@ -294,7 +299,19 @@ func (service *SubnetService) Cleanup() error { return nil } -func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, nsUID string) []model.Tag { +func (service *SubnetService) GetSubnetsByIndex(key, value string) []*model.VpcSubnet { + return service.SubnetStore.GetByIndex(key, value) +} + +func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, ns string) []model.Tag { + namespace := &v1.Namespace{} + namespacedName := types.NamespacedName{ + Name: ns, + } + if err := service.Client.Get(context.Background(), namespacedName, namespace); err != nil { + return nil + } + nsUID := string(namespace.UID) var tags []model.Tag switch o := obj.(type) { case *v1alpha1.Subnet: @@ -319,6 +336,9 @@ func (service *SubnetService) GenerateSubnetNSTags(obj client.Object, nsUID stri model.Tag{Scope: common.String(common.TagScopeVMNamespace), Tag: common.String(obj.GetNamespace())}) } } + for k, v := range namespace.Labels { + tags = append(tags, model.Tag{Scope: common.String(k), Tag: common.String(v)}) + } return tags } diff --git a/pkg/nsx/services/subnetport/subnetport.go b/pkg/nsx/services/subnetport/subnetport.go index d437dfcc6..f10ce5280 100644 --- a/pkg/nsx/services/subnetport/subnetport.go +++ b/pkg/nsx/services/subnetport/subnetport.go @@ -267,6 +267,11 @@ func (service *SubnetPortService) GetSubnetPathForSubnetPortFromStore(nsxSubnetP return *existingSubnetPort.ParentPath } +func (service *SubnetPortService) GetPortsOfSubnet(nsxSubnetID string) (ports []*model.VpcSubnetPort) { + subnetPortList := service.SubnetPortStore.GetByIndex(servicecommon.IndexKeySubnetID, nsxSubnetID) + return subnetPortList +} + func (service *SubnetPortService) Cleanup() error { subnetPorts := service.SubnetPortStore.List() log.Info("cleanup subnetports", "count", len(subnetPorts))