Add support for pre-created VPC in VPCNetworkConfiguration CR

1. Skip NSX VPC creations if VPC field is set in the VPCNetworkConfiguration CR
   spec.
2. Using the pre-created VPC's private IPs. connectivity profile and other
   necessary settings to set NetworkInfo CR status.
3. Add tag "nsx/managed-by: nsx-op" on the NSX VPCs which are created by
   nsx-operator.

pkg/controllers/namespace/namespace_controller.go

@@ -210,7 +210,6 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 			log.Error(err, "failed to build namespace and network config bindings", "Namepspace", ns)
 			return common.ResultRequeueAfter10sec, nil
 		}
-		// read annotation "nsx.vmware.com/shared_vpc_namespace", if ns contains this annotation, it means it will share infra VPC
 		ncName, ncExist := annotations[types.AnnotationVPCNetworkConfig]
 
 		// If ns do not have network config name tag, then use default vpc network config name

pkg/controllers/networkinfo/networkinfo_controller.go

@@ -124,8 +124,18 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			return common.ResultRequeueAfter10sec, err
 		}
 
+		var privateIPs []string
+		var vpcConnectivityProfilePath string
+		if vpc.IsPreCreatedVPC(nc) {
+			privateIPs = createdVpc.PrivateIps
+			vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile
+		} else {
+			privateIPs = nc.PrivateIPs
+			vpcConnectivityProfilePath = nc.VPCConnectivityProfile
+		}
+
 		snatIP, path, cidr := "", "", ""
-		parts := strings.Split(nc.VPCConnectivityProfile, "/")
+		parts := strings.Split(vpcConnectivityProfilePath, "/")
 		if len(parts) < 1 {
 			log.Error(err, "failed to check VPCConnectivityProfile length", "VPCConnectivityProfile", nc.VPCConnectivityProfile)
 			return common.ResultRequeue, err
@@ -160,7 +170,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 					VPCPath:                 *createdVpc.Path,
 					DefaultSNATIP:           "",
 					LoadBalancerIPAddresses: "",
-					PrivateIPs:              nc.PrivateIPs,
+					PrivateIPs:              privateIPs,
 				}
 				updateFail(r, &ctx, obj, &err, r.Client, state)
 				return common.ResultRequeueAfter10sec, err
@@ -196,7 +206,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 					VPCPath:                 *createdVpc.Path,
 					DefaultSNATIP:           snatIP,
 					LoadBalancerIPAddresses: "",
-					PrivateIPs:              nc.PrivateIPs,
+					PrivateIPs:              privateIPs,
 				}
 				updateFail(r, &ctx, obj, &err, r.Client, state)
 				return common.ResultRequeueAfter10sec, err
@@ -208,7 +218,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			VPCPath:                 *createdVpc.Path,
 			DefaultSNATIP:           snatIP,
 			LoadBalancerIPAddresses: cidr,
-			PrivateIPs:              nc.PrivateIPs,
+			PrivateIPs:              privateIPs,
 		}
 		// AKO needs to know the AVI subnet path created by NSX
 		setVPCNetworkConfigurationStatusWithLBS(&ctx, r.Client, ncName, state.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id))

pkg/controllers/networkinfo/networkinfo_utils.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"slices"
 
 	"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
 	v1 "k8s.io/api/core/v1"
@@ -52,10 +53,14 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network
 	if len(networkInfo.VPCs) > 0 {
 		existingVPC = &networkInfo.VPCs[0]
 	}
-	if !reflect.DeepEqual(*existingVPC, *createdVPC) {
-		networkInfo.VPCs = []v1alpha1.VPCState{*createdVPC}
-		client.Update(*ctx, networkInfo)
+	slices.Sort(existingVPC.PrivateIPs)
+	slices.Sort(createdVPC.PrivateIPs)
+	if reflect.DeepEqual(*existingVPC, *createdVPC) {
+		return
 	}
+	networkInfo.VPCs = []v1alpha1.VPCState{*createdVPC}
+	client.Update(*ctx, networkInfo)
+	return
 }
 
 func setVPCNetworkConfigurationStatusWithLBS(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) {

pkg/controllers/networkinfo/vpcnetworkconfig_handler.go

@@ -115,6 +115,7 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*comm
 		DefaultSubnetSize:      vpcConfigCR.Spec.DefaultSubnetSize,
 		PodSubnetAccessMode:    vpcConfigCR.Spec.PodSubnetAccessMode,
 		ShortID:                vpcConfigCR.Spec.ShortID,
+		VPCPath:                vpcConfigCR.Spec.VPC,
 	}
 	return ninfo, nil
 }

pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go

@@ -85,6 +85,12 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 		PodSubnetAccessMode: "Private",
 		NSXProject:          "/orgs/anotherOrg/projects/anotherProject",
 	}
+	spec3 := v1alpha1.VPCNetworkConfigurationSpec{
+		DefaultSubnetSize:   28,
+		PodSubnetAccessMode: "Private",
+		NSXProject:          "/orgs/anotherOrg/projects/anotherProject",
+		VPC:                 "vpc33",
+	}
 	testCRD1 := v1alpha1.VPCNetworkConfiguration{
 		Spec: spec1,
 	}
@@ -104,6 +110,16 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 	}
 	testCRD3.Name = "test-3"
 
+	testCRD4 := v1alpha1.VPCNetworkConfiguration{
+		ObjectMeta: metav1.ObjectMeta{
+			Annotations: map[string]string{
+				types.AnnotationDefaultNetworkConfig: "false",
+			},
+		},
+		Spec: spec3,
+	}
+	testCRD3.Name = "test-4"
+
 	tests := []struct {
 		name                   string
 		nc                     v1alpha1.VPCNetworkConfiguration
@@ -115,10 +131,12 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 		accessMode             string
 		isDefault              bool
 		vpcConnectivityProfile string
+		vpcPath                string
 	}{
-		{"test-nsxtProjectPathToId", testCRD1, "test-gw-path-1", "test-edge-path-1", "default", "nsx_operator_e2e_test", 64, "Public", false, ""},
-		{"with-VPCConnectivityProfile", testCRD2, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", false, "test-VPCConnectivityProfile"},
-		{"with-defaultNetworkConfig", testCRD3, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", true, ""},
+		{"test-nsxtProjectPathToId", testCRD1, "test-gw-path-1", "test-edge-path-1", "default", "nsx_operator_e2e_test", 64, "Public", false, "", ""},
+		{"with-VPCConnectivityProfile", testCRD2, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", false, "test-VPCConnectivityProfile", ""},
+		{"with-defaultNetworkConfig", testCRD3, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", true, "", ""},
+		{"with-preCreatedVPC", testCRD4, "", "", "anotherOrg", "anotherProject", 28, "Private", false, "", "vpc33"},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -131,7 +149,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) {
 			assert.Equal(t, tt.subnetSize, nc.DefaultSubnetSize)
 			assert.Equal(t, tt.accessMode, nc.PodSubnetAccessMode)
 			assert.Equal(t, tt.isDefault, nc.IsDefault)
+			assert.Equal(t, tt.vpcPath, nc.VPCPath)
 		})
 	}
-
 }

pkg/nsx/services/common/types.go

@@ -59,6 +59,8 @@ const (
 	TagScopeIPSubnetName               string = "nsx-op/ipsubnet_name"
 	TagScopeVMNamespaceUID             string = "nsx-op/vm_namespace_uid"
 	TagScopeVMNamespace                string = "nsx-op/vm_namespace"
+	TagScopeVPCManagedBy               string = "nsx/managed-by"
+	AutoCreatedVPCTagValue             string = "nsx-op"
 	LabelDefaultSubnetSet              string = "nsxoperator.vmware.com/default-subnetset-for"
 	LabelDefaultVMSubnetSet            string = "VirtualMachine"
 	LabelDefaultPodSubnetSet           string = "Pod"
@@ -218,4 +220,5 @@ type VPCNetworkConfigInfo struct {
 	DefaultSubnetSize      int
 	PodSubnetAccessMode    string
 	ShortID                string
+	VPCPath                string
 }

pkg/nsx/services/vpc/builder.go

@@ -74,6 +74,8 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe
 			vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled}
 		}
 		vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID)
+		vpc.Tags = append(vpc.Tags, model.Tag{
+			Scope: common.String(common.TagScopeVPCManagedBy), Tag: common.String(common.AutoCreatedVPCTagValue)})
 	}
 
 	if nc.VPCConnectivityProfile != "" {

pkg/nsx/services/vpc/builder_test.go

@@ -134,6 +134,7 @@ func TestBuildNSXVPC(t *testing.T) {
 					{Scope: common.String("nsx-op/version"), Tag: common.String("1.0.0")},
 					{Scope: common.String("nsx-op/namespace"), Tag: common.String("ns1")},
 					{Scope: common.String("nsx-op/namespace_uid"), Tag: common.String("nsuid1")},
+					{Scope: common.String("nsx/managed-by"), Tag: common.String("nsx-op")},
 				},
 			},
 		},
@@ -152,6 +153,7 @@ func TestBuildNSXVPC(t *testing.T) {
 					{Scope: common.String("nsx-op/version"), Tag: common.String("1.0.0")},
 					{Scope: common.String("nsx-op/namespace"), Tag: common.String("ns1")},
 					{Scope: common.String("nsx-op/namespace_uid"), Tag: common.String("nsuid1")},
+					{Scope: common.String("nsx/managed-by"), Tag: common.String("nsx-op")},
 				},
 			},
 		},

pkg/nsx/services/vpc/vpc.go

@@ -49,7 +49,7 @@ var (
 )
 
 type VPCNetworkInfoStore struct {
-	sync.Mutex
+	sync.RWMutex
 	VPCNetworkConfigMap map[string]common.VPCNetworkConfigInfo
 }
 
@@ -143,6 +143,10 @@ func (s *VPCService) GetVPCNetworkConfigByNamespace(ns string) *common.VPCNetwor
 // TBD: for now, if network config info do not contains private cidr, we consider this is
 // incorrect configuration, and skip creating this VPC CR
 func (s *VPCService) ValidateNetworkConfig(nc common.VPCNetworkConfigInfo) bool {
+	if IsPreCreatedVPC(nc) {
+		// if network config is using a pre-created VPC, skip the check on PrivateIPs.
+		return true
+	}
 	return nc.PrivateIPs != nil && len(nc.PrivateIPs) != 0
 }
 
@@ -521,6 +525,16 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC
 		return nil, err
 	}
 
+	// Return pre-created VPC resource if it is used in the VPCNetworkConfiguration
+	if IsPreCreatedVPC(*nc) {
+		preVPC, err := s.GetVPCFromNSXByPath(nc.VPCPath)
+		if err != nil {
+			log.Error(err, "Failed to get existing VPC from NSX", "vpcPath", nc.VPCPath)
+			return nil, err
+		}
+		return preVPC, nil
+	}
+
 	// check if this namespace vpc share from others, if yes
 	// then check if the shared vpc created or not, if yes
 	// then directly return this vpc, if not, requeue
@@ -771,6 +785,19 @@ func (s *VPCService) Cleanup(ctx context.Context) error {
 
 func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo {
 	var VPCInfoList []common.VPCResourceInfo
+	nc := service.GetVPCNetworkConfigByNamespace(ns)
+	// Return the pre-created VPC resource info if it is set in VPCNetworkConfiguration.
+	if nc != nil && IsPreCreatedVPC(*nc) {
+		vpcResourceInfo, err := common.ParseVPCResourcePath(nc.VPCPath)
+		if err != nil {
+			log.Error(err, "Failed to get vpc info from vpc path", "vpc path", nc.VPCPath)
+		} else {
+			VPCInfoList = append(VPCInfoList, vpcResourceInfo)
+		}
+		return VPCInfoList
+	}
+
+	// List VPCs from local store.
 	vpcs := service.GetVPCsByNamespace(ns) // Transparently call the VPCService.GetVPCsByNamespace method
 	for _, v := range vpcs {
 		vpcResourceInfo, err := common.ParseVPCResourcePath(*v.Path)
@@ -836,3 +863,23 @@ func (vpcService *VPCService) getLBProvider() string {
 	}
 	return LBProviderAVI
 }
+
+func (service *VPCService) GetVPCFromNSXByPath(vpcPath string) (*model.Vpc, error) {
+	vpcResInfo, err := common.ParseVPCResourcePath(vpcPath)
+	if err != nil {
+		log.Error(err, "failed to parse VPCResourceInfo from the given VPC path", "VPC", vpcPath)
+		return nil, err
+	}
+	vpc, err := service.NSXClient.VPCClient.Get(vpcResInfo.OrgID, vpcResInfo.ProjectID, vpcResInfo.VPCID)
+	err = nsxutil.NSXApiError(err)
+	if err != nil {
+		log.Error(err, "failed to read VPC object from NSX", "VPC", vpcPath)
+		return nil, err
+	}
+
+	return &vpc, nil
+}
+
+func IsPreCreatedVPC(nc common.VPCNetworkConfigInfo) bool {
+	return nc.VPCPath != ""
+}