diff --git a/pkg/controllers/namespace/namespace_controller.go b/pkg/controllers/namespace/namespace_controller.go index 372e7f660..1d26c1615 100644 --- a/pkg/controllers/namespace/namespace_controller.go +++ b/pkg/controllers/namespace/namespace_controller.go @@ -210,7 +210,6 @@ func (r *NamespaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( log.Error(err, "failed to build namespace and network config bindings", "Namepspace", ns) return common.ResultRequeueAfter10sec, nil } - // read annotation "nsx.vmware.com/shared_vpc_namespace", if ns contains this annotation, it means it will share infra VPC ncName, ncExist := annotations[types.AnnotationVPCNetworkConfig] // If ns do not have network config name tag, then use default vpc network config name diff --git a/pkg/controllers/networkinfo/networkinfo_controller.go b/pkg/controllers/networkinfo/networkinfo_controller.go index f10290145..11265dc59 100644 --- a/pkg/controllers/networkinfo/networkinfo_controller.go +++ b/pkg/controllers/networkinfo/networkinfo_controller.go @@ -124,8 +124,18 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) return common.ResultRequeueAfter10sec, err } + var privateIPs []string + var vpcConnectivityProfilePath string + if vpc.IsPreCreatedVPC(nc) { + privateIPs = createdVpc.PrivateIps + vpcConnectivityProfilePath = *createdVpc.VpcConnectivityProfile + } else { + privateIPs = nc.PrivateIPs + vpcConnectivityProfilePath = nc.VPCConnectivityProfile + } + snatIP, path, cidr := "", "", "" - parts := strings.Split(nc.VPCConnectivityProfile, "/") + parts := strings.Split(vpcConnectivityProfilePath, "/") if len(parts) < 1 { log.Error(err, "failed to check VPCConnectivityProfile length", "VPCConnectivityProfile", nc.VPCConnectivityProfile) return common.ResultRequeue, err @@ -160,7 +170,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) VPCPath: *createdVpc.Path, DefaultSNATIP: "", LoadBalancerIPAddresses: "", - PrivateIPs: nc.PrivateIPs, + PrivateIPs: privateIPs, } updateFail(r, &ctx, obj, &err, r.Client, state) return common.ResultRequeueAfter10sec, err @@ -196,7 +206,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) VPCPath: *createdVpc.Path, DefaultSNATIP: snatIP, LoadBalancerIPAddresses: "", - PrivateIPs: nc.PrivateIPs, + PrivateIPs: privateIPs, } updateFail(r, &ctx, obj, &err, r.Client, state) return common.ResultRequeueAfter10sec, err @@ -208,7 +218,7 @@ func (r *NetworkInfoReconciler) Reconcile(ctx context.Context, req ctrl.Request) VPCPath: *createdVpc.Path, DefaultSNATIP: snatIP, LoadBalancerIPAddresses: cidr, - PrivateIPs: nc.PrivateIPs, + PrivateIPs: privateIPs, } // AKO needs to know the AVI subnet path created by NSX setVPCNetworkConfigurationStatusWithLBS(&ctx, r.Client, ncName, state.Name, path, r.Service.GetNSXLBSPath(*createdVpc.Id)) diff --git a/pkg/controllers/networkinfo/networkinfo_utils.go b/pkg/controllers/networkinfo/networkinfo_utils.go index 5d9eace40..53e6ac00d 100644 --- a/pkg/controllers/networkinfo/networkinfo_utils.go +++ b/pkg/controllers/networkinfo/networkinfo_utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "slices" "github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model" v1 "k8s.io/api/core/v1" @@ -52,10 +53,14 @@ func setNetworkInfoVPCStatus(ctx *context.Context, networkInfo *v1alpha1.Network if len(networkInfo.VPCs) > 0 { existingVPC = &networkInfo.VPCs[0] } - if !reflect.DeepEqual(*existingVPC, *createdVPC) { - networkInfo.VPCs = []v1alpha1.VPCState{*createdVPC} - client.Update(*ctx, networkInfo) + slices.Sort(existingVPC.PrivateIPs) + slices.Sort(createdVPC.PrivateIPs) + if reflect.DeepEqual(*existingVPC, *createdVPC) { + return } + networkInfo.VPCs = []v1alpha1.VPCState{*createdVPC} + client.Update(*ctx, networkInfo) + return } func setVPCNetworkConfigurationStatusWithLBS(ctx *context.Context, client client.Client, ncName string, vpcName string, aviSubnetPath string, nsxLBSPath string) { diff --git a/pkg/controllers/networkinfo/vpcnetworkconfig_handler.go b/pkg/controllers/networkinfo/vpcnetworkconfig_handler.go index e10ff944f..94922e77c 100644 --- a/pkg/controllers/networkinfo/vpcnetworkconfig_handler.go +++ b/pkg/controllers/networkinfo/vpcnetworkconfig_handler.go @@ -115,6 +115,7 @@ func buildNetworkConfigInfo(vpcConfigCR v1alpha1.VPCNetworkConfiguration) (*comm DefaultSubnetSize: vpcConfigCR.Spec.DefaultSubnetSize, PodSubnetAccessMode: vpcConfigCR.Spec.PodSubnetAccessMode, ShortID: vpcConfigCR.Spec.ShortID, + VPCPath: vpcConfigCR.Spec.VPC, } return ninfo, nil } diff --git a/pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go b/pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go index e29297b76..7ad2fcdb4 100644 --- a/pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go +++ b/pkg/controllers/networkinfo/vpcnetworkconfig_handler_test.go @@ -85,6 +85,12 @@ func TestBuildNetworkConfigInfo(t *testing.T) { PodSubnetAccessMode: "Private", NSXProject: "/orgs/anotherOrg/projects/anotherProject", } + spec3 := v1alpha1.VPCNetworkConfigurationSpec{ + DefaultSubnetSize: 28, + PodSubnetAccessMode: "Private", + NSXProject: "/orgs/anotherOrg/projects/anotherProject", + VPC: "vpc33", + } testCRD1 := v1alpha1.VPCNetworkConfiguration{ Spec: spec1, } @@ -104,6 +110,16 @@ func TestBuildNetworkConfigInfo(t *testing.T) { } testCRD3.Name = "test-3" + testCRD4 := v1alpha1.VPCNetworkConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + types.AnnotationDefaultNetworkConfig: "false", + }, + }, + Spec: spec3, + } + testCRD3.Name = "test-4" + tests := []struct { name string nc v1alpha1.VPCNetworkConfiguration @@ -115,10 +131,12 @@ func TestBuildNetworkConfigInfo(t *testing.T) { accessMode string isDefault bool vpcConnectivityProfile string + vpcPath string }{ - {"test-nsxtProjectPathToId", testCRD1, "test-gw-path-1", "test-edge-path-1", "default", "nsx_operator_e2e_test", 64, "Public", false, ""}, - {"with-VPCConnectivityProfile", testCRD2, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", false, "test-VPCConnectivityProfile"}, - {"with-defaultNetworkConfig", testCRD3, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", true, ""}, + {"test-nsxtProjectPathToId", testCRD1, "test-gw-path-1", "test-edge-path-1", "default", "nsx_operator_e2e_test", 64, "Public", false, "", ""}, + {"with-VPCConnectivityProfile", testCRD2, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", false, "test-VPCConnectivityProfile", ""}, + {"with-defaultNetworkConfig", testCRD3, "test-gw-path-2", "test-edge-path-2", "anotherOrg", "anotherProject", 32, "Private", true, "", ""}, + {"with-preCreatedVPC", testCRD4, "", "", "anotherOrg", "anotherProject", 28, "Private", false, "", "vpc33"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -131,7 +149,7 @@ func TestBuildNetworkConfigInfo(t *testing.T) { assert.Equal(t, tt.subnetSize, nc.DefaultSubnetSize) assert.Equal(t, tt.accessMode, nc.PodSubnetAccessMode) assert.Equal(t, tt.isDefault, nc.IsDefault) + assert.Equal(t, tt.vpcPath, nc.VPCPath) }) } - } diff --git a/pkg/controllers/securitypolicy/pod_hanlder_test.go b/pkg/controllers/securitypolicy/pod_handler_test.go similarity index 100% rename from pkg/controllers/securitypolicy/pod_hanlder_test.go rename to pkg/controllers/securitypolicy/pod_handler_test.go diff --git a/pkg/nsx/services/common/types.go b/pkg/nsx/services/common/types.go index 0b3ad6ac3..5cd4cd275 100644 --- a/pkg/nsx/services/common/types.go +++ b/pkg/nsx/services/common/types.go @@ -59,6 +59,8 @@ const ( TagScopeIPSubnetName string = "nsx-op/ipsubnet_name" TagScopeVMNamespaceUID string = "nsx-op/vm_namespace_uid" TagScopeVMNamespace string = "nsx-op/vm_namespace" + TagScopeVPCManagedBy string = "nsx/managed-by" + AutoCreatedVPCTagValue string = "nsx-op" LabelDefaultSubnetSet string = "nsxoperator.vmware.com/default-subnetset-for" LabelDefaultVMSubnetSet string = "VirtualMachine" LabelDefaultPodSubnetSet string = "Pod" @@ -218,4 +220,5 @@ type VPCNetworkConfigInfo struct { DefaultSubnetSize int PodSubnetAccessMode string ShortID string + VPCPath string } diff --git a/pkg/nsx/services/vpc/builder.go b/pkg/nsx/services/vpc/builder.go index 4b8c09176..1996da549 100644 --- a/pkg/nsx/services/vpc/builder.go +++ b/pkg/nsx/services/vpc/builder.go @@ -74,6 +74,8 @@ func buildNSXVPC(obj *v1alpha1.NetworkInfo, nsObj *v1.Namespace, nc common.VPCNe vpc.LoadBalancerVpcEndpoint = &model.LoadBalancerVPCEndpoint{Enabled: &loadBalancerVPCEndpointEnabled} } vpc.Tags = util.BuildBasicTags(cluster, obj, nsObj.UID) + vpc.Tags = append(vpc.Tags, model.Tag{ + Scope: common.String(common.TagScopeVPCManagedBy), Tag: common.String(common.AutoCreatedVPCTagValue)}) } if nc.VPCConnectivityProfile != "" { diff --git a/pkg/nsx/services/vpc/builder_test.go b/pkg/nsx/services/vpc/builder_test.go index f925e4104..68ab81060 100644 --- a/pkg/nsx/services/vpc/builder_test.go +++ b/pkg/nsx/services/vpc/builder_test.go @@ -134,6 +134,7 @@ func TestBuildNSXVPC(t *testing.T) { {Scope: common.String("nsx-op/version"), Tag: common.String("1.0.0")}, {Scope: common.String("nsx-op/namespace"), Tag: common.String("ns1")}, {Scope: common.String("nsx-op/namespace_uid"), Tag: common.String("nsuid1")}, + {Scope: common.String("nsx/managed-by"), Tag: common.String("nsx-op")}, }, }, }, @@ -152,6 +153,7 @@ func TestBuildNSXVPC(t *testing.T) { {Scope: common.String("nsx-op/version"), Tag: common.String("1.0.0")}, {Scope: common.String("nsx-op/namespace"), Tag: common.String("ns1")}, {Scope: common.String("nsx-op/namespace_uid"), Tag: common.String("nsuid1")}, + {Scope: common.String("nsx/managed-by"), Tag: common.String("nsx-op")}, }, }, }, diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 3ea1cfa49..e64f0cb9f 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -49,7 +49,7 @@ var ( ) type VPCNetworkInfoStore struct { - sync.Mutex + sync.RWMutex VPCNetworkConfigMap map[string]common.VPCNetworkConfigInfo } @@ -143,6 +143,10 @@ func (s *VPCService) GetVPCNetworkConfigByNamespace(ns string) *common.VPCNetwor // TBD: for now, if network config info do not contains private cidr, we consider this is // incorrect configuration, and skip creating this VPC CR func (s *VPCService) ValidateNetworkConfig(nc common.VPCNetworkConfigInfo) bool { + if IsPreCreatedVPC(nc) { + // if network config is using a pre-created VPC, skip the check on PrivateIPs. + return true + } return nc.PrivateIPs != nil && len(nc.PrivateIPs) != 0 } @@ -521,6 +525,16 @@ func (s *VPCService) CreateOrUpdateVPC(obj *v1alpha1.NetworkInfo, nc *common.VPC return nil, err } + // Return pre-created VPC resource if it is used in the VPCNetworkConfiguration + if IsPreCreatedVPC(*nc) { + preVPC, err := s.GetVPCFromNSXByPath(nc.VPCPath) + if err != nil { + log.Error(err, "Failed to get existing VPC from NSX", "vpcPath", nc.VPCPath) + return nil, err + } + return preVPC, nil + } + // check if this namespace vpc share from others, if yes // then check if the shared vpc created or not, if yes // then directly return this vpc, if not, requeue @@ -771,6 +785,19 @@ func (s *VPCService) Cleanup(ctx context.Context) error { func (service *VPCService) ListVPCInfo(ns string) []common.VPCResourceInfo { var VPCInfoList []common.VPCResourceInfo + nc := service.GetVPCNetworkConfigByNamespace(ns) + // Return the pre-created VPC resource info if it is set in VPCNetworkConfiguration. + if nc != nil && IsPreCreatedVPC(*nc) { + vpcResourceInfo, err := common.ParseVPCResourcePath(nc.VPCPath) + if err != nil { + log.Error(err, "Failed to get vpc info from vpc path", "vpc path", nc.VPCPath) + } else { + VPCInfoList = append(VPCInfoList, vpcResourceInfo) + } + return VPCInfoList + } + + // List VPCs from local store. vpcs := service.GetVPCsByNamespace(ns) // Transparently call the VPCService.GetVPCsByNamespace method for _, v := range vpcs { vpcResourceInfo, err := common.ParseVPCResourcePath(*v.Path) @@ -836,3 +863,23 @@ func (vpcService *VPCService) getLBProvider() string { } return LBProviderAVI } + +func (service *VPCService) GetVPCFromNSXByPath(vpcPath string) (*model.Vpc, error) { + vpcResInfo, err := common.ParseVPCResourcePath(vpcPath) + if err != nil { + log.Error(err, "failed to parse VPCResourceInfo from the given VPC path", "VPC", vpcPath) + return nil, err + } + vpc, err := service.NSXClient.VPCClient.Get(vpcResInfo.OrgID, vpcResInfo.ProjectID, vpcResInfo.VPCID) + err = nsxutil.NSXApiError(err) + if err != nil { + log.Error(err, "failed to read VPC object from NSX", "VPC", vpcPath) + return nil, err + } + + return &vpc, nil +} + +func IsPreCreatedVPC(nc common.VPCNetworkConfigInfo) bool { + return nc.VPCPath != "" +} diff --git a/test/e2e/nsx_subnet_test.go b/test/e2e/nsx_subnet_test.go index d0a31a379..61a81ebb1 100644 --- a/test/e2e/nsx_subnet_test.go +++ b/test/e2e/nsx_subnet_test.go @@ -314,7 +314,7 @@ func SubnetCIDR(t *testing.T) { err = nil } assertNil(t, err) - err = testData.waitForCRReadyOrDeleted(defaultTimeout, "subnets.crd.nsx.vmware.com", E2ENamespace, subnet.Name, Ready) + err = testData.waitForCRReadyOrDeleted(defaultTimeout*2, "subnets.crd.nsx.vmware.com", E2ENamespace, subnet.Name, Ready) assertNil(t, err) allocatedSubnet, err = testData.crdClientset.CrdV1alpha1().Subnets(E2ENamespace).Get(context.TODO(), subnet.Name, v1.GetOptions{}) assertNil(t, err)