diff --git a/api/v1alpha1/gatewaylbconfiguration_types.go b/api/v1alpha1/gatewaylbconfiguration_types.go index 974d1996..95842054 100644 --- a/api/v1alpha1/gatewaylbconfiguration_types.go +++ b/api/v1alpha1/gatewaylbconfiguration_types.go @@ -42,7 +42,11 @@ type GatewayLBConfigurationSpec struct { // Profile of the gateway VMSS to apply the gateway configuration. // +optional - GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"` + GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"` + + // Whether to provision public IP prefixes for outbound. + //+kubebuilder:default=true + ProvisionPublicIps bool `json:"provisionPublicIps"` // BYO Resource ID of public IP prefix to be used as outbound. // +optional @@ -55,13 +59,13 @@ type GatewayLBConfigurationStatus struct { // Important: Run "make" to regenerate code after modifying this file // Gateway IP for wireguard connection. - FrontendIP string `json:"frontendIP,omitempty"` + FrontendIp string `json:"frontendIp,omitempty"` // Listening port of the gateway side wireguard daemon. ServerPort int32 `json:"serverPort,omitempty"` - // Public IP Prefix CIDR used for this gateway configuration. - PublicIpPrefix string `json:"publicIpPrefix,omitempty"` + // Egress IP Prefix CIDR used for this gateway configuration. + EgressIpPrefix string `json:"egressIpPrefix,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/gatewayvmconfiguration_types.go b/api/v1alpha1/gatewayvmconfiguration_types.go index 4d258038..71fbf144 100644 --- a/api/v1alpha1/gatewayvmconfiguration_types.go +++ b/api/v1alpha1/gatewayvmconfiguration_types.go @@ -42,7 +42,11 @@ type GatewayVMConfigurationSpec struct { // Profile of the gateway VMSS to apply the gateway configuration. // +optional - GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"` + GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"` + + // Whether to provision public IP prefixes for outbound. + //+kubebuilder:default=true + ProvisionPublicIps bool `json:"provisionPublicIps"` // BYO Resource ID of public IP prefix to be used as outbound. // +optional diff --git a/api/v1alpha1/staticgatewayconfiguration_types.go b/api/v1alpha1/staticgatewayconfiguration_types.go index 06c7fb79..df4b80a1 100644 --- a/api/v1alpha1/staticgatewayconfiguration_types.go +++ b/api/v1alpha1/staticgatewayconfiguration_types.go @@ -32,13 +32,13 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -// GatewayVMSSProfile finds an existing gateway VMSS (virtual machine scale set). -type GatewayVMSSProfile struct { +// GatewayVmssProfile finds an existing gateway VMSS (virtual machine scale set). +type GatewayVmssProfile struct { // Resource group of the VMSS. Must be in the same subscription. - VMSSResourceGroup string `json:"vmssResourceGroup,omitempty"` + VmssResourceGroup string `json:"vmssResourceGroup,omitempty"` // Name of the VMSS - VMSSName string `json:"vmssName,omitempty"` + VmssName string `json:"vmssName,omitempty"` // Public IP prefix size to be applied to this VMSS. //+kubebuilder:validation:Minimum=0 @@ -46,6 +46,18 @@ type GatewayVMSSProfile struct { PublicIpPrefixSize int32 `json:"publicIpPrefixSize,omitempty"` } +// RouteType defines the type of defaultRoute. +// +kubebuilder:validation:Enum=azureNetworking;staticEgressGateway +type RouteType string + +const ( + // RouteStaticEgressGateway defines static egress gateway as the default route. + RouteStaticEgressGateway RouteType = "staticEgressGateway" + + // RouteAzureNetworking defines azure networking as the default route. + RouteAzureNetworking RouteType = "azureNetworking" +) + // StaticGatewayConfigurationSpec defines the desired state of StaticGatewayConfiguration type StaticGatewayConfigurationSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster @@ -57,20 +69,28 @@ type StaticGatewayConfigurationSpec struct { // Profile of the gateway VMSS to apply the gateway configuration. // +optional - GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"` + GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"` + + // Pod default route, should be either azureNetworking (pod's eth0) or staticEgressGateway (default). + //+kubebuilder:default=staticEgressGateway + DefaultRoute RouteType `json:"defaultRoute,omitempty"` + + // Whether to provision public IP prefixes for outbound. + //+kubebuilder:default=true + ProvisionPublicIps bool `json:"provisionPublicIps"` - // BYO Resource ID of public IP prefix to be used as outbound. + // BYO Resource ID of public IP prefix to be used as outbound. This can only be specified when provisionPublicIps is true. // +optional PublicIpPrefixId string `json:"publicIpPrefixId,omitempty"` - // CIDRs to be excluded from outbound. - ExcludeCIDRs []string `json:"excludeCIDRs,omitempty"` + // CIDRs to be excluded from the default route. + ExcludeCidrs []string `json:"excludeCidrs,omitempty"` } // GatewayWireguardProfile provides details about gateway side wireguard configuration. type GatewayWireguardProfile struct { // Gateway IP for wireguard connection. - WireguardServerIP string `json:"wireguardServerIP,omitempty"` + WireguardServerIp string `json:"wireguardServerIp,omitempty"` // Listening port of the gateway side wireguard daemon. WireguardServerPort int32 `json:"wireguardServerPort,omitempty"` @@ -87,8 +107,8 @@ type StaticGatewayConfigurationStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file - // Public IP Prefix CIDR used for this gateway configuration. - PublicIpPrefix string `json:"publicIpPrefix,omitempty"` + // Egress IP Prefix CIDR used for this gateway configuration. + EgressIpPrefix string `json:"egressIpPrefix,omitempty"` // Gateway side wireguard profile. GatewayWireguardProfile `json:"gatewayWireguardProfile,omitempty"` diff --git a/api/v1alpha1/staticgatewayconfiguration_webhook.go b/api/v1alpha1/staticgatewayconfiguration_webhook.go index 7ccad4cf..ecbbce39 100644 --- a/api/v1alpha1/staticgatewayconfiguration_webhook.go +++ b/api/v1alpha1/staticgatewayconfiguration_webhook.go @@ -80,39 +80,45 @@ func (r *StaticGatewayConfiguration) ValidateDelete() (admission.Warnings, error } func (r *StaticGatewayConfiguration) validateSGC() error { - // need to validate either GatewayNodepoolName or GatewayVMSSProfile is provided, but not both + // need to validate either GatewayNodepoolName or GatewayVmssProfile is provided, but not both var allErrs field.ErrorList if r.Spec.GatewayNodepoolName == "" && r.vmssProfileIsEmpty() { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewaynodepoolname"), - fmt.Sprintf("GatewayNodepoolName: %s, GatewayVMSSProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVMSSProfile), - "Either GatewayNodepoolName or GatewayVMSSProfile must be provided")) + fmt.Sprintf("GatewayNodepoolName: %s, GatewayVmssProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVmssProfile), + "Either GatewayNodepoolName or GatewayVmssProfile must be provided")) } if r.Spec.GatewayNodepoolName != "" && !r.vmssProfileIsEmpty() { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewaynodepoolname"), - fmt.Sprintf("GatewayNodepoolName: %s, GatewayVMSSProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVMSSProfile), - "Only one of GatewayNodepoolName and GatewayVMSSProfile should be provided")) + fmt.Sprintf("GatewayNodepoolName: %s, GatewayVmssProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVmssProfile), + "Only one of GatewayNodepoolName and GatewayVmssProfile should be provided")) } if !r.vmssProfileIsEmpty() { - if r.Spec.GatewayVMSSProfile.VMSSResourceGroup == "" { + if r.Spec.GatewayVmssProfile.VmssResourceGroup == "" { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("vmssresourcegroup"), - r.Spec.GatewayVMSSProfile.VMSSResourceGroup, + r.Spec.GatewayVmssProfile.VmssResourceGroup, "Gateway vmss resource group is empty")) } - if r.Spec.GatewayVMSSProfile.VMSSName == "" { + if r.Spec.GatewayVmssProfile.VmssName == "" { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("vmssname"), - r.Spec.GatewayVMSSProfile.VMSSName, + r.Spec.GatewayVmssProfile.VmssName, "Gateway vmss name is empty")) } - if r.Spec.GatewayVMSSProfile.PublicIpPrefixSize < 0 || r.Spec.GatewayVMSSProfile.PublicIpPrefixSize > 31 { + if r.Spec.GatewayVmssProfile.PublicIpPrefixSize < 0 || r.Spec.GatewayVmssProfile.PublicIpPrefixSize > 31 { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("publicipprefixsize"), - r.Spec.GatewayVMSSProfile.PublicIpPrefixSize, + r.Spec.GatewayVmssProfile.PublicIpPrefixSize, "Gateway vmss public ip prefix size should be between 0 and 31 inclusively")) } } + if !r.Spec.ProvisionPublicIps && r.Spec.PublicIpPrefixId != "" { + allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("publicipprefixid"), + r.Spec.PublicIpPrefixId, + "PublicIpPrefixId should be empty when ProvisionPublicIps is false")) + } + if len(allErrs) == 0 { return nil } @@ -122,7 +128,7 @@ func (r *StaticGatewayConfiguration) validateSGC() error { } func (r *StaticGatewayConfiguration) vmssProfileIsEmpty() bool { - return r.Spec.GatewayVMSSProfile.VMSSResourceGroup == "" && - r.Spec.GatewayVMSSProfile.VMSSName == "" && - r.Spec.GatewayVMSSProfile.PublicIpPrefixSize == 0 + return r.Spec.GatewayVmssProfile.VmssResourceGroup == "" && + r.Spec.GatewayVmssProfile.VmssName == "" && + r.Spec.GatewayVmssProfile.PublicIpPrefixSize == 0 } diff --git a/api/v1alpha1/staticgatewayconfiguration_webhook_test.go b/api/v1alpha1/staticgatewayconfiguration_webhook_test.go index 1dac1554..d925aa2f 100644 --- a/api/v1alpha1/staticgatewayconfiguration_webhook_test.go +++ b/api/v1alpha1/staticgatewayconfiguration_webhook_test.go @@ -29,49 +29,62 @@ import "testing" func TestValidateSGC(t *testing.T) { tests := map[string]struct { GatewayNodepoolName string - VMSSResourceGroup string - VMSSName string + VmssResourceGroup string + VmssName string PublicIpPrefixSize int32 + ProvisionPublicIps bool + PublicIpPrefixId string ExpectErr bool }{ "It should pass when only gateway nodepool name is provided": { GatewayNodepoolName: "test", }, "It should pass when vmss profile is provided": { - VMSSResourceGroup: "test", - VMSSName: "test", + VmssResourceGroup: "test", + VmssName: "test", PublicIpPrefixSize: 31, }, "It should fail when both gateway nodepool name and vmss profile are provided": { GatewayNodepoolName: "test", - VMSSResourceGroup: "test", + VmssResourceGroup: "test", ExpectErr: true, }, "It should fail when no gateway profile is provided": { ExpectErr: true, }, "It should fail when vmss resource group is not provided": { - VMSSName: "test", + VmssName: "test", PublicIpPrefixSize: 31, ExpectErr: true, }, "It should fail when vmss name is not provided": { - VMSSResourceGroup: "test", + VmssResourceGroup: "test", PublicIpPrefixSize: 31, ExpectErr: true, }, "It should fail when PublicIpPrefixSize < 0": { - VMSSResourceGroup: "test", - VMSSName: "test", + VmssResourceGroup: "test", + VmssName: "test", PublicIpPrefixSize: -1, ExpectErr: true, }, "It should fail when PublicIpPrefixSize > 31": { - VMSSResourceGroup: "test", - VMSSName: "test", + VmssResourceGroup: "test", + VmssName: "test", PublicIpPrefixSize: 32, ExpectErr: true, }, + "It should fail when PublicIPPrefixId is provided but ProvisionPublicIps is false": { + GatewayNodepoolName: "test", + ProvisionPublicIps: false, + PublicIpPrefixId: "test", + ExpectErr: true, + }, + "It should pass when PublicIPPrefixId is provided and ProvisionPublicIps is true": { + GatewayNodepoolName: "test", + ProvisionPublicIps: true, + PublicIpPrefixId: "test", + }, } for name, test := range tests { @@ -79,11 +92,13 @@ func TestValidateSGC(t *testing.T) { sgw := &StaticGatewayConfiguration{ Spec: StaticGatewayConfigurationSpec{ GatewayNodepoolName: test.GatewayNodepoolName, - GatewayVMSSProfile: GatewayVMSSProfile{ - VMSSResourceGroup: test.VMSSResourceGroup, - VMSSName: test.VMSSName, + GatewayVmssProfile: GatewayVmssProfile{ + VmssResourceGroup: test.VmssResourceGroup, + VmssName: test.VmssName, PublicIpPrefixSize: test.PublicIpPrefixSize, }, + ProvisionPublicIps: test.ProvisionPublicIps, + PublicIpPrefixId: test.PublicIpPrefixId, }, } err := sgw.validateSGC() diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 99f052a5..f3c0681a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -100,7 +100,7 @@ func (in *GatewayLBConfigurationList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewayLBConfigurationSpec) DeepCopyInto(out *GatewayLBConfigurationSpec) { *out = *in - out.GatewayVMSSProfile = in.GatewayVMSSProfile + out.GatewayVmssProfile = in.GatewayVmssProfile } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayLBConfigurationSpec. @@ -308,7 +308,7 @@ func (in *GatewayVMConfigurationList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GatewayVMConfigurationSpec) DeepCopyInto(out *GatewayVMConfigurationSpec) { *out = *in - out.GatewayVMSSProfile = in.GatewayVMSSProfile + out.GatewayVmssProfile = in.GatewayVmssProfile } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayVMConfigurationSpec. @@ -337,16 +337,16 @@ func (in *GatewayVMConfigurationStatus) DeepCopy() *GatewayVMConfigurationStatus } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GatewayVMSSProfile) DeepCopyInto(out *GatewayVMSSProfile) { +func (in *GatewayVmssProfile) DeepCopyInto(out *GatewayVmssProfile) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayVMSSProfile. -func (in *GatewayVMSSProfile) DeepCopy() *GatewayVMSSProfile { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayVmssProfile. +func (in *GatewayVmssProfile) DeepCopy() *GatewayVmssProfile { if in == nil { return nil } - out := new(GatewayVMSSProfile) + out := new(GatewayVmssProfile) in.DeepCopyInto(out) return out } @@ -537,9 +537,9 @@ func (in *StaticGatewayConfigurationList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StaticGatewayConfigurationSpec) DeepCopyInto(out *StaticGatewayConfigurationSpec) { *out = *in - out.GatewayVMSSProfile = in.GatewayVMSSProfile - if in.ExcludeCIDRs != nil { - in, out := &in.ExcludeCIDRs, &out.ExcludeCIDRs + out.GatewayVmssProfile = in.GatewayVmssProfile + if in.ExcludeCidrs != nil { + in, out := &in.ExcludeCidrs, &out.ExcludeCidrs *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/cmd/kube-egress-cni/main.go b/cmd/kube-egress-cni/main.go index 9500382c..84812e61 100644 --- a/cmd/kube-egress-cni/main.go +++ b/cmd/kube-egress-cni/main.go @@ -194,8 +194,9 @@ func cmdAdd(args *skel.CmdArgs) error { } exceptionsCidrs := append(resp.GetExceptionCidrs(), config.ExcludedCIDRs...) + defaultToGateway := resp.GetDefaultRoute() == v1.DefaultRoute_DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY if os.Getenv("IS_UNIT_TEST_ENV") != "true" { - if err := routes.SetPodRoutes(consts.WireguardLinkName, exceptionsCidrs, "/proc/sys", result); err != nil { + if err := routes.SetPodRoutes(consts.WireguardLinkName, exceptionsCidrs, defaultToGateway, "/proc/sys", result); err != nil { return fmt.Errorf("failed to setup pod routes: %w", err) } } diff --git a/config/crd/bases/egressgateway.kubernetes.azure.com_gatewaylbconfigurations.yaml b/config/crd/bases/egressgateway.kubernetes.azure.com_gatewaylbconfigurations.yaml index ac8fb1c7..41559b3d 100644 --- a/config/crd/bases/egressgateway.kubernetes.azure.com_gatewaylbconfigurations.yaml +++ b/config/crd/bases/egressgateway.kubernetes.azure.com_gatewaylbconfigurations.yaml @@ -54,19 +54,25 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. type: string + required: + - provisionPublicIps type: object status: description: GatewayLBConfigurationStatus defines the observed state of GatewayLBConfiguration properties: - frontendIP: - description: Gateway IP for wireguard connection. + egressIpPrefix: + description: Egress IP Prefix CIDR used for this gateway configuration. type: string - publicIpPrefix: - description: Public IP Prefix CIDR used for this gateway configuration. + frontendIp: + description: Gateway IP for wireguard connection. type: string serverPort: description: Listening port of the gateway side wireguard daemon. diff --git a/config/crd/bases/egressgateway.kubernetes.azure.com_gatewayvmconfigurations.yaml b/config/crd/bases/egressgateway.kubernetes.azure.com_gatewayvmconfigurations.yaml index d5e56b04..7417f4bf 100644 --- a/config/crd/bases/egressgateway.kubernetes.azure.com_gatewayvmconfigurations.yaml +++ b/config/crd/bases/egressgateway.kubernetes.azure.com_gatewayvmconfigurations.yaml @@ -54,9 +54,15 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. type: string + required: + - provisionPublicIps type: object status: description: GatewayVMConfigurationStatus defines the observed state of diff --git a/config/crd/bases/egressgateway.kubernetes.azure.com_staticgatewayconfigurations.yaml b/config/crd/bases/egressgateway.kubernetes.azure.com_staticgatewayconfigurations.yaml index b92907bd..2682119c 100644 --- a/config/crd/bases/egressgateway.kubernetes.azure.com_staticgatewayconfigurations.yaml +++ b/config/crd/bases/egressgateway.kubernetes.azure.com_staticgatewayconfigurations.yaml @@ -36,8 +36,16 @@ spec: description: StaticGatewayConfigurationSpec defines the desired state of StaticGatewayConfiguration properties: - excludeCIDRs: - description: CIDRs to be excluded from outbound. + defaultRoute: + default: staticEgressGateway + description: Pod default route, should be either azureNetworking (pod's + eth0) or staticEgressGateway (default). + enum: + - azureNetworking + - staticEgressGateway + type: string + excludeCidrs: + description: CIDRs to be excluded from the default route. items: type: string type: array @@ -60,14 +68,24 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. + This can only be specified when provisionPublicIps is true. type: string + required: + - provisionPublicIps type: object status: description: StaticGatewayConfigurationStatus defines the observed state of StaticGatewayConfiguration properties: + egressIpPrefix: + description: Egress IP Prefix CIDR used for this gateway configuration. + type: string gatewayWireguardProfile: description: Gateway side wireguard profile. properties: @@ -112,7 +130,7 @@ spec: wireguardPublicKey: description: Gateway side wireguard public key. type: string - wireguardServerIP: + wireguardServerIp: description: Gateway IP for wireguard connection. type: string wireguardServerPort: @@ -120,9 +138,6 @@ spec: format: int32 type: integer type: object - publicIpPrefix: - description: Public IP Prefix CIDR used for this gateway configuration. - type: string type: object type: object served: true diff --git a/controllers/cnimanager/server.go b/controllers/cnimanager/server.go index 2cd56e50..639bc253 100644 --- a/controllers/cnimanager/server.go +++ b/controllers/cnimanager/server.go @@ -59,7 +59,7 @@ func (s *NicService) NicAdd(ctx context.Context, in *cniprotocol.NicAddRequest) if err := s.k8sClient.Get(ctx, client.ObjectKey{Name: in.GetGatewayName(), Namespace: in.GetPodConfig().GetPodNamespace()}, gwConfig); err != nil { return nil, status.Errorf(codes.Unknown, "failed to retrieve StaticGatewayConfiguration %s/%s: %s", in.GetPodConfig().GetPodNamespace(), in.GetGatewayName(), err) } - if len(gwConfig.Status.WireguardServerIP) == 0 { + if len(gwConfig.Status.WireguardServerIp) == 0 { return nil, status.Errorf(codes.FailedPrecondition, "the gateway is not ready yet.") } pod := &corev1.Pod{} @@ -78,11 +78,17 @@ func (s *NicService) NicAdd(ctx context.Context, in *cniprotocol.NicAddRequest) }); err != nil { return nil, status.Errorf(codes.Unknown, "failed to update PodWireguardEndpoint %s/%s: %s", in.GetPodConfig().GetPodNamespace(), in.GetPodConfig().GetPodName(), err) } + + defaultRoute := cniprotocol.DefaultRoute_DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY + if gwConfig.Spec.DefaultRoute == current.RouteAzureNetworking { + defaultRoute = cniprotocol.DefaultRoute_DEFAULT_ROUTE_AZURE_NETWORKING + } return &cniprotocol.NicAddResponse{ - EndpointIp: gwConfig.Status.WireguardServerIP, + EndpointIp: gwConfig.Status.WireguardServerIp, ListenPort: gwConfig.Status.WireguardServerPort, PublicKey: gwConfig.Status.WireguardPublicKey, - ExceptionCidrs: gwConfig.Spec.ExcludeCIDRs, + ExceptionCidrs: gwConfig.Spec.ExcludeCidrs, + DefaultRoute: defaultRoute, }, nil } diff --git a/controllers/cnimanager/server_test.go b/controllers/cnimanager/server_test.go index df64584f..b48aed27 100644 --- a/controllers/cnimanager/server_test.go +++ b/controllers/cnimanager/server_test.go @@ -62,7 +62,7 @@ var _ = Describe("Server", func() { }, Status: current.StaticGatewayConfigurationStatus{ GatewayWireguardProfile: current.GatewayWireguardProfile{ - WireguardServerIP: "192.168.1.1/32", + WireguardServerIp: "192.168.1.1/32", WireguardPublicKey: "somerandompublickey", WireguardServerPort: 54321, }, @@ -134,8 +134,9 @@ var _ = Describe("Server", func() { resp, err := service.NicAdd(context.Background(), nicAddInputRequest) Expect(err).NotTo(HaveOccurred()) Expect(resp.PublicKey).To(Equal(gatewayProfile.Status.GatewayWireguardProfile.WireguardPublicKey)) - Expect(resp.EndpointIp).To(Equal(gatewayProfile.Status.GatewayWireguardProfile.WireguardServerIP)) + Expect(resp.EndpointIp).To(Equal(gatewayProfile.Status.GatewayWireguardProfile.WireguardServerIp)) Expect(resp.ListenPort).To(Equal(gatewayProfile.Status.GatewayWireguardProfile.WireguardServerPort)) + Expect(resp.DefaultRoute).To(Equal(cniprotocol.DefaultRoute_DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY)) podEndpoint := ¤t.PodWireguardEndpoint{} err = fakeClient.Get(context.Background(), client.ObjectKey{ Name: nicAddInputRequest.PodConfig.PodName, @@ -147,6 +148,15 @@ var _ = Describe("Server", func() { Expect(podEndpoint.Spec.PodIpAddress).To(Equal(nicAddInputRequest.AllowedIp)) }) }) + When("gateway has azureNetworking as default route", func() { + It("should return default route as azureNetworking", func() { + gatewayProfile.Spec.DefaultRoute = current.RouteAzureNetworking + fakeClient.Update(context.Background(), gatewayProfile) //nolint:errcheck + resp, err := service.NicAdd(context.Background(), nicAddInputRequest) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.DefaultRoute).To(Equal(cniprotocol.DefaultRoute_DEFAULT_ROUTE_AZURE_NETWORKING)) + }) + }) When("gateway is not found", func() { It("should return error and don't create pod endpoint", func() { fakeClient.Delete(context.Background(), gatewayProfile) //nolint:errcheck diff --git a/controllers/daemon/podwireguardendpoint_controller_test.go b/controllers/daemon/podwireguardendpoint_controller_test.go index 7749147e..dd88b0c9 100644 --- a/controllers/daemon/podwireguardendpoint_controller_test.go +++ b/controllers/daemon/podwireguardendpoint_controller_test.go @@ -105,9 +105,9 @@ var _ = Describe("Daemon PodWireguardEndpoint controller unit tests", func() { UID: testUID, }, Spec: egressgatewayv1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: vmssRG, - VMSSName: vmssName, + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: vmssRG, + VmssName: vmssName, PublicIpPrefixSize: 31, }, }, @@ -511,9 +511,9 @@ var _ = Describe("Daemon PodWireguardEndpoint controller unit tests", func() { UID: "1234567891", }, Spec: egressgatewayv1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: vmssRG, - VMSSName: vmssName, + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: vmssRG, + VmssName: vmssName, PublicIpPrefixSize: 31, }, }, diff --git a/controllers/daemon/staticgatewayconfiguration_controller.go b/controllers/daemon/staticgatewayconfiguration_controller.go index 46254957..97fc1c88 100644 --- a/controllers/daemon/staticgatewayconfiguration_controller.go +++ b/controllers/daemon/staticgatewayconfiguration_controller.go @@ -181,7 +181,7 @@ func (r *StaticGatewayConfigurationReconciler) reconcile( } // add lb ip (if not exists) to eth0 - if err := r.reconcileIlbIPOnHost(ctx, gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, false /* deleting */); err != nil { + if err := r.reconcileIlbIPOnHost(ctx, gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, false /* deleting */); err != nil { return err } @@ -397,7 +397,7 @@ func (r *StaticGatewayConfigurationReconciler) getVMIP( func isReady(gwConfig *egressgatewayv1alpha1.StaticGatewayConfiguration) bool { wgProfile := gwConfig.Status.GatewayWireguardProfile - return gwConfig.Status.PublicIpPrefix != "" && wgProfile.WireguardServerIP != "" && + return gwConfig.Status.EgressIpPrefix != "" && wgProfile.WireguardServerIp != "" && wgProfile.WireguardServerPort != 0 && wgProfile.WireguardPublicKey != "" && wgProfile.WireguardPrivateKeySecretRef != nil } @@ -407,9 +407,9 @@ func applyToNode(gwConfig *egressgatewayv1alpha1.StaticGatewayConfiguration) boo name, ok := nodeTags[consts.AKSNodepoolTagKey] return ok && strings.EqualFold(name, gwConfig.Spec.GatewayNodepoolName) } else { - vmssProfile := gwConfig.Spec.GatewayVMSSProfile - return strings.EqualFold(vmssProfile.VMSSName, nodeMeta.Compute.VMScaleSetName) && - strings.EqualFold(vmssProfile.VMSSResourceGroup, nodeMeta.Compute.ResourceGroupName) + vmssProfile := gwConfig.Spec.GatewayVmssProfile + return strings.EqualFold(vmssProfile.VmssName, nodeMeta.Compute.VMScaleSetName) && + strings.EqualFold(vmssProfile.VmssResourceGroup, nodeMeta.Compute.ResourceGroupName) } } @@ -986,7 +986,7 @@ func (r *StaticGatewayConfigurationReconciler) updateGatewayNodeStatus( } func getGatewayNamespaceName(gwConfig *egressgatewayv1alpha1.StaticGatewayConfiguration) string { - return fmt.Sprintf("gw-%s-%s", string(gwConfig.GetUID()), strings.Replace(gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, ".", "_", -1)) + return fmt.Sprintf("gw-%s-%s", string(gwConfig.GetUID()), strings.Replace(gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, ".", "_", -1)) } func getVethHostLinkName(gwConfig *egressgatewayv1alpha1.StaticGatewayConfiguration) string { diff --git a/controllers/daemon/staticgatewayconfiguration_controller_test.go b/controllers/daemon/staticgatewayconfiguration_controller_test.go index eed06997..2929caf6 100644 --- a/controllers/daemon/staticgatewayconfiguration_controller_test.go +++ b/controllers/daemon/staticgatewayconfiguration_controller_test.go @@ -185,9 +185,9 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func UID: testUID, }, Spec: egressgatewayv1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: vmssRG, - VMSSName: vmssName, + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: vmssRG, + VmssName: vmssName, PublicIpPrefixSize: 31, }, }, @@ -234,7 +234,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func mnl.EXPECT().LinkByName("eth0").Return(eth0, nil) mnl.EXPECT().AddrList(eth0, nl.FAMILY_ALL).Return([]netlink.Addr{}, nil) mnl.EXPECT().AddrAdd(eth0, &netlink.Addr{IPNet: getIPNetWithActualIP(ilbIPCidr)}).Return(nil) - err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, false) + err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, false) Expect(err).To(BeNil()) }) @@ -243,7 +243,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func eth0 := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}} mnl.EXPECT().LinkByName("eth0").Return(eth0, nil) mnl.EXPECT().AddrList(eth0, nl.FAMILY_ALL).Return([]netlink.Addr{{IPNet: getIPNetWithActualIP(ilbIPCidr)}}, nil) - err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, false) + err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, false) Expect(err).To(BeNil()) }) @@ -253,7 +253,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func mnl.EXPECT().LinkByName("eth0").Return(eth0, nil) mnl.EXPECT().AddrList(eth0, nl.FAMILY_ALL).Return([]netlink.Addr{{IPNet: getIPNetWithActualIP(ilbIPCidr)}}, nil) mnl.EXPECT().AddrDel(eth0, &netlink.Addr{IPNet: getIPNetWithActualIP(ilbIPCidr)}).Return(nil) - err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, true) + err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, true) Expect(err).To(BeNil()) }) @@ -262,7 +262,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func eth0 := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Name: "eth0"}} mnl.EXPECT().LinkByName("eth0").Return(eth0, nil) mnl.EXPECT().AddrList(eth0, nl.FAMILY_ALL).Return([]netlink.Addr{}, nil) - err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIP, true) + err := r.reconcileIlbIPOnHost(context.TODO(), gwConfig.Status.GatewayWireguardProfile.WireguardServerIp, true) Expect(err).To(BeNil()) }) @@ -653,9 +653,9 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func UID: testUID, }, Spec: egressgatewayv1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: vmssRG, - VMSSName: vmssName, + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: vmssRG, + VmssName: vmssName, PublicIpPrefixSize: 31, }, }, @@ -793,9 +793,9 @@ func getMockAzureManager(ctrl *gomock.Controller) *azmanager.AzureManager { func getTestGwConfigStatus() egressgatewayv1alpha1.StaticGatewayConfigurationStatus { return egressgatewayv1alpha1.StaticGatewayConfigurationStatus{ - PublicIpPrefix: "1.2.3.4/31", + EgressIpPrefix: "1.2.3.4/31", GatewayWireguardProfile: egressgatewayv1alpha1.GatewayWireguardProfile{ - WireguardServerIP: ilbIP, + WireguardServerIp: ilbIP, WireguardServerPort: 6000, WireguardPublicKey: pubK, WireguardPrivateKeySecretRef: &corev1.ObjectReference{ diff --git a/controllers/manager/gatewaylbconfiguration_controller.go b/controllers/manager/gatewaylbconfiguration_controller.go index ff89ad39..7f86fba8 100644 --- a/controllers/manager/gatewaylbconfiguration_controller.go +++ b/controllers/manager/gatewaylbconfiguration_controller.go @@ -150,7 +150,7 @@ func (r *GatewayLBConfigurationReconciler) reconcile( if lbConfig.Status == nil { lbConfig.Status = &egressgatewayv1alpha1.GatewayLBConfigurationStatus{} } - lbConfig.Status.FrontendIP = ip + lbConfig.Status.FrontendIp = ip lbConfig.Status.ServerPort = port if !equality.Semantic.DeepEqual(existing, lbConfig) { @@ -162,14 +162,14 @@ func (r *GatewayLBConfigurationReconciler) reconcile( frontend, po, prefix := "nil", "nil", "nil" if lbConfig.Status != nil { - frontend = lbConfig.Status.FrontendIP + frontend = lbConfig.Status.FrontendIp po = fmt.Sprintf("%d", lbConfig.Status.ServerPort) - prefix = lbConfig.Status.PublicIpPrefix + prefix = lbConfig.Status.EgressIpPrefix } r.Recorder.Eventf(lbConfig, corev1.EventTypeNormal, "Reconciled", - "GatewayLBConfiguration updated with frontendIP(%s), port(%s), and pip prefix(%s)", + "GatewayLBConfiguration updated with frontendIP(%s), port(%s), and egress prefix(%s)", frontend, po, prefix, ) log.Info("GatewayLBConfiguration reconciled") @@ -267,7 +267,7 @@ func (r *GatewayLBConfigurationReconciler) getGatewayVMSS( } } } else { - vmss, err := r.GetVMSS(lbConfig.Spec.VMSSResourceGroup, lbConfig.Spec.VMSSName) + vmss, err := r.GetVMSS(lbConfig.Spec.VmssResourceGroup, lbConfig.Spec.VmssName) if err != nil { return nil, err } @@ -679,7 +679,8 @@ func (r *GatewayLBConfigurationReconciler) reconcileGatewayVMConfig( } if _, err := controllerutil.CreateOrPatch(ctx, r, vmConfig, func() error { vmConfig.Spec.GatewayNodepoolName = lbConfig.Spec.GatewayNodepoolName - vmConfig.Spec.GatewayVMSSProfile = lbConfig.Spec.GatewayVMSSProfile + vmConfig.Spec.GatewayVmssProfile = lbConfig.Spec.GatewayVmssProfile + vmConfig.Spec.ProvisionPublicIps = lbConfig.Spec.ProvisionPublicIps vmConfig.Spec.PublicIpPrefixId = lbConfig.Spec.PublicIpPrefixId return controllerutil.SetControllerReference(lbConfig, vmConfig, r.Client.Scheme()) }); err != nil { @@ -692,7 +693,7 @@ func (r *GatewayLBConfigurationReconciler) reconcileGatewayVMConfig( if lbConfig.Status == nil { lbConfig.Status = &egressgatewayv1alpha1.GatewayLBConfigurationStatus{} } - lbConfig.Status.PublicIpPrefix = vmConfig.Status.EgressIpPrefix + lbConfig.Status.EgressIpPrefix = vmConfig.Status.EgressIpPrefix } return nil diff --git a/controllers/manager/gatewaylbconfiguration_controller_test.go b/controllers/manager/gatewaylbconfiguration_controller_test.go index 365a223e..7165c834 100644 --- a/controllers/manager/gatewaylbconfiguration_controller_test.go +++ b/controllers/manager/gatewaylbconfiguration_controller_test.go @@ -110,11 +110,12 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { }, Spec: egressgatewayv1alpha1.GatewayLBConfigurationSpec{ GatewayNodepoolName: "testgw", - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: "vmssRG", - VMSSName: "vmss", + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: "vmssRG", + VmssName: "vmss", PublicIpPrefixSize: 31, }, + ProvisionPublicIps: true, }, } }) @@ -340,7 +341,7 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { mockVMSSClient.EXPECT().List(gomock.Any(), testRG).Return([]*compute.VirtualMachineScaleSet{vmss}, nil) _, reconcileErr = r.Reconcile(context.TODO(), req) Expect(reconcileErr).To(BeNil()) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) }) Context("reconcile lbRule, lbProbe and vmConfig", func() { @@ -368,9 +369,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.FrontendIP).To(Equal("10.0.0.4")) + Expect(foundLBConfig.Status.FrontendIp).To(Equal("10.0.0.4")) Expect(foundLBConfig.Status.ServerPort).To(Equal(int32(6000))) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) }) It("should not update LB when lb rule and probe are expected", func() { @@ -380,9 +381,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { res, reconcileErr = r.Reconcile(context.TODO(), req) Expect(reconcileErr).To(BeNil()) Expect(res).To(Equal(ctrl.Result{})) - Expect(foundLBConfig.Status.FrontendIP).To(Equal("10.0.0.4")) + Expect(foundLBConfig.Status.FrontendIp).To(Equal("10.0.0.4")) Expect(foundLBConfig.Status.ServerPort).To(Equal(int32(6000))) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) }) It("should drop incorrect lbRule and create new one", func() { @@ -400,9 +401,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.FrontendIP).To(Equal("10.0.0.4")) + Expect(foundLBConfig.Status.FrontendIp).To(Equal("10.0.0.4")) Expect(foundLBConfig.Status.ServerPort).To(Equal(int32(6000))) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) }) It("should drop incorrect lbProbe and create new one", func() { @@ -437,9 +438,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.FrontendIP).To(Equal("10.0.0.4")) + Expect(foundLBConfig.Status.FrontendIp).To(Equal("10.0.0.4")) Expect(foundLBConfig.Status.ServerPort).To(Equal(int32(6000))) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) } }) @@ -464,9 +465,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.FrontendIP).To(Equal("10.0.0.4")) + Expect(foundLBConfig.Status.FrontendIp).To(Equal("10.0.0.4")) Expect(foundLBConfig.Status.ServerPort).To(Equal(int32(6001))) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6001), and pip prefix()"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6001), and egress prefix()"}, recorder.Events) }) }) }) @@ -497,8 +498,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { Expect(err).To(BeNil()) Expect(foundVMConfig.Spec.GatewayNodepoolName).To(Equal(lbConfig.Spec.GatewayNodepoolName)) - Expect(foundVMConfig.Spec.GatewayVMSSProfile).To(Equal(lbConfig.Spec.GatewayVMSSProfile)) + Expect(foundVMConfig.Spec.GatewayVmssProfile).To(Equal(lbConfig.Spec.GatewayVmssProfile)) Expect(foundVMConfig.Spec.PublicIpPrefixId).To(Equal(lbConfig.Spec.PublicIpPrefixId)) + Expect(foundVMConfig.Spec.ProvisionPublicIps).To(Equal(lbConfig.Spec.ProvisionPublicIps)) existing := metav1.GetControllerOf(foundVMConfig) Expect(existing).NotTo(BeNil()) @@ -506,8 +508,8 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.PublicIpPrefix).To(BeEmpty()) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix()"}, recorder.Events) + Expect(foundLBConfig.Status.EgressIpPrefix).To(BeEmpty()) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix()"}, recorder.Events) }) It("should update status from existing vmConfig", func() { @@ -518,12 +520,13 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { }, Spec: egressgatewayv1alpha1.GatewayVMConfigurationSpec{ GatewayNodepoolName: "testgw", - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: "vmssRG", - VMSSName: "vmss", + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: "vmssRG", + VmssName: "vmss", PublicIpPrefixSize: 31, }, - PublicIpPrefixId: "testPipPrefix", + PublicIpPrefixId: "testPipPrefix", + ProvisionPublicIps: true, }, Status: &egressgatewayv1alpha1.GatewayVMConfigurationStatus{ EgressIpPrefix: "1.2.3.4/31", @@ -538,8 +541,8 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundLBConfig) Expect(getErr).To(BeNil()) - Expect(foundLBConfig.Status.PublicIpPrefix).To(Equal("1.2.3.4/31")) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix(1.2.3.4/31)"}, recorder.Events) + Expect(foundLBConfig.Status.EgressIpPrefix).To(Equal("1.2.3.4/31")) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix(1.2.3.4/31)"}, recorder.Events) }) It("should update existing vmConfig accordingly", func() { @@ -550,9 +553,9 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { }, Spec: egressgatewayv1alpha1.GatewayVMConfigurationSpec{ GatewayNodepoolName: "testgw1", - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: "vmssRG1", - VMSSName: "vmss1", + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: "vmssRG1", + VmssName: "vmss1", PublicIpPrefixSize: 30, }, PublicIpPrefixId: "testPipPrefix1", @@ -571,7 +574,8 @@ var _ = Describe("GatewayLBConfiguration controller unit tests", func() { getErr = getResource(cl, foundVMConfig) Expect(getErr).To(BeNil()) Expect(foundVMConfig.Spec.GatewayNodepoolName).To(Equal(lbConfig.Spec.GatewayNodepoolName)) - assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and pip prefix(1.2.3.4/31)"}, recorder.Events) + Expect(foundVMConfig.Spec.ProvisionPublicIps).To(Equal(lbConfig.Spec.ProvisionPublicIps)) + assertEqualEvents([]string{"Normal Reconciled GatewayLBConfiguration updated with frontendIP(10.0.0.4), port(6000), and egress prefix(1.2.3.4/31)"}, recorder.Events) }) }) diff --git a/controllers/manager/gatewayvmconfiguration_controller.go b/controllers/manager/gatewayvmconfiguration_controller.go index 239b6db6..99719a10 100644 --- a/controllers/manager/gatewayvmconfiguration_controller.go +++ b/controllers/manager/gatewayvmconfiguration_controller.go @@ -139,7 +139,8 @@ func (r *GatewayVMConfigurationReconciler) reconcile( return ctrl.Result{}, err } - if err := r.reconcileVMSS(ctx, vmConfig, vmss, ipPrefixID, true); err != nil { + var privateIPs []string + if privateIPs, err = r.reconcileVMSS(ctx, vmConfig, vmss, ipPrefixID, true); err != nil { log.Error(err, "failed to reconcile VMSS") return ctrl.Result{}, err } @@ -150,10 +151,15 @@ func (r *GatewayVMConfigurationReconciler) reconcile( return ctrl.Result{}, err } } + if vmConfig.Status == nil { vmConfig.Status = &egressgatewayv1alpha1.GatewayVMConfigurationStatus{} } - vmConfig.Status.EgressIpPrefix = ipPrefix + if vmConfig.Spec.ProvisionPublicIps { + vmConfig.Status.EgressIpPrefix = ipPrefix + } else { + vmConfig.Status.EgressIpPrefix = strings.Join(privateIPs, ",") + } if !equality.Semantic.DeepEqual(existing, vmConfig) { log.Info(fmt.Sprintf("Updating GatewayVMConfiguration %s/%s", vmConfig.Namespace, vmConfig.Name)) @@ -166,7 +172,7 @@ func (r *GatewayVMConfigurationReconciler) reconcile( if vmConfig.Status != nil && vmConfig.Status.EgressIpPrefix != "" { prefix = vmConfig.Status.EgressIpPrefix } - r.Recorder.Eventf(vmConfig, corev1.EventTypeNormal, "Reconciled", "GatewayVMConfiguration provisioned with pip prefix %s", prefix) + r.Recorder.Eventf(vmConfig, corev1.EventTypeNormal, "Reconciled", "GatewayVMConfiguration provisioned with egress prefix %s", prefix) log.Info("GatewayVMConfiguration reconciled") return ctrl.Result{}, nil } @@ -189,7 +195,7 @@ func (r *GatewayVMConfigurationReconciler) ensureDeleted( return ctrl.Result{}, err } - if err := r.reconcileVMSS(ctx, vmConfig, vmss, "", false); err != nil { + if _, err := r.reconcileVMSS(ctx, vmConfig, vmss, "", false); err != nil { log.Error(err, "failed to reconcile VMSS") return ctrl.Result{}, err } @@ -235,7 +241,7 @@ func (r *GatewayVMConfigurationReconciler) getGatewayVMSS( } } } else { - vmss, err := r.GetVMSS(vmConfig.Spec.VMSSResourceGroup, vmConfig.Spec.VMSSName) + vmss, err := r.GetVMSS(vmConfig.Spec.VmssResourceGroup, vmConfig.Spec.VmssName) if err != nil { return nil, 0, err } @@ -259,6 +265,13 @@ func (r *GatewayVMConfigurationReconciler) ensurePublicIPPrefix( vmConfig *egressgatewayv1alpha1.GatewayVMConfiguration, ) (string, string, bool, error) { log := log.FromContext(ctx) + + // no need to provision public ip prefix is only private egress is needed + if !vmConfig.Spec.ProvisionPublicIps { + // return isManaged as false so that previously created managed public ip prefix can be deleted + return "", "", false, nil + } + if vmConfig.Spec.PublicIpPrefixId != "" { // if there is public prefix ip specified, prioritize this one matches := publicIPPrefixRE.FindStringSubmatch(vmConfig.Spec.PublicIpPrefixId) @@ -349,21 +362,21 @@ func (r *GatewayVMConfigurationReconciler) reconcileVMSS( vmss *compute.VirtualMachineScaleSet, ipPrefixID string, wantIPConfig bool, -) error { +) ([]string, error) { log := log.FromContext(ctx) ipConfigName := managedSubresourceName(vmConfig) needUpdate := false if vmss.Properties == nil || vmss.Properties.VirtualMachineProfile == nil || vmss.Properties.VirtualMachineProfile.NetworkProfile == nil { - return fmt.Errorf("vmss has empty network profile") + return nil, fmt.Errorf("vmss has empty network profile") } lbBackendpoolID := r.GetLBBackendAddressPoolID(to.Val(vmss.Properties.UniqueID)) interfaces := vmss.Properties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations needUpdate, err := r.reconcileVMSSNetworkInterface(ctx, ipConfigName, ipPrefixID, to.Val(lbBackendpoolID), wantIPConfig, interfaces) if err != nil { - return fmt.Errorf("failed to reconcile vmss interface(%s): %w", to.Val(vmss.Name), err) + return nil, fmt.Errorf("failed to reconcile vmss interface(%s): %w", to.Val(vmss.Name), err) } if needUpdate { @@ -377,21 +390,27 @@ func (r *GatewayVMConfigurationReconciler) reconcileVMSS( }, } if _, err := r.CreateOrUpdateVMSS("", to.Val(vmss.Name), newVmss); err != nil { - return fmt.Errorf("failed to update vmss(%s): %w", to.Val(vmss.Name), err) + return nil, fmt.Errorf("failed to update vmss(%s): %w", to.Val(vmss.Name), err) } } // check and update VMSS instances + var privateIPs []string instances, err := r.ListVMSSInstances("", to.Val(vmss.Name)) if err != nil { - return fmt.Errorf("failed to get vm instances from vmss(%s): %w", to.Val(vmss.Name), err) + return nil, fmt.Errorf("failed to get vm instances from vmss(%s): %w", to.Val(vmss.Name), err) } for _, instance := range instances { - if err := r.reconcileVMSSVM(ctx, vmConfig, to.Val(vmss.Name), instance, ipPrefixID, to.Val(lbBackendpoolID), wantIPConfig); err != nil { - return err + privateIP, err := r.reconcileVMSSVM(ctx, vmConfig, to.Val(vmss.Name), instance, ipPrefixID, to.Val(lbBackendpoolID), wantIPConfig) + if err != nil { + return nil, err + } + if wantIPConfig && ipPrefixID == "" { + privateIPs = append(privateIPs, privateIP) } } - return nil + + return privateIPs, nil } func (r *GatewayVMConfigurationReconciler) reconcileVMSSVM( @@ -402,18 +421,18 @@ func (r *GatewayVMConfigurationReconciler) reconcileVMSSVM( ipPrefixID string, lbBackendpoolID string, wantIPConfig bool, -) error { +) (string, error) { log := log.FromContext(ctx) ipConfigName := managedSubresourceName(vmConfig) if vm.Properties == nil || vm.Properties.NetworkProfileConfiguration == nil { - return fmt.Errorf("vmss vm(%s) has empty network profile", to.Val(vm.InstanceID)) + return "", fmt.Errorf("vmss vm(%s) has empty network profile", to.Val(vm.InstanceID)) } interfaces := vm.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations needUpdate, err := r.reconcileVMSSNetworkInterface(ctx, ipConfigName, ipPrefixID, lbBackendpoolID, wantIPConfig, interfaces) if err != nil { - return fmt.Errorf("failed to reconcile vm interface(%s): %w", to.Val(vm.InstanceID), err) + return "", fmt.Errorf("failed to reconcile vm interface(%s): %w", to.Val(vm.InstanceID), err) } if needUpdate { log.Info("Updating vmss instance", "vmInstanceID", to.Val(vm.InstanceID)) @@ -425,10 +444,36 @@ func (r *GatewayVMConfigurationReconciler) reconcileVMSSVM( }, } if _, err := r.UpdateVMSSInstance("", vmssName, to.Val(vm.InstanceID), newVM); err != nil { - return fmt.Errorf("failed to update vmss instance(%s): %w", to.Val(vm.InstanceID), err) + return "", fmt.Errorf("failed to update vmss instance(%s): %w", to.Val(vm.InstanceID), err) } } - return nil + + privateIP := "" + if wantIPConfig && ipPrefixID == "" { + // to reduce arm api call, only get private IPs when ipConfig is created and no public ip prefix is specified + out: + for _, nic := range interfaces { + if nic.Properties != nil && to.Val(nic.Properties.Primary) { + vmNic, err := r.GetVMSSInterface("", vmssName, to.Val(vm.InstanceID), to.Val(nic.Name)) + if err != nil { + return "", fmt.Errorf("failed to get vmss(%s) instance(%s) nic(%s): %w", vmssName, to.Val(vm.InstanceID), to.Val(nic.Name), err) + } + if vmNic.Properties == nil || vmNic.Properties.IPConfigurations == nil { + return "", fmt.Errorf("vmss(%s) instance(%s) nic(%s) has empty ip configurations", vmssName, to.Val(vm.InstanceID), to.Val(nic.Name)) + } + for _, ipConfig := range vmNic.Properties.IPConfigurations { + if ipConfig != nil && ipConfig.Properties != nil && strings.EqualFold(to.Val(ipConfig.Name), ipConfigName) { + privateIP = to.Val(ipConfig.Properties.PrivateIPAddress) + break out + } + } + } + } + if privateIP == "" { + return "", fmt.Errorf("failed to find private IP from vmss(%s), instance(%s), ipConfig(%s)", vmssName, to.Val(vm.InstanceID), ipConfigName) + } + } + return privateIP, nil } func (r *GatewayVMConfigurationReconciler) reconcileVMSSNetworkInterface( @@ -536,19 +581,24 @@ func (r *GatewayVMConfigurationReconciler) getExpectedIPConfig( } } } + + var pipConfig *compute.VirtualMachineScaleSetPublicIPAddressConfiguration + if ipPrefixID != "" { + pipConfig = &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ + Name: to.Ptr(ipConfigName), + Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ + PublicIPPrefix: &compute.SubResource{ + ID: to.Ptr(ipPrefixID), + }, + }, + } + } return &compute.VirtualMachineScaleSetIPConfiguration{ Name: to.Ptr(ipConfigName), Properties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ - Primary: to.Ptr(false), - PrivateIPAddressVersion: to.Ptr(compute.IPVersionIPv4), - PublicIPAddressConfiguration: &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ - Name: to.Ptr(ipConfigName), - Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ - PublicIPPrefix: &compute.SubResource{ - ID: to.Ptr(ipPrefixID), - }, - }, - }, + Primary: to.Ptr(false), + PrivateIPAddressVersion: to.Ptr(compute.IPVersionIPv4), + PublicIPAddressConfiguration: pipConfig, Subnet: &compute.APIEntityReference{ ID: subnetID, }, diff --git a/controllers/manager/gatewayvmconfiguration_controller_test.go b/controllers/manager/gatewayvmconfiguration_controller_test.go index 86c6e4ae..836eb348 100644 --- a/controllers/manager/gatewayvmconfiguration_controller_test.go +++ b/controllers/manager/gatewayvmconfiguration_controller_test.go @@ -41,6 +41,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" + "sigs.k8s.io/cloud-provider-azure/pkg/azclient/interfaceclient/mock_interfaceclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/publicipprefixclient/mock_publicipprefixclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/virtualmachinescalesetclient/mock_virtualmachinescalesetclient" "sigs.k8s.io/cloud-provider-azure/pkg/azclient/virtualmachinescalesetvmclient/mock_virtualmachinescalesetvmclient" @@ -93,11 +94,12 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { }, Spec: egressgatewayv1alpha1.GatewayVMConfigurationSpec{ GatewayNodepoolName: "testgw", - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: vmssRG, - VMSSName: vmssName, + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: vmssRG, + VmssName: vmssName, PublicIpPrefixSize: 31, }, + ProvisionPublicIps: true, }, } }) @@ -223,6 +225,15 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { vmConfig.Spec.PublicIpPrefixId = "" }) + It("should return nil if public ip prefix is not required", func() { + vmConfig.Spec.ProvisionPublicIps = false + prefix, prefixID, isManaged, err := r.ensurePublicIPPrefix(context.TODO(), 31, vmConfig) + Expect(prefix).To(BeEmpty()) + Expect(prefixID).To(BeEmpty()) + Expect(isManaged).To(BeFalse()) + Expect(err).To(BeNil()) + }) + It("should return error if prefix ID provided is not valid", func() { vmConfig.Spec.PublicIpPrefixId = "/subscriptions/sub1" _, _, _, err := r.ensurePublicIPPrefix(context.TODO(), 31, vmConfig) @@ -530,7 +541,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { It("should return error if vmss does not have properties", func() { existingVMSS := &compute.VirtualMachineScaleSet{} - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(err).To(Equal(fmt.Errorf("vmss has empty network profile"))) }) @@ -544,7 +555,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { }, }, } - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(errors.Unwrap(err)).To(Equal(fmt.Errorf("vmss(vm) primary network interface not found"))) }) @@ -552,7 +563,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { existingVMSS := getEmptyVMSS() mockVMSSClient := az.VmssClient.(*mock_virtualmachinescalesetclient.MockInterface) mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testRG, vmssName, gomock.Any()).Return(nil, fmt.Errorf("failed")) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(errors.Unwrap(err)).To(Equal(fmt.Errorf("failed"))) }) @@ -563,7 +574,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testRG, vmssName, gomock.Any()).Return(expectedVMSS, nil) mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(nil, fmt.Errorf("failed")) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(errors.Unwrap(err)).To(Equal(fmt.Errorf("failed"))) }) @@ -575,7 +586,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) vms := []*compute.VirtualMachineScaleSetVM{&compute.VirtualMachineScaleSetVM{InstanceID: to.Ptr("0")}} mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(err).To(Equal(fmt.Errorf("vmss vm(0) has empty network profile"))) }) @@ -594,7 +605,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { }, }} mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(errors.Unwrap(err)).To(Equal(fmt.Errorf("vmss(vm) primary network interface not found"))) }) @@ -607,7 +618,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { vms := []*compute.VirtualMachineScaleSetVM{getEmptyVMSSVM()} mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) mockVMSSVMClient.EXPECT().Update(gomock.Any(), testRG, vmssName, "0", gomock.Any()).Return(nil, fmt.Errorf("failed")) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(errors.Unwrap(err)).To(Equal(fmt.Errorf("failed"))) }) @@ -631,7 +642,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { expectedVM.InstanceID = to.Ptr("0") return expectedVM, nil }) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(err).To(BeNil()) }) @@ -642,7 +653,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { vms := []*compute.VirtualMachineScaleSetVM{existingVM} mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(err).To(BeNil()) }) @@ -670,7 +681,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { expectedVM.InstanceID = to.Ptr("0") return expectedVM, nil }) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", true) Expect(err).To(BeNil()) }) @@ -694,7 +705,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { Expect(vm).To(Equal(to.Val(expectedVM))) return expectedVM, nil }) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", false) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", false) Expect(err).To(BeNil()) }) @@ -704,7 +715,95 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { vms := []*compute.VirtualMachineScaleSetVM{existingVM} mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) - err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", false) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "prefix", false) + Expect(err).To(BeNil()) + }) + + It("should configure vmss and vm without publicIPConfiguration and return vmss instace private IPs when ipPrefixID is empty", func() { + existingVMSS, expectedVMSS := getEmptyVMSS(), getConfiguredVMSSWithoutPublicIPConfig() + mockVMSSClient := az.VmssClient.(*mock_virtualmachinescalesetclient.MockInterface) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testRG, vmssName, gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName string, vmss compute.VirtualMachineScaleSet) (*compute.VirtualMachineScaleSet, error) { + Expect(vmss).To(Equal(to.Val(expectedVMSS))) + expectedVMSS.Name = to.Ptr(vmssName) + return expectedVMSS, nil + }) + mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) + vms := []*compute.VirtualMachineScaleSetVM{getEmptyVMSSVM()} + expectedVM := getConfiguredVMSSVMWithoutPublicIPConfig() + mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) + mockVMSSVMClient.EXPECT().Update(gomock.Any(), testRG, vmssName, "0", gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName, instanceID string, vm compute.VirtualMachineScaleSetVM) (*compute.VirtualMachineScaleSetVM, error) { + Expect(vm).To(Equal(to.Val(expectedVM))) + expectedVM.InstanceID = to.Ptr("0") + return expectedVM, nil + }) + mockInterfaceClient := az.InterfaceClient.(*mock_interfaceclient.MockInterface) + mockInterfaceClient.EXPECT().GetVirtualMachineScaleSetNetworkInterface(gomock.Any(), testRG, vmssName, "0", "nic", gomock.Any()).Return( + network.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse{ + Interface: getConfiguredVMSSVMInterface(), + }, nil) + privateIPs, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "", true) + Expect(len(privateIPs)).To(Equal(1)) + Expect(privateIPs[0]).To(Equal("10.0.0.6")) + Expect(err).To(BeNil()) + }) + + It("should remove vmss and vm publicIPConfiguration and return vmss instace private IPs when ipPrefixID is updated to be empty", func() { + existingVMSS, expectedVMSS := getConfiguredVMSSWithNameAndUID(), getConfiguredVMSSWithoutPublicIPConfig() + mockVMSSClient := az.VmssClient.(*mock_virtualmachinescalesetclient.MockInterface) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testRG, vmssName, gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName string, vmss compute.VirtualMachineScaleSet) (*compute.VirtualMachineScaleSet, error) { + Expect(vmss).To(Equal(to.Val(expectedVMSS))) + expectedVMSS.Name = to.Ptr(vmssName) + return expectedVMSS, nil + }) + mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) + existingVM := getConfiguredVMSSVM() + existingVM.InstanceID = to.Ptr("0") + vms := []*compute.VirtualMachineScaleSetVM{existingVM} + expectedVM := getConfiguredVMSSVMWithoutPublicIPConfig() + mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return(vms, nil) + mockVMSSVMClient.EXPECT().Update(gomock.Any(), testRG, vmssName, "0", gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName, instanceID string, vm compute.VirtualMachineScaleSetVM) (*compute.VirtualMachineScaleSetVM, error) { + Expect(vm).To(Equal(to.Val(expectedVM))) + expectedVM.InstanceID = to.Ptr("0") + return expectedVM, nil + }) + mockInterfaceClient := az.InterfaceClient.(*mock_interfaceclient.MockInterface) + mockInterfaceClient.EXPECT().GetVirtualMachineScaleSetNetworkInterface(gomock.Any(), testRG, vmssName, "0", "nic", gomock.Any()).Return( + network.InterfacesClientGetVirtualMachineScaleSetNetworkInterfaceResponse{ + Interface: getConfiguredVMSSVMInterface(), + }, nil) + privateIPs, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "", true) + Expect(len(privateIPs)).To(Equal(1)) + Expect(privateIPs[0]).To(Equal("10.0.0.6")) + Expect(err).To(BeNil()) + }) + + It("should remove vmss and vm secondary IPConfig without publicIPConfiguration when it should be deleted", func() { + existingVMSS, expectedVMSS := getConfiguredVMSSWithoutPublicIPConfig(), getEmptyVMSS() + existingVMSS.Name = to.Ptr(vmssName) + existingVMSS.Properties.UniqueID = to.Ptr(testVMSSUID) + expectedVMSS.Name = nil + expectedVMSS.Properties.UniqueID = nil + mockVMSSClient := az.VmssClient.(*mock_virtualmachinescalesetclient.MockInterface) + mockVMSSClient.EXPECT().CreateOrUpdate(gomock.Any(), testRG, vmssName, gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName string, vmss compute.VirtualMachineScaleSet) (*compute.VirtualMachineScaleSet, error) { + Expect(vmss).To(Equal(to.Val(expectedVMSS))) + return expectedVMSS, nil + }) + mockVMSSVMClient := az.VmssVMClient.(*mock_virtualmachinescalesetvmclient.MockInterface) + existingVM, expectedVM := getConfiguredVMSSVMWithoutPublicIPConfig(), getEmptyVMSSVM() + existingVM.InstanceID = to.Ptr("0") + expectedVM.InstanceID = nil + mockVMSSVMClient.EXPECT().List(gomock.Any(), testRG, vmssName).Return([]*compute.VirtualMachineScaleSetVM{existingVM}, nil) + mockVMSSVMClient.EXPECT().Update(gomock.Any(), testRG, vmssName, "0", gomock.Any()). + DoAndReturn(func(ctx context.Context, rg, vmssName, instanceID string, vm compute.VirtualMachineScaleSetVM) (*compute.VirtualMachineScaleSetVM, error) { + Expect(vm).To(Equal(to.Val(expectedVM))) + return expectedVM, nil + }) + _, err := r.reconcileVMSS(context.TODO(), vmConfig, existingVMSS, "", false) Expect(err).To(BeNil()) }) }) @@ -714,6 +813,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { az = getMockAzureManager(gomock.NewController(GinkgoT())) controllerutil.AddFinalizer(vmConfig, consts.VMConfigFinalizerName) vmConfig.Spec.PublicIpPrefixId = "/subscriptions/testSub/resourceGroups/rg/providers/Microsoft.Network/publicIPPrefixes/prefix" + vmConfig.Spec.ProvisionPublicIps = true cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithStatusSubresource(vmConfig).WithRuntimeObjects(vmConfig).Build() r = &GatewayVMConfigurationReconciler{Client: cl, AzureManager: az, Recorder: recorder} }) @@ -819,7 +919,7 @@ var _ = Describe("GatewayVMConfiguration controller unit tests", func() { getErr = getResource(cl, foundVMConfig) Expect(getErr).To(BeNil()) Expect(foundVMConfig.Status.EgressIpPrefix).To(Equal("1.2.3.4/31")) - assertEqualEvents([]string{"Normal Reconciled GatewayVMConfiguration provisioned with pip prefix 1.2.3.4/31"}, recorder.Events) + assertEqualEvents([]string{"Normal Reconciled GatewayVMConfiguration provisioned with egress prefix 1.2.3.4/31"}, recorder.Events) }) }) @@ -930,6 +1030,7 @@ func getEmptyVMSSVM() *compute.VirtualMachineScaleSetVM { NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ NetworkInterfaceConfigurations: []*compute.VirtualMachineScaleSetNetworkConfiguration{ { + Name: to.Ptr("nic"), Properties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ Primary: to.Ptr(true), IPConfigurations: []*compute.VirtualMachineScaleSetIPConfiguration{ @@ -958,7 +1059,7 @@ func getConfiguredVMSSWithNameAndUID() *compute.VirtualMachineScaleSet { return vmss } -func getConfiguredVMSS() *compute.VirtualMachineScaleSet { +func getConfiguredVMSSWithoutPublicIPConfig() *compute.VirtualMachineScaleSet { return &compute.VirtualMachineScaleSet{ Location: to.Ptr("location"), Properties: &compute.VirtualMachineScaleSetProperties{ @@ -985,15 +1086,7 @@ func getConfiguredVMSS() *compute.VirtualMachineScaleSet { Properties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ Primary: to.Ptr(false), PrivateIPAddressVersion: to.Ptr(compute.IPVersionIPv4), - PublicIPAddressConfiguration: &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ - Name: to.Ptr("testns_test"), - Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ - PublicIPPrefix: &compute.SubResource{ - ID: to.Ptr("prefix"), - }, - }, - }, - Subnet: &compute.APIEntityReference{ID: to.Ptr("subnet")}, + Subnet: &compute.APIEntityReference{ID: to.Ptr("subnet")}, }, }, }, @@ -1006,12 +1099,28 @@ func getConfiguredVMSS() *compute.VirtualMachineScaleSet { } } -func getConfiguredVMSSVM() *compute.VirtualMachineScaleSetVM { +func getConfiguredVMSS() *compute.VirtualMachineScaleSet { + vmss := getConfiguredVMSSWithoutPublicIPConfig() + vmss.Properties.VirtualMachineProfile.NetworkProfile.NetworkInterfaceConfigurations[0]. + Properties.IPConfigurations[1].Properties.PublicIPAddressConfiguration = + &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ + Name: to.Ptr("testns_test"), + Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ + PublicIPPrefix: &compute.SubResource{ + ID: to.Ptr("prefix"), + }, + }, + } + return vmss +} + +func getConfiguredVMSSVMWithoutPublicIPConfig() *compute.VirtualMachineScaleSetVM { return &compute.VirtualMachineScaleSetVM{ Properties: &compute.VirtualMachineScaleSetVMProperties{ NetworkProfileConfiguration: &compute.VirtualMachineScaleSetVMNetworkProfileConfiguration{ NetworkInterfaceConfigurations: []*compute.VirtualMachineScaleSetNetworkConfiguration{ { + Name: to.Ptr("nic"), Properties: &compute.VirtualMachineScaleSetNetworkConfigurationProperties{ Primary: to.Ptr(true), IPConfigurations: []*compute.VirtualMachineScaleSetIPConfiguration{ @@ -1031,15 +1140,7 @@ func getConfiguredVMSSVM() *compute.VirtualMachineScaleSetVM { Properties: &compute.VirtualMachineScaleSetIPConfigurationProperties{ Primary: to.Ptr(false), PrivateIPAddressVersion: to.Ptr(compute.IPVersionIPv4), - PublicIPAddressConfiguration: &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ - Name: to.Ptr("testns_test"), - Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ - PublicIPPrefix: &compute.SubResource{ - ID: to.Ptr("prefix"), - }, - }, - }, - Subnet: &compute.APIEntityReference{ID: to.Ptr("subnet")}, + Subnet: &compute.APIEntityReference{ID: to.Ptr("subnet")}, }, }, }, @@ -1050,3 +1151,31 @@ func getConfiguredVMSSVM() *compute.VirtualMachineScaleSetVM { }, } } + +func getConfiguredVMSSVM() *compute.VirtualMachineScaleSetVM { + vm := getConfiguredVMSSVMWithoutPublicIPConfig() + vm.Properties.NetworkProfileConfiguration.NetworkInterfaceConfigurations[0].Properties.IPConfigurations[1].Properties.PublicIPAddressConfiguration = &compute.VirtualMachineScaleSetPublicIPAddressConfiguration{ + Name: to.Ptr("testns_test"), + Properties: &compute.VirtualMachineScaleSetPublicIPAddressConfigurationProperties{ + PublicIPPrefix: &compute.SubResource{ + ID: to.Ptr("prefix"), + }, + }, + } + return vm +} + +func getConfiguredVMSSVMInterface() network.Interface { + return network.Interface{ + Properties: &network.InterfacePropertiesFormat{ + IPConfigurations: []*network.InterfaceIPConfiguration{ + { + Name: to.Ptr("testns_test"), + Properties: &network.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAddress: to.Ptr("10.0.0.6"), + }, + }, + }, + }, + } +} diff --git a/controllers/manager/staticgatewayconfiguration_controller.go b/controllers/manager/staticgatewayconfiguration_controller.go index f354c139..b8be438c 100644 --- a/controllers/manager/staticgatewayconfiguration_controller.go +++ b/controllers/manager/staticgatewayconfiguration_controller.go @@ -117,10 +117,10 @@ func (r *StaticGatewayConfigurationReconciler) reconcile( }) prefix := "nil" - if gwConfig.Status.PublicIpPrefix != "" { - prefix = gwConfig.Status.PublicIpPrefix + if gwConfig.Status.EgressIpPrefix != "" { + prefix = gwConfig.Status.EgressIpPrefix } - r.Recorder.Eventf(gwConfig, corev1.EventTypeNormal, "Reconciled", "StaticGatewayConfiguration provisioned with pip prefix %s", prefix) + r.Recorder.Eventf(gwConfig, corev1.EventTypeNormal, "Reconciled", "StaticGatewayConfiguration provisioned with egress prefix %s", prefix) log.Info("staticGatewayConfiguration reconciled") return err } @@ -229,7 +229,8 @@ func (r *StaticGatewayConfigurationReconciler) reconcileGatewayLBConfig( } if _, err := controllerutil.CreateOrPatch(ctx, r, lbConfig, func() error { lbConfig.Spec.GatewayNodepoolName = gwConfig.Spec.GatewayNodepoolName - lbConfig.Spec.GatewayVMSSProfile = gwConfig.Spec.GatewayVMSSProfile + lbConfig.Spec.GatewayVmssProfile = gwConfig.Spec.GatewayVmssProfile + lbConfig.Spec.ProvisionPublicIps = gwConfig.Spec.ProvisionPublicIps lbConfig.Spec.PublicIpPrefixId = gwConfig.Spec.PublicIpPrefixId return controllerutil.SetControllerReference(gwConfig, lbConfig, r.Client.Scheme()) }); err != nil { @@ -237,9 +238,9 @@ func (r *StaticGatewayConfigurationReconciler) reconcileGatewayLBConfig( return err } if lbConfig.DeletionTimestamp.IsZero() && lbConfig.Status != nil { - gwConfig.Status.WireguardServerIP = lbConfig.Status.FrontendIP + gwConfig.Status.WireguardServerIp = lbConfig.Status.FrontendIp gwConfig.Status.WireguardServerPort = lbConfig.Status.ServerPort - gwConfig.Status.PublicIpPrefix = lbConfig.Status.PublicIpPrefix + gwConfig.Status.EgressIpPrefix = lbConfig.Status.EgressIpPrefix } return nil diff --git a/controllers/manager/staticgatewayconfiguration_controller_test.go b/controllers/manager/staticgatewayconfiguration_controller_test.go index 7e1af0e6..f0d36f41 100644 --- a/controllers/manager/staticgatewayconfiguration_controller_test.go +++ b/controllers/manager/staticgatewayconfiguration_controller_test.go @@ -95,12 +95,13 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu }, Spec: egressgatewayv1alpha1.StaticGatewayConfigurationSpec{ GatewayNodepoolName: "testgw", - GatewayVMSSProfile: egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: "vmssRG", - VMSSName: "vmss", + GatewayVmssProfile: egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: "vmssRG", + VmssName: "vmss", PublicIpPrefixSize: 31, }, - PublicIpPrefixId: "testPipPrefix", + PublicIpPrefixId: "testPipPrefix", + ProvisionPublicIps: true, }, } Expect(k8sClient.Create(ctx, gwConfig)).ToNot(HaveOccurred()) @@ -150,7 +151,9 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu return k8sClient.Get(ctx, client.ObjectKeyFromObject(gwConfig), lbConfig) }, timeout, interval).ShouldNot(HaveOccurred()) Expect(lbConfig.Spec.GatewayNodepoolName).To(BeEquivalentTo(gwConfig.Spec.GatewayNodepoolName)) - Expect(lbConfig.Spec.GatewayVMSSProfile).To(BeEquivalentTo(gwConfig.Spec.GatewayVMSSProfile)) + Expect(lbConfig.Spec.GatewayVmssProfile).To(BeEquivalentTo(gwConfig.Spec.GatewayVmssProfile)) + Expect(lbConfig.Spec.ProvisionPublicIps).To(BeEquivalentTo(gwConfig.Spec.ProvisionPublicIps)) + Expect(lbConfig.Spec.PublicIpPrefixId).To(BeEquivalentTo(gwConfig.Spec.PublicIpPrefixId)) owner := metav1.GetControllerOf(lbConfig) Expect(owner).NotTo(BeNil()) @@ -160,7 +163,7 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu Eventually(func() error { return k8sClient.Get(ctx, client.ObjectKeyFromObject(gwConfig), updatedGWConfig) }, timeout, interval).ShouldNot(HaveOccurred()) - Expect(updatedGWConfig.Status.WireguardServerIP).To(BeEmpty()) + Expect(updatedGWConfig.Status.WireguardServerIp).To(BeEmpty()) Expect(updatedGWConfig.Status.WireguardServerPort).To(BeZero()) }) }) @@ -170,9 +173,9 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu lbConfig := &egressgatewayv1alpha1.GatewayLBConfiguration{} Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(gwConfig), lbConfig)).ToNot(HaveOccurred()) lbConfig.Status = &egressgatewayv1alpha1.GatewayLBConfigurationStatus{ - FrontendIP: "1.1.1.1", + FrontendIp: "1.1.1.1", ServerPort: 6000, - PublicIpPrefix: "1.2.3.4/31", + EgressIpPrefix: "1.2.3.4/31", } Expect(k8sClient.Status().Update(ctx, lbConfig)).ToNot(HaveOccurred()) }) @@ -184,9 +187,9 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu return nil, err } return map[string]interface{}{ - "ip": updatedGWConfig.Status.WireguardServerIP, + "ip": updatedGWConfig.Status.WireguardServerIp, "port": updatedGWConfig.Status.WireguardServerPort, - "prefix": updatedGWConfig.Status.PublicIpPrefix, + "prefix": updatedGWConfig.Status.EgressIpPrefix, }, nil }, timeout, interval).Should(BeEquivalentTo(map[string]interface{}{ "ip": "1.1.1.1", @@ -201,11 +204,13 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu updateGWConfig := &egressgatewayv1alpha1.StaticGatewayConfiguration{} Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(gwConfig), updateGWConfig)).ToNot(HaveOccurred()) updateGWConfig.Spec.GatewayNodepoolName = "testgw1" - updateGWConfig.Spec.GatewayVMSSProfile = egressgatewayv1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: "vmssRG1", - VMSSName: "vmss1", + updateGWConfig.Spec.GatewayVmssProfile = egressgatewayv1alpha1.GatewayVmssProfile{ + VmssResourceGroup: "vmssRG1", + VmssName: "vmss1", PublicIpPrefixSize: 30, } + updateGWConfig.Spec.ProvisionPublicIps = false + updateGWConfig.Spec.PublicIpPrefixId = "" Expect(k8sClient.Update(ctx, updateGWConfig)).ToNot(HaveOccurred()) }) @@ -216,16 +221,20 @@ var _ = Describe("StaticGatewayConfiguration controller in testenv", Ordered, fu return nil, err } return map[string]interface{}{ - "np": lbConfig.Spec.GatewayNodepoolName, - "rg": lbConfig.Spec.VMSSResourceGroup, - "vmss": lbConfig.Spec.VMSSName, - "size": lbConfig.Spec.PublicIpPrefixSize, + "np": lbConfig.Spec.GatewayNodepoolName, + "rg": lbConfig.Spec.VmssResourceGroup, + "vmss": lbConfig.Spec.VmssName, + "size": lbConfig.Spec.PublicIpPrefixSize, + "prefixId": lbConfig.Spec.PublicIpPrefixId, + "provision": lbConfig.Spec.ProvisionPublicIps, }, nil }, timeout, interval).Should(BeEquivalentTo(map[string]interface{}{ - "np": "testgw1", - "rg": "vmssRG1", - "vmss": "vmss1", - "size": int32(30), + "np": "testgw1", + "rg": "vmssRG1", + "vmss": "vmss1", + "size": int32(30), + "prefixId": "", + "provision": false, })) }) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 2895848d..3b00dee0 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -28,6 +28,7 @@ import ( "context" "net" "regexp" + "strings" network "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" . "github.com/onsi/ginkgo/v2" @@ -79,7 +80,7 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { testns = "" }) - It("should let pod egress from the egress gateway", func() { + It("should let pod egress from the egress gateway and not affect pod not using the gateway", func() { rg, vmss, _, prefixLen, err := utils.GetGatewayVmssProfile(k8sClient) Expect(err).NotTo(HaveOccurred()) @@ -90,11 +91,12 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Namespace: testns, }, Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, PublicIpPrefixSize: prefixLen, }, + ProvisionPublicIps: true, }, } err = utils.CreateK8sObject(sgw, k8sClient) @@ -114,43 +116,17 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { By("Checking pod egress IP belongs to egress gateway outbound IP range") _, ipNet, _ := net.ParseCIDR(pipPrefix) Expect(ipNet.Contains(net.ParseIP(podEgressIP))).To(BeTrue()) - }) - - It("should not affect pod not using egress gateway", func() { - rg, vmss, _, prefixLen, err := utils.GetGatewayVmssProfile(k8sClient) - Expect(err).NotTo(HaveOccurred()) - - By("Creating a StaticGatewayConfiguration") - sgw := &v1alpha1.StaticGatewayConfiguration{ - ObjectMeta: metav1.ObjectMeta{ - Name: "sgw1", - Namespace: testns, - }, - Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, - PublicIpPrefixSize: prefixLen, - }, - }, - } - err = utils.CreateK8sObject(sgw, k8sClient) - Expect(err).NotTo(HaveOccurred()) - pipPrefix, err := utils.WaitStaticGatewayProvision(sgw, k8sClient) - Expect(err).NotTo(HaveOccurred()) - utils.Logf("Got egress gateway prefix: %s", pipPrefix) By("Creating a test pod NOT using egress gateway") - pod := utils.CreateCurlPodManifest(testns, "", "ifconfig.me") - err = utils.CreateK8sObject(pod, k8sClient) + pod2 := utils.CreateCurlPodManifest(testns, "", "ifconfig.me") + err = utils.CreateK8sObject(pod2, k8sClient) Expect(err).NotTo(HaveOccurred()) - podEgressIP, err := utils.GetExpectedPodLog(pod, podLogClient, podIPRE) + podEgressIP2, err := utils.GetExpectedPodLog(pod2, podLogClient, podIPRE) Expect(err).NotTo(HaveOccurred()) - utils.Logf("Get pod egress IP: %s", podEgressIP) + utils.Logf("Get pod egress IP: %s", podEgressIP2) By("Checking pod egress IP DOES NOT belong to egress gateway outbound IP range") - _, ipNet, _ := net.ParseCIDR(pipPrefix) - Expect(ipNet.Contains(net.ParseIP(podEgressIP))).To(BeFalse()) + Expect(ipNet.Contains(net.ParseIP(podEgressIP2))).To(BeFalse()) }) It("should support BYO public ip prefix as gateway configuration", func() { @@ -187,12 +163,13 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Namespace: testns, }, Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, PublicIpPrefixSize: prefixLen, }, - PublicIpPrefixId: to.Val(prefix.ID), + ProvisionPublicIps: true, + PublicIpPrefixId: to.Val(prefix.ID), }, } err = utils.CreateK8sObject(sgw, k8sClient) @@ -217,11 +194,12 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Namespace: testns, }, Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, PublicIpPrefixSize: prefixLen, }, + ProvisionPublicIps: true, }, } err = utils.CreateK8sObject(sgw, k8sClient) @@ -267,12 +245,13 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Namespace: testns, }, Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, PublicIpPrefixSize: prefixLen, }, - ExcludeCIDRs: cidrs, + ExcludeCidrs: cidrs, + ProvisionPublicIps: true, }, } err = utils.CreateK8sObject(sgw, k8sClient) @@ -306,11 +285,12 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Namespace: testns, }, Spec: v1alpha1.StaticGatewayConfigurationSpec{ - GatewayVMSSProfile: v1alpha1.GatewayVMSSProfile{ - VMSSResourceGroup: rg, - VMSSName: vmss, + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, PublicIpPrefixSize: prefixLen, }, + ProvisionPublicIps: true, }, } utils.Logf("Creating gateway %s", name) @@ -336,6 +316,46 @@ var _ = Describe("Test staticGatewayConfiguration deployment", func() { Expect(ipNet.Contains(net.ParseIP(podEgressIP))).To(BeTrue()) } }) + + It("should allow default route as AzureNetworking and disabling public egress", func() { + rg, vmss, _, prefixLen, err := utils.GetGatewayVmssProfile(k8sClient) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a StaticGatewayConfiguration") + sgw := &v1alpha1.StaticGatewayConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sgw1", + Namespace: testns, + }, + Spec: v1alpha1.StaticGatewayConfigurationSpec{ + GatewayVmssProfile: v1alpha1.GatewayVmssProfile{ + VmssResourceGroup: rg, + VmssName: vmss, + PublicIpPrefixSize: prefixLen, + }, + ProvisionPublicIps: false, + DefaultRoute: "azureNetworking", + }, + } + err = utils.CreateK8sObject(sgw, k8sClient) + Expect(err).NotTo(HaveOccurred()) + egressPrefix, err := utils.WaitStaticGatewayProvision(sgw, k8sClient) + Expect(err).NotTo(HaveOccurred()) + egressIPs := strings.Split(egressPrefix, ",") + Expect(len(egressIPs)).To(BeNumerically(">", 1)) // should be equal to gateway node count + utils.Logf("Got egress gateway prefix: %v", egressIPs) + + By("Creating a test pod using the gateway") + pod := utils.CreateCurlPodManifest(testns, "sgw1", "ifconfig.me") + err = utils.CreateK8sObject(pod, k8sClient) + Expect(err).NotTo(HaveOccurred()) + podEgressIP, err := utils.GetExpectedPodLog(pod, podLogClient, podIPRE) + Expect(err).NotTo(HaveOccurred()) + utils.Logf("Get pod egress IP: %s", podEgressIP) + + By("Checking pod egress IP DOES NOT belong to egress gateway outbound IP range") + Expect(egressIPs).ShouldNot(ContainElement(podEgressIP)) + }) }) func genTestNamespace() string { diff --git a/e2e/utils/pod.go b/e2e/utils/pod.go index e1671839..6744beac 100644 --- a/e2e/utils/pod.go +++ b/e2e/utils/pod.go @@ -30,7 +30,6 @@ import ( "regexp" corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" @@ -48,18 +47,18 @@ func CreateCurlPodManifest(nsName, gwName, curlTarget string) *corev1.Pod { Namespace: nsName, Annotations: annotations, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ { Name: "app", Image: "registry.k8s.io/e2e-test-images/agnhost:2.36", - ImagePullPolicy: v1.PullIfNotPresent, + ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{ "/bin/sh", "-c", "curl -s -m 5 --retry-delay 60 --retry 10 " + curlTarget, }, }, }, - RestartPolicy: v1.RestartPolicyNever, + RestartPolicy: corev1.RestartPolicyNever, }, } } @@ -75,32 +74,32 @@ func CreateNginxPodManifest(nsName, gwName string) *corev1.Pod { Namespace: nsName, Annotations: annotations, }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ { Name: "app", Image: "nginx", - ImagePullPolicy: v1.PullIfNotPresent, - Ports: []v1.ContainerPort{{ContainerPort: 80}}, + ImagePullPolicy: corev1.PullIfNotPresent, + Ports: []corev1.ContainerPort{{ContainerPort: 80}}, }, }, - RestartPolicy: v1.RestartPolicyNever, + RestartPolicy: corev1.RestartPolicyNever, }, } } -func getPodLog(c clientset.Interface, name, namespace string, opts *v1.PodLogOptions) ([]byte, error) { +func getPodLog(c clientset.Interface, name, namespace string, opts *corev1.PodLogOptions) ([]byte, error) { return c.CoreV1().Pods(namespace).GetLogs(name, opts).Do(context.Background()).Raw() } -func printPodInfo(c clientset.Interface, pod *v1.Pod) { - log, _ := getPodLog(c, pod.Name, pod.Namespace, &v1.PodLogOptions{}) +func printPodInfo(c clientset.Interface, pod *corev1.Pod) { + log, _ := getPodLog(c, pod.Name, pod.Namespace, &corev1.PodLogOptions{}) Logf("Pod %s/%s log:\n%s", pod.Namespace, pod.Name, string(log)) - log, _ = getPodLog(c, pod.Name, pod.Namespace, &v1.PodLogOptions{Previous: true}) + log, _ = getPodLog(c, pod.Name, pod.Namespace, &corev1.PodLogOptions{Previous: true}) Logf("Pod %s/%s previous log:\n%s", pod.Namespace, pod.Name, string(log)) } -func GetExpectedPodLog(pod *v1.Pod, c clientset.Interface, expectLogRegex *regexp.Regexp) (string, error) { +func GetExpectedPodLog(pod *corev1.Pod, c clientset.Interface, expectLogRegex *regexp.Regexp) (string, error) { var log []byte err := wait.PollUntilContextTimeout(context.Background(), poll, pollTimeoutForProvision, true, func(ctx context.Context) (bool, error) { pod, err := c.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{}) @@ -110,9 +109,9 @@ func GetExpectedPodLog(pod *v1.Pod, c clientset.Interface, expectLogRegex *regex } return false, err } - if pod.Status.Phase != v1.PodSucceeded { + if pod.Status.Phase != corev1.PodSucceeded { Logf("Waiting for the pod to succeed, current status: %s", pod.Status.Phase) - if pod.Status.Phase == v1.PodFailed { + if pod.Status.Phase == corev1.PodFailed { printPodInfo(c, pod) return false, fmt.Errorf("test pod is in Failed phase") } @@ -122,7 +121,7 @@ func GetExpectedPodLog(pod *v1.Pod, c clientset.Interface, expectLogRegex *regex Logf("Waiting for the container to be completed") return false, nil } - log, err = getPodLog(c, pod.Name, pod.Namespace, &v1.PodLogOptions{}) + log, err = getPodLog(c, pod.Name, pod.Namespace, &corev1.PodLogOptions{}) if err != nil { Logf("Got %v when retrieving test pod log, retrying", err) return false, nil @@ -136,7 +135,7 @@ func GetExpectedPodLog(pod *v1.Pod, c clientset.Interface, expectLogRegex *regex return found, nil } -func WaitGetPodIP(pod *v1.Pod, c clientset.Interface) (string, error) { +func WaitGetPodIP(pod *corev1.Pod, c clientset.Interface) (string, error) { var podIP string err := wait.PollUntilContextTimeout(context.Background(), poll, pollTimeout, true, func(ctx context.Context) (bool, error) { pod, err := c.CoreV1().Pods(pod.Namespace).Get(context.Background(), pod.Name, metav1.GetOptions{}) @@ -146,12 +145,12 @@ func WaitGetPodIP(pod *v1.Pod, c clientset.Interface) (string, error) { } return false, err } - if pod.Status.Phase == v1.PodRunning { + if pod.Status.Phase == corev1.PodRunning { podIP = pod.Status.PodIP return true, nil } Logf("Waiting for the pod to Running, current status: %s", pod.Status.Phase) - if pod.Status.Phase == v1.PodFailed { + if pod.Status.Phase == corev1.PodFailed { printPodInfo(c, pod) return false, fmt.Errorf("test pod is in Failed phase") } diff --git a/e2e/utils/utils.go b/e2e/utils/utils.go index 7267ac4d..d1824efe 100644 --- a/e2e/utils/utils.go +++ b/e2e/utils/utils.go @@ -47,7 +47,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/config" "github.com/Azure/kube-egress-gateway/api/v1alpha1" - current "github.com/Azure/kube-egress-gateway/api/v1alpha1" "github.com/Azure/kube-egress-gateway/pkg/azureclients" ) @@ -65,7 +64,7 @@ var ( func CreateK8sClient() (k8sClient client.Client, podLogClient clientset.Interface, err error) { apischeme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(apischeme)) - utilruntime.Must(current.AddToScheme(apischeme)) + utilruntime.Must(v1alpha1.AddToScheme(apischeme)) restConfig := config.GetConfigOrDie() k8sClient, err = client.New(restConfig, client.Options{Scheme: apischeme}) if err != nil { @@ -155,8 +154,8 @@ func WaitStaticGatewayProvision(sgw *v1alpha1.StaticGatewayConfiguration, c clie } return false, err } - if len(sgw.Status.PublicIpPrefix) > 0 { - pipPrefix = sgw.Status.PublicIpPrefix + if len(sgw.Status.EgressIpPrefix) > 0 { + pipPrefix = sgw.Status.EgressIpPrefix return true, nil } return false, nil diff --git a/helm/kube-egress-gateway/Chart.lock b/helm/kube-egress-gateway/Chart.lock deleted file mode 100644 index b33b0a9f..00000000 --- a/helm/kube-egress-gateway/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: cert-manager - repository: https://charts.jetstack.io - version: v1.12.0 -digest: sha256:53d7c67c6ffe1c0bd5f85483d855353541d829127716ffc07b385b43c46e8e06 -generated: "2023-08-15T06:10:48.278515335Z" diff --git a/helm/kube-egress-gateway/Chart.yaml b/helm/kube-egress-gateway/Chart.yaml index bb5e586d..db7581c6 100644 --- a/helm/kube-egress-gateway/Chart.yaml +++ b/helm/kube-egress-gateway/Chart.yaml @@ -10,10 +10,3 @@ maintainers: sources: - https://github.com/Azure/kube-egress-gateway version: 0.1.0 - -dependencies: - - name: cert-manager - version: v1.12.0 - repository: https://charts.jetstack.io - alias: cert-manager - condition: cert-manager.enabled diff --git a/helm/kube-egress-gateway/README.md b/helm/kube-egress-gateway/README.md index 5874d58b..cf552ebd 100644 --- a/helm/kube-egress-gateway/README.md +++ b/helm/kube-egress-gateway/README.md @@ -11,13 +11,7 @@ Clone this repository, kube-egress-gateway chart is maintained in `helm/kube-egr git clone https://github.com/Azure/kube-egress-gateway.git ``` -The `kube-egress-gateway` project relies on [cert-manager](https://cert-manager.io/) to provide webhook certificate issuance and rotation. You can install it separately (recommended) or install it as a subchart included in this Helm chart. - -### Option 1 (RECOMMENDED) - -We recommend users to install cert-manager before installing this chart. - -To manually install cert-manager, you can follow [cert-manager official doc](https://cert-manager.io/docs/installation/helm/#option-1-installing-crds-with-kubectl). +The `kube-egress-gateway` project relies on [cert-manager](https://cert-manager.io/) to provide webhook certificate issuance and rotation. You can install cert-manager following their [installation guide](https://cert-manager.io/docs/installation/). Then to install `kube-egress-gateway`, you may run below `helm` command: @@ -33,22 +27,6 @@ $ helm install \ See more details about azure_config.yaml in [azure cloud configurations](#azure-cloud-configurations) -### Option 2 - -You may also install cert-manager as a subchart with `enabled` set to `true`: - -```bash -$ helm install \ - kube-egress-gateway ./helm/kube-egress-gateway \ - --namespace kube-egress-gateway-system \ - --create-namespace \ - --set common.imageRepository=mcr.microsoft.com/oss/kubernetes \ - --set common.imageTag=v1.0.0 \ - --set cert-manager.enabled=true \ # to enable cert-manager installation - --set cert-manager.namespace= \ # to install cert-manager components in specified namespace, default to cert-manager - -f azure_config.yaml -``` - # Configurable values Below we provide a list of configurations you can set when invoking `helm install` against your Kubernetes cluster. You can also check [values.yaml](./values.yaml). @@ -150,7 +128,3 @@ The Helm chart installs 5 components with different images: gateway-controller-m | `gatewayCNIIpam.imageName` | `kube-egress-gateway-cni-ipam` | Name of gatewayCNI-Ipam image. | | `gatewayCNIIpam.imageTag` | | Tag of gatewayCNI-Ipam image. | | `gatewayCNIIpam.imagePullPolicy` | `IfNotPresent` | Image pull policy for gatewayCNI-Ipam's image. | - -## cert-manager configurations - -`kube-egress-gateway` relies on `cert-manager` to provide webhook certificate issuance and renewal. For configurations to cert-manager itself, please refer to [cert-manager official website](https://cert-manager.io/docs/installation/helm/). By default cert-manager subchart is disabled. To enable, add `--set cert-manager.enabled=true` in your `helm install` command. To deploy cert-manager components in a specific namespace, add `--set cert-manager.namespace=` in your `helm install` command. The default namespace is `cert-manager`. diff --git a/helm/kube-egress-gateway/charts/cert-manager-v1.12.0.tgz b/helm/kube-egress-gateway/charts/cert-manager-v1.12.0.tgz deleted file mode 100644 index 6f261eab..00000000 Binary files a/helm/kube-egress-gateway/charts/cert-manager-v1.12.0.tgz and /dev/null differ diff --git a/helm/kube-egress-gateway/crds/crds.yaml b/helm/kube-egress-gateway/crds/crds.yaml index 2225cfd9..ae408d61 100644 --- a/helm/kube-egress-gateway/crds/crds.yaml +++ b/helm/kube-egress-gateway/crds/crds.yaml @@ -37,8 +37,16 @@ spec: spec: description: StaticGatewayConfigurationSpec defines the desired state of StaticGatewayConfiguration properties: - excludeCIDRs: - description: CIDRs to be excluded from outbound. + defaultRoute: + default: staticEgressGateway + description: Pod default route, should be either azureNetworking (pod's + eth0) or staticEgressGateway (default). + enum: + - azureNetworking + - staticEgressGateway + type: string + excludeCidrs: + description: CIDRs to be excluded from the default route. items: type: string type: array @@ -61,13 +69,23 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. + This can only be specified when provisionPublicIps is true. type: string + required: + - provisionPublicIps type: object status: description: StaticGatewayConfigurationStatus defines the observed state of StaticGatewayConfiguration properties: + egressIpPrefix: + description: Egress IP Prefix CIDR used for this gateway configuration. + type: string gatewayWireguardProfile: description: Gateway side wireguard profile. properties: @@ -100,7 +118,7 @@ spec: wireguardPublicKey: description: Gateway side wireguard public key. type: string - wireguardServerIP: + wireguardServerIp: description: Gateway IP for wireguard connection. type: string wireguardServerPort: @@ -108,9 +126,6 @@ spec: format: int32 type: integer type: object - publicIpPrefix: - description: Public IP Prefix CIDR used for this gateway configuration. - type: string type: object type: object served: true @@ -166,18 +181,24 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. type: string + required: + - provisionPublicIps type: object status: description: GatewayLBConfigurationStatus defines the observed state of GatewayLBConfiguration properties: - frontendIP: - description: Gateway IP for wireguard connection. + egressIpPrefix: + description: Egress IP Prefix CIDR used for this gateway configuration. type: string - publicIpPrefix: - description: Public IP Prefix CIDR used for this gateway configuration. + frontendIp: + description: Gateway IP for wireguard connection. type: string serverPort: description: Listening port of the gateway side wireguard daemon. @@ -238,9 +259,15 @@ spec: description: Resource group of the VMSS. Must be in the same subscription. type: string type: object + provisionPublicIps: + default: true + description: Whether to provision public IP prefixes for outbound. + type: boolean publicIpPrefixId: description: BYO Resource ID of public IP prefix to be used as outbound. type: string + required: + - provisionPublicIps type: object status: description: GatewayVMConfigurationStatus defines the observed state of GatewayVMConfiguration diff --git a/helm/kube-egress-gateway/values.yaml b/helm/kube-egress-gateway/values.yaml index 20d4ac8e..0998cd56 100644 --- a/helm/kube-egress-gateway/values.yaml +++ b/helm/kube-egress-gateway/values.yaml @@ -1,7 +1,3 @@ -cert-manager: - enabled: false - namespace: "cert-manager" - common: imageRepository: "local" imageTag: "test" diff --git a/helm/repo/index.yaml b/helm/repo/index.yaml index 29657dcc..705a925b 100644 --- a/helm/repo/index.yaml +++ b/helm/repo/index.yaml @@ -2,15 +2,9 @@ apiVersion: v1 entries: kube-egress-gateway: - apiVersion: v2 - created: "2023-08-15T09:12:41.391920884Z" - dependencies: - - alias: cert-manager - condition: cert-manager.enabled - name: cert-manager - repository: https://charts.jetstack.io - version: v1.12.0 + created: "2023-10-09T05:42:02.274197002Z" description: A Helm chart for installing Azure kube-egress-gateway components - digest: ce119979bfe71cfb4770240b978499a766961d46a2096ae6e8a9e896d466fd87 + digest: b1682df4e086cd8941549bdd568bef228d07917253b50c25150a1295dc2fdea2 maintainers: - email: jwtty0919@gmail.com name: Wantong Jiang @@ -23,4 +17,4 @@ entries: urls: - kube-egress-gateway-0.1.0.tgz version: 0.1.0 -generated: "2023-08-15T09:12:41.38810042Z" +generated: "2023-10-09T05:42:02.273566265Z" diff --git a/helm/repo/kube-egress-gateway-0.1.0.tgz b/helm/repo/kube-egress-gateway-0.1.0.tgz index 4b9d4a85..2aaadb51 100644 Binary files a/helm/repo/kube-egress-gateway-0.1.0.tgz and b/helm/repo/kube-egress-gateway-0.1.0.tgz differ diff --git a/pkg/cni/routes/routes.go b/pkg/cni/routes/routes.go index 3aa014dd..44d2d34d 100644 --- a/pkg/cni/routes/routes.go +++ b/pkg/cni/routes/routes.go @@ -56,16 +56,17 @@ func init() { } } -func SetPodRoutes(ifName string, exceptionCidrs []string, sysctlDir string, result *current.Result) error { - // 1. removes existing routes - // 2. add original default route gateway to eth0 - // 3. routes exceptional cidrs (traffic avoiding gateway) to base interface (eth0) - // 4. add default route to wireguard interface +func SetPodRoutes(ifName string, exceptionCidrs []string, defaultToGateway bool, sysctlDir string, result *current.Result) error { eth0Link, err := routesRunner.netlink.LinkByName("eth0") if err != nil { return fmt.Errorf("failed to retrieve eth0 interface: %w", err) } + wgLink, err := routesRunner.netlink.LinkByName(ifName) + if err != nil { + return fmt.Errorf("failed to retrieve wireguard interface: %w", err) + } + routes, err := routesRunner.netlink.RouteList(eth0Link, nl.FAMILY_ALL) if err != nil { return fmt.Errorf("failed to list all routes on eth0: %w", err) @@ -77,67 +78,82 @@ func SetPodRoutes(ifName string, exceptionCidrs []string, sysctlDir string, resu if route.Dst == nil && route.Family == nl.FAMILY_V4 { defaultRoute = &route } - if err := routesRunner.netlink.RouteDel(&route); err != nil { - return fmt.Errorf("failed to delete route (%s): %w", route, err) - } } - result.Routes = nil - if defaultRoute == nil { return errors.New("failed to find default route") } - gatewayDestination := net.IPNet{IP: defaultRoute.Gw, Mask: net.CIDRMask(32, 32)} - err = routesRunner.netlink.RouteReplace(&netlink.Route{ - Dst: &gatewayDestination, + eth0RouteTmpl := netlink.Route{ + Gw: defaultRoute.Gw, LinkIndex: eth0Link.Attrs().Index, - Scope: netlink.SCOPE_LINK, - }) - if err != nil { - return fmt.Errorf("failed to add original gateway route: %w", err) + Protocol: unix.RTPROT_STATIC, + } + + wgRouteTmpl := netlink.Route{ + Gw: nil, + Via: &netlink.Via{ + Addr: net.ParseIP("fe80::1"), + AddrFamily: nl.FAMILY_V6, + }, + LinkIndex: wgLink.Attrs().Index, + Scope: netlink.SCOPE_UNIVERSE, + Family: nl.FAMILY_V4, + } + + if defaultToGateway { + // 1. removes existing routes + // 2. add original default route gateway to eth0 + // 3. routes exceptional cidrs (traffic avoiding gateway) to base interface (eth0) + // 4. add default route to wireguard interface + for _, route := range routes { + if err := routesRunner.netlink.RouteDel(&route); err != nil { + return fmt.Errorf("failed to delete route (%s): %w", route, err) + } + } + result.Routes = nil + + gatewayDestination := net.IPNet{IP: defaultRoute.Gw, Mask: net.CIDRMask(32, 32)} + err = routesRunner.netlink.RouteReplace(&netlink.Route{ + Dst: &gatewayDestination, + LinkIndex: eth0Link.Attrs().Index, + Scope: netlink.SCOPE_LINK, + }) + if err != nil { + return fmt.Errorf("failed to add original gateway route: %w", err) + } + result.Routes = append(result.Routes, &types.Route{Dst: gatewayDestination}) + + _, defaultRouteCidr, _ := net.ParseCIDR("0.0.0.0/0") + wgDefaultRoute := wgRouteTmpl + wgDefaultRoute.Dst = defaultRouteCidr + result.Routes = append(result.Routes, &types.Route{Dst: *defaultRouteCidr, GW: net.ParseIP("fe80::1")}) + + err = routesRunner.netlink.RouteReplace(&wgDefaultRoute) + if err != nil { + return fmt.Errorf("failed to add default wireguard route (%s): %w", wgDefaultRoute, err) + } } - result.Routes = append(result.Routes, &types.Route{Dst: gatewayDestination}) for _, exception := range exceptionCidrs { _, cidr, err := net.ParseCIDR(exception) if err != nil { return fmt.Errorf("failed to parse cidr (%s): %w", exception, err) } - gatewayRoute := netlink.Route{ - Dst: cidr, - Gw: defaultRoute.Gw, - LinkIndex: eth0Link.Attrs().Index, - Protocol: unix.RTPROT_STATIC, + var gatewayRoute netlink.Route + var gwIP net.IP + if defaultToGateway { + gatewayRoute = eth0RouteTmpl + gwIP = defaultRoute.Gw + } else { + gatewayRoute = wgRouteTmpl + gwIP = net.ParseIP("fe80::1") } + gatewayRoute.Dst = cidr err = routesRunner.netlink.RouteReplace(&gatewayRoute) if err != nil { return fmt.Errorf("failed to add route (%s): %w", gatewayRoute, err) } - result.Routes = append(result.Routes, &types.Route{Dst: *cidr, GW: defaultRoute.Gw}) - } - - wgLink, err := routesRunner.netlink.LinkByName(ifName) - if err != nil { - return fmt.Errorf("failed to retrieve wireguard interface: %w", err) - } - - _, defaultRouteCidr, _ := net.ParseCIDR("0.0.0.0/0") - wgDefaultRoute := netlink.Route{ - Dst: defaultRouteCidr, - Gw: nil, - Via: &netlink.Via{ - Addr: net.ParseIP("fe80::1"), - AddrFamily: nl.FAMILY_V6, - }, - LinkIndex: wgLink.Attrs().Index, - Scope: netlink.SCOPE_UNIVERSE, - Family: nl.FAMILY_V4, - } - result.Routes = append(result.Routes, &types.Route{Dst: *defaultRouteCidr, GW: net.ParseIP("fe80::1")}) - - err = routesRunner.netlink.RouteReplace(&wgDefaultRoute) - if err != nil { - return fmt.Errorf("failed to add default wireguard route (%s): %w", wgDefaultRoute, err) + result.Routes = append(result.Routes, &types.Route{Dst: *cidr, GW: gwIP}) } err = addRoutingForIngress(eth0Link, *defaultRoute, sysctlDir) diff --git a/pkg/cni/routes/routes_test.go b/pkg/cni/routes/routes_test.go index 25674da9..50b7acd9 100644 --- a/pkg/cni/routes/routes_test.go +++ b/pkg/cni/routes/routes_test.go @@ -86,101 +86,164 @@ func TestSetPodRoutes(t *testing.T) { LinkIndex: 1, Table: 8738, } - expectedRouteResult := []*types.Route{ - {Dst: net.IPNet{IP: defaultGw, Mask: net.CIDRMask(32, 32)}}, - {Dst: *net1, GW: defaultGw}, - {Dst: *net2, GW: defaultGw}, - {Dst: *dnet, GW: net.ParseIP("fe80::1")}, - } - gomock.InOrder( - // retrieve eth0 link - mnl.EXPECT().LinkByName("eth0").Return(eth0, nil), - // get existing routes - mnl.EXPECT().RouteList(eth0, netlink.FAMILY_ALL).Return(existingRoutes, nil), - // delete existing routes - mnl.EXPECT().RouteDel(&existingRoutes[0]).Return(nil), - mnl.EXPECT().RouteDel(&existingRoutes[1]).Return(nil), - // add route to default gateway via eth0 - mnl.EXPECT().RouteReplace(&netlink.Route{ - Dst: &net.IPNet{IP: defaultGw, Mask: net.CIDRMask(32, 32)}, - LinkIndex: 1, - Scope: netlink.SCOPE_LINK, - }).Return(nil), - // add routes to exceptional CIDRs via eth0 - mnl.EXPECT().RouteReplace(&netlink.Route{ - Dst: net1, - Gw: defaultGw, - LinkIndex: 1, - Protocol: unix.RTPROT_STATIC, - }).Return(nil), - mnl.EXPECT().RouteReplace(&netlink.Route{ - Dst: net2, - Gw: defaultGw, - LinkIndex: 1, - Protocol: unix.RTPROT_STATIC, - }).Return(nil), - // retrieve wg0 link - mnl.EXPECT().LinkByName("wg0").Return(wg0, nil), - // add default route via wg0 - mnl.EXPECT().RouteReplace(&netlink.Route{ - Dst: dnet, - Gw: nil, - Via: &netlink.Via{ - Addr: net.ParseIP("fe80::1"), - AddrFamily: nl.FAMILY_V6, - }, - LinkIndex: 2, - Scope: netlink.SCOPE_UNIVERSE, - Family: nl.FAMILY_V4, - }), - // add iptables rules - mipt.EXPECT().New().Return(mtable, nil), - mtable.EXPECT().AppendUnique("mangle", "PREROUTING", "-i", "eth0", "-j", "MARK", "--set-mark", "8738").Return(nil), - mtable.EXPECT().AppendUnique("mangle", "PREROUTING", "-j", "CONNMARK", "--save-mark").Return(nil), - mtable.EXPECT().AppendUnique("mangle", "OUTPUT", "-m", "connmark", "--mark", "8738", "-j", "CONNMARK", "--restore-mark").Return(nil), - mnl.EXPECT().RuleAdd(rule).Return(nil), - // add route in 8738 table - mnl.EXPECT().RouteReplace(&defaultRoute).Return(nil), - ) - - if err := os.MkdirAll(allDir, os.ModePerm); err != nil { - t.Fatalf("Failed to mkdir %s: %v", allDir, err) - } - defer func() { - _ = os.RemoveAll(testDir) - }() - if err := os.MkdirAll(eth0Dir, os.ModePerm); err != nil { - t.Fatalf("Failed to mkdir %s: %v", eth0Dir, err) - } - if _, err := os.Create(allFile); err != nil { - t.Fatalf("Failed to create file %s: %v", allFile, err) + + defaultGatewayRouteSetupProcess := func() { + gomock.InOrder( + // retrieve eth0 link + mnl.EXPECT().LinkByName("eth0").Return(eth0, nil), + // retrieve wg0 link + mnl.EXPECT().LinkByName("wg0").Return(wg0, nil), + // get existing routes + mnl.EXPECT().RouteList(eth0, netlink.FAMILY_ALL).Return(existingRoutes, nil), + // delete existing routes + mnl.EXPECT().RouteDel(&existingRoutes[0]).Return(nil), + mnl.EXPECT().RouteDel(&existingRoutes[1]).Return(nil), + // add route to default gateway via eth0 + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: defaultGw, Mask: net.CIDRMask(32, 32)}, + LinkIndex: 1, + Scope: netlink.SCOPE_LINK, + }).Return(nil), + // add default route via wg0 + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: dnet, + Gw: nil, + Via: &netlink.Via{ + Addr: net.ParseIP("fe80::1"), + AddrFamily: nl.FAMILY_V6, + }, + LinkIndex: 2, + Scope: netlink.SCOPE_UNIVERSE, + Family: nl.FAMILY_V4, + }), + // add routes to exceptional CIDRs via eth0 + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: net1, + Gw: defaultGw, + LinkIndex: 1, + Protocol: unix.RTPROT_STATIC, + }).Return(nil), + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: net2, + Gw: defaultGw, + LinkIndex: 1, + Protocol: unix.RTPROT_STATIC, + }).Return(nil), + ) } - if _, err := os.Create(eth0File); err != nil { - t.Fatalf("Failed to create file %s: %v", eth0File, err) + defaultAzureNetworkingRouteSetupProcess := func() { + gomock.InOrder( + // retrieve eth0 link + mnl.EXPECT().LinkByName("eth0").Return(eth0, nil), + // retrieve wg0 link + mnl.EXPECT().LinkByName("wg0").Return(wg0, nil), + // get existing routes + mnl.EXPECT().RouteList(eth0, netlink.FAMILY_ALL).Return(existingRoutes, nil), + // add routes to exceptional CIDRs via wg0 + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: net1, + Gw: nil, + Via: &netlink.Via{ + Addr: net.ParseIP("fe80::1"), + AddrFamily: nl.FAMILY_V6, + }, + LinkIndex: 2, + Scope: netlink.SCOPE_UNIVERSE, + Family: nl.FAMILY_V4, + }).Return(nil), + mnl.EXPECT().RouteReplace(&netlink.Route{ + Dst: net2, + Gw: nil, + Via: &netlink.Via{ + Addr: net.ParseIP("fe80::1"), + AddrFamily: nl.FAMILY_V6, + }, + LinkIndex: 2, + Scope: netlink.SCOPE_UNIVERSE, + Family: nl.FAMILY_V4, + }).Return(nil), + ) } - result := ¤t.Result{} - err := SetPodRoutes("wg0", []string{"1.2.3.4/32", "172.17.0.4/16"}, testDir, result) - if err != nil { - t.Fatalf("SetPodRoutes returns unexpected error: %v", err) + tests := []struct { + desc string + defaultToGateway bool + expectedRouteResult []*types.Route + routeSetupProcess func() + }{ + { + desc: "default to gateway", + defaultToGateway: true, + expectedRouteResult: []*types.Route{ + {Dst: net.IPNet{IP: defaultGw, Mask: net.CIDRMask(32, 32)}}, + {Dst: *dnet, GW: net.ParseIP("fe80::1")}, + {Dst: *net1, GW: defaultGw}, + {Dst: *net2, GW: defaultGw}, + }, + routeSetupProcess: defaultGatewayRouteSetupProcess, + }, + { + desc: "default to azure network", + defaultToGateway: false, + expectedRouteResult: []*types.Route{ + {Dst: *net1, GW: net.ParseIP("fe80::1")}, + {Dst: *net2, GW: net.ParseIP("fe80::1")}, + }, + routeSetupProcess: defaultAzureNetworkingRouteSetupProcess, + }, } + for _, test := range tests { + test.routeSetupProcess() + gomock.InOrder( + // add iptables rules + mipt.EXPECT().New().Return(mtable, nil), + mtable.EXPECT().AppendUnique("mangle", "PREROUTING", "-i", "eth0", "-j", "MARK", "--set-mark", "8738").Return(nil), + mtable.EXPECT().AppendUnique("mangle", "PREROUTING", "-j", "CONNMARK", "--save-mark").Return(nil), + mtable.EXPECT().AppendUnique("mangle", "OUTPUT", "-m", "connmark", "--mark", "8738", "-j", "CONNMARK", "--restore-mark").Return(nil), + mnl.EXPECT().RuleAdd(rule).Return(nil), + // add route in 8738 table + mnl.EXPECT().RouteReplace(&defaultRoute).Return(nil), + ) - if !reflect.DeepEqual(result.Routes, expectedRouteResult) { - t.Fatalf("Got unexpected routes in result: %v, expected: %v", result.Routes, expectedRouteResult) - } + if err := os.MkdirAll(allDir, os.ModePerm); err != nil { + t.Fatalf("Failed to mkdir %s: %v", allDir, err) + } + defer func() { + _ = os.RemoveAll(testDir) + }() + if err := os.MkdirAll(eth0Dir, os.ModePerm); err != nil { + t.Fatalf("Failed to mkdir %s: %v", eth0Dir, err) + } + if _, err := os.Create(allFile); err != nil { + t.Fatalf("Failed to create file %s: %v", allFile, err) + } + if _, err := os.Create(eth0File); err != nil { + t.Fatalf("Failed to create file %s: %v", eth0File, err) + } - bytes, err := os.ReadFile(allFile) - if err != nil { - t.Fatalf("Failed to read file %s: %v", allFile, err) - } - if string(bytes) != "2" { - t.Fatalf("Got unexpected data in file %s: %s", allFile, string(bytes)) - } - bytes, err = os.ReadFile(eth0File) - if err != nil { - t.Fatalf("Failed to read file %s: %v", eth0File, err) - } - if string(bytes) != "2" { - t.Fatalf("Got unexpected data in file %s: %s", eth0File, string(bytes)) + result := ¤t.Result{} + err := SetPodRoutes("wg0", []string{"1.2.3.4/32", "172.17.0.4/16"}, test.defaultToGateway, testDir, result) + if err != nil { + t.Fatalf("SetPodRoutes returns unexpected error: %v", err) + } + + if !reflect.DeepEqual(result.Routes, test.expectedRouteResult) { + t.Fatalf("Got unexpected routes in result: %v, expected: %v", result.Routes, test.expectedRouteResult) + } + + bytes, err := os.ReadFile(allFile) + if err != nil { + t.Fatalf("Failed to read file %s: %v", allFile, err) + } + if string(bytes) != "2" { + t.Fatalf("Got unexpected data in file %s: %s", allFile, string(bytes)) + } + bytes, err = os.ReadFile(eth0File) + if err != nil { + t.Fatalf("Failed to read file %s: %v", eth0File, err) + } + if string(bytes) != "2" { + t.Fatalf("Got unexpected data in file %s: %s", eth0File, string(bytes)) + } } } diff --git a/pkg/cniprotocol/v1/cni.pb.go b/pkg/cniprotocol/v1/cni.pb.go index b70c059a..7ead3bf4 100644 --- a/pkg/cniprotocol/v1/cni.pb.go +++ b/pkg/cniprotocol/v1/cni.pb.go @@ -44,6 +44,55 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type DefaultRoute int32 + +const ( + DefaultRoute_DEFAULT_ROUTE_UNSPECIFIED DefaultRoute = 0 + DefaultRoute_DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY DefaultRoute = 1 + DefaultRoute_DEFAULT_ROUTE_AZURE_NETWORKING DefaultRoute = 2 +) + +// Enum value maps for DefaultRoute. +var ( + DefaultRoute_name = map[int32]string{ + 0: "DEFAULT_ROUTE_UNSPECIFIED", + 1: "DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY", + 2: "DEFAULT_ROUTE_AZURE_NETWORKING", + } + DefaultRoute_value = map[string]int32{ + "DEFAULT_ROUTE_UNSPECIFIED": 0, + "DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY": 1, + "DEFAULT_ROUTE_AZURE_NETWORKING": 2, + } +) + +func (x DefaultRoute) Enum() *DefaultRoute { + p := new(DefaultRoute) + *p = x + return p +} + +func (x DefaultRoute) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DefaultRoute) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_cniprotocol_v1_cni_proto_enumTypes[0].Descriptor() +} + +func (DefaultRoute) Type() protoreflect.EnumType { + return &file_pkg_cniprotocol_v1_cni_proto_enumTypes[0] +} + +func (x DefaultRoute) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DefaultRoute.Descriptor instead. +func (DefaultRoute) EnumDescriptor() ([]byte, []int) { + return file_pkg_cniprotocol_v1_cni_proto_rawDescGZIP(), []int{0} +} + type PodInfo struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -185,10 +234,11 @@ type NicAddResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - EndpointIp string `protobuf:"bytes,1,opt,name=endpoint_ip,json=endpointIp,proto3" json:"endpoint_ip,omitempty"` - ListenPort int32 `protobuf:"varint,2,opt,name=listen_port,json=listenPort,proto3" json:"listen_port,omitempty"` - PublicKey string `protobuf:"bytes,3,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"` - ExceptionCidrs []string `protobuf:"bytes,4,rep,name=exception_cidrs,json=exceptionCidrs,proto3" json:"exception_cidrs,omitempty"` + EndpointIp string `protobuf:"bytes,1,opt,name=endpoint_ip,json=endpointIp,proto3" json:"endpoint_ip,omitempty"` + ListenPort int32 `protobuf:"varint,2,opt,name=listen_port,json=listenPort,proto3" json:"listen_port,omitempty"` + PublicKey string `protobuf:"bytes,3,opt,name=public_key,json=publicKey,proto3" json:"public_key,omitempty"` + ExceptionCidrs []string `protobuf:"bytes,4,rep,name=exception_cidrs,json=exceptionCidrs,proto3" json:"exception_cidrs,omitempty"` + DefaultRoute DefaultRoute `protobuf:"varint,5,opt,name=default_route,json=defaultRoute,proto3,enum=pkg.cniprotocol.v1.DefaultRoute" json:"default_route,omitempty"` } func (x *NicAddResponse) Reset() { @@ -251,6 +301,13 @@ func (x *NicAddResponse) GetExceptionCidrs() []string { return nil } +func (x *NicAddResponse) GetDefaultRoute() DefaultRoute { + if x != nil { + return x.DefaultRoute + } + return DefaultRoute_DEFAULT_ROUTE_UNSPECIFIED +} + // CNIDeleteRequest is the request for cni del function. type NicDelRequest struct { state protoimpl.MessageState @@ -457,7 +514,7 @@ var file_pkg_cniprotocol_v1_cni_proto_rawDesc = []byte{ 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x9a, 0x01, + 0x52, 0x0b, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xe1, 0x01, 0x0a, 0x0e, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x69, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x49, @@ -467,51 +524,63 @@ var file_pkg_cniprotocol_v1_cni_proto_rawDesc = []byte{ 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x0f, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x69, 0x64, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x63, 0x65, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x69, 0x64, 0x72, 0x73, 0x22, 0x4b, 0x0a, 0x0d, 0x4e, 0x69, - 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, - 0x6f, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1b, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x70, 0x6f, - 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x10, 0x0a, 0x0e, 0x4e, 0x69, 0x63, 0x44, 0x65, - 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x50, 0x0a, 0x12, 0x50, 0x6f, 0x64, - 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3a, 0x0a, 0x0a, 0x70, 0x6f, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x09, 0x70, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0xb1, 0x01, 0x0a, 0x13, - 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, - 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, - 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, - 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, - 0x8e, 0x02, 0x0a, 0x0a, 0x4e, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x4f, - 0x0a, 0x06, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x12, 0x21, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, - 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x69, - 0x63, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, 0x6b, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x69, 0x64, 0x72, 0x73, 0x12, 0x45, 0x0a, 0x0d, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x72, 0x6f, 0x75, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x20, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, + 0x75, 0x74, 0x65, 0x52, 0x0c, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x52, 0x6f, 0x75, 0x74, + 0x65, 0x22, 0x4b, 0x0a, 0x0d, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x6f, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x70, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x10, + 0x0a, 0x0e, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x50, 0x0a, 0x12, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3a, 0x0a, 0x0a, 0x70, 0x6f, 0x64, 0x5f, 0x63, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x70, 0x6b, 0x67, + 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, + 0x50, 0x6f, 0x64, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x70, 0x6f, 0x64, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x22, 0xb1, 0x01, 0x0a, 0x13, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, + 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0b, 0x61, 0x6e, + 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x38, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x61, 0x6e, 0x6e, 0x6f, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x2a, 0x7a, 0x0a, 0x0c, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6c, + 0x74, 0x52, 0x6f, 0x75, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, + 0x54, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, + 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x27, 0x0a, 0x23, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, + 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x49, 0x43, 0x5f, 0x45, 0x47, + 0x52, 0x45, 0x53, 0x53, 0x5f, 0x47, 0x41, 0x54, 0x45, 0x57, 0x41, 0x59, 0x10, 0x01, 0x12, 0x22, + 0x0a, 0x1e, 0x44, 0x45, 0x46, 0x41, 0x55, 0x4c, 0x54, 0x5f, 0x52, 0x4f, 0x55, 0x54, 0x45, 0x5f, + 0x41, 0x5a, 0x55, 0x52, 0x45, 0x5f, 0x4e, 0x45, 0x54, 0x57, 0x4f, 0x52, 0x4b, 0x49, 0x4e, 0x47, + 0x10, 0x02, 0x32, 0x8e, 0x02, 0x0a, 0x0a, 0x4e, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x12, 0x4f, 0x0a, 0x06, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x12, 0x21, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, - 0x2e, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4f, 0x0a, 0x06, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x70, 0x6b, 0x67, 0x2e, - 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4e, - 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x70, + 0x2e, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x69, 0x63, 0x41, 0x64, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x4f, 0x0a, 0x06, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x12, 0x21, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, - 0x31, 0x2e, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x5e, 0x0a, 0x0b, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x12, - 0x26, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, - 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, - 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x41, - 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x2d, 0x65, 0x67, 0x72, 0x65, 0x73, 0x73, - 0x2d, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6e, 0x69, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x31, 0x2e, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x22, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x69, 0x63, 0x44, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x5e, 0x0a, 0x0b, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, + 0x76, 0x65, 0x12, 0x26, 0x2e, 0x70, 0x6b, 0x67, 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, + 0x65, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x70, 0x6b, 0x67, + 0x2e, 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2e, 0x76, 0x31, 0x2e, + 0x50, 0x6f, 0x64, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x39, 0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x2d, 0x65, 0x67, 0x72, + 0x65, 0x73, 0x73, 0x2d, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2f, 0x70, 0x6b, 0x67, 0x2f, + 0x63, 0x6e, 0x69, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x2f, 0x76, 0x31, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -526,33 +595,36 @@ func file_pkg_cniprotocol_v1_cni_proto_rawDescGZIP() []byte { return file_pkg_cniprotocol_v1_cni_proto_rawDescData } +var file_pkg_cniprotocol_v1_cni_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_cniprotocol_v1_cni_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_pkg_cniprotocol_v1_cni_proto_goTypes = []interface{}{ - (*PodInfo)(nil), // 0: pkg.cniprotocol.v1.PodInfo - (*NicAddRequest)(nil), // 1: pkg.cniprotocol.v1.NicAddRequest - (*NicAddResponse)(nil), // 2: pkg.cniprotocol.v1.NicAddResponse - (*NicDelRequest)(nil), // 3: pkg.cniprotocol.v1.NicDelRequest - (*NicDelResponse)(nil), // 4: pkg.cniprotocol.v1.NicDelResponse - (*PodRetrieveRequest)(nil), // 5: pkg.cniprotocol.v1.PodRetrieveRequest - (*PodRetrieveResponse)(nil), // 6: pkg.cniprotocol.v1.PodRetrieveResponse - nil, // 7: pkg.cniprotocol.v1.PodRetrieveResponse.AnnotationsEntry + (DefaultRoute)(0), // 0: pkg.cniprotocol.v1.DefaultRoute + (*PodInfo)(nil), // 1: pkg.cniprotocol.v1.PodInfo + (*NicAddRequest)(nil), // 2: pkg.cniprotocol.v1.NicAddRequest + (*NicAddResponse)(nil), // 3: pkg.cniprotocol.v1.NicAddResponse + (*NicDelRequest)(nil), // 4: pkg.cniprotocol.v1.NicDelRequest + (*NicDelResponse)(nil), // 5: pkg.cniprotocol.v1.NicDelResponse + (*PodRetrieveRequest)(nil), // 6: pkg.cniprotocol.v1.PodRetrieveRequest + (*PodRetrieveResponse)(nil), // 7: pkg.cniprotocol.v1.PodRetrieveResponse + nil, // 8: pkg.cniprotocol.v1.PodRetrieveResponse.AnnotationsEntry } var file_pkg_cniprotocol_v1_cni_proto_depIdxs = []int32{ - 0, // 0: pkg.cniprotocol.v1.NicAddRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo - 0, // 1: pkg.cniprotocol.v1.NicDelRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo - 0, // 2: pkg.cniprotocol.v1.PodRetrieveRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo - 7, // 3: pkg.cniprotocol.v1.PodRetrieveResponse.annotations:type_name -> pkg.cniprotocol.v1.PodRetrieveResponse.AnnotationsEntry - 1, // 4: pkg.cniprotocol.v1.NicService.NicAdd:input_type -> pkg.cniprotocol.v1.NicAddRequest - 3, // 5: pkg.cniprotocol.v1.NicService.NicDel:input_type -> pkg.cniprotocol.v1.NicDelRequest - 5, // 6: pkg.cniprotocol.v1.NicService.PodRetrieve:input_type -> pkg.cniprotocol.v1.PodRetrieveRequest - 2, // 7: pkg.cniprotocol.v1.NicService.NicAdd:output_type -> pkg.cniprotocol.v1.NicAddResponse - 4, // 8: pkg.cniprotocol.v1.NicService.NicDel:output_type -> pkg.cniprotocol.v1.NicDelResponse - 6, // 9: pkg.cniprotocol.v1.NicService.PodRetrieve:output_type -> pkg.cniprotocol.v1.PodRetrieveResponse - 7, // [7:10] is the sub-list for method output_type - 4, // [4:7] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 1, // 0: pkg.cniprotocol.v1.NicAddRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo + 0, // 1: pkg.cniprotocol.v1.NicAddResponse.default_route:type_name -> pkg.cniprotocol.v1.DefaultRoute + 1, // 2: pkg.cniprotocol.v1.NicDelRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo + 1, // 3: pkg.cniprotocol.v1.PodRetrieveRequest.pod_config:type_name -> pkg.cniprotocol.v1.PodInfo + 8, // 4: pkg.cniprotocol.v1.PodRetrieveResponse.annotations:type_name -> pkg.cniprotocol.v1.PodRetrieveResponse.AnnotationsEntry + 2, // 5: pkg.cniprotocol.v1.NicService.NicAdd:input_type -> pkg.cniprotocol.v1.NicAddRequest + 4, // 6: pkg.cniprotocol.v1.NicService.NicDel:input_type -> pkg.cniprotocol.v1.NicDelRequest + 6, // 7: pkg.cniprotocol.v1.NicService.PodRetrieve:input_type -> pkg.cniprotocol.v1.PodRetrieveRequest + 3, // 8: pkg.cniprotocol.v1.NicService.NicAdd:output_type -> pkg.cniprotocol.v1.NicAddResponse + 5, // 9: pkg.cniprotocol.v1.NicService.NicDel:output_type -> pkg.cniprotocol.v1.NicDelResponse + 7, // 10: pkg.cniprotocol.v1.NicService.PodRetrieve:output_type -> pkg.cniprotocol.v1.PodRetrieveResponse + 8, // [8:11] is the sub-list for method output_type + 5, // [5:8] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_pkg_cniprotocol_v1_cni_proto_init() } @@ -651,13 +723,14 @@ func file_pkg_cniprotocol_v1_cni_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_cniprotocol_v1_cni_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 8, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_cniprotocol_v1_cni_proto_goTypes, DependencyIndexes: file_pkg_cniprotocol_v1_cni_proto_depIdxs, + EnumInfos: file_pkg_cniprotocol_v1_cni_proto_enumTypes, MessageInfos: file_pkg_cniprotocol_v1_cni_proto_msgTypes, }.Build() File_pkg_cniprotocol_v1_cni_proto = out.File diff --git a/pkg/cniprotocol/v1/cni.proto b/pkg/cniprotocol/v1/cni.proto index bf964303..2cdb66a5 100644 --- a/pkg/cniprotocol/v1/cni.proto +++ b/pkg/cniprotocol/v1/cni.proto @@ -32,6 +32,12 @@ message PodInfo{ string pod_namespace = 2; } +enum DefaultRoute { + DEFAULT_ROUTE_UNSPECIFIED = 0; + DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY = 1; + DEFAULT_ROUTE_AZURE_NETWORKING = 2; +} + // CNIAddRequest is the request for cni add function. message NicAddRequest { PodInfo pod_config = 1; @@ -47,6 +53,7 @@ message NicAddResponse { int32 listen_port = 2; string public_key = 3; repeated string exception_cidrs = 4; + DefaultRoute default_route = 5; } // CNIDeleteRequest is the request for cni del function.