support private egress only and default route selection

api/v1alpha1/gatewaylbconfiguration_types.go

@@ -42,7 +42,11 @@ type GatewayLBConfigurationSpec struct {
 
 	// Profile of the gateway VMSS to apply the gateway configuration.
 	// +optional
-	GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"`
+	GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"`
+
+	// Whether to provision public IP prefixes for outbound.
+	//+kubebuilder:default=true
+	ProvisionPublicIps bool `json:"provisionPublicIps"`
 
 	// BYO Resource ID of public IP prefix to be used as outbound.
 	// +optional
@@ -55,13 +59,13 @@ type GatewayLBConfigurationStatus struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	// Gateway IP for wireguard connection.
-	FrontendIP string `json:"frontendIP,omitempty"`
+	FrontendIp string `json:"frontendIp,omitempty"`
 
 	// Listening port of the gateway side wireguard daemon.
 	ServerPort int32 `json:"serverPort,omitempty"`
 
-	// Public IP Prefix CIDR used for this gateway configuration.
-	PublicIpPrefix string `json:"publicIpPrefix,omitempty"`
+	// Egress IP Prefix CIDR used for this gateway configuration.
+	EgressIpPrefix string `json:"egressIpPrefix,omitempty"`
 }
 
 //+kubebuilder:object:root=true

api/v1alpha1/gatewayvmconfiguration_types.go

@@ -42,7 +42,11 @@ type GatewayVMConfigurationSpec struct {
 
 	// Profile of the gateway VMSS to apply the gateway configuration.
 	// +optional
-	GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"`
+	GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"`
+
+	// Whether to provision public IP prefixes for outbound.
+	//+kubebuilder:default=true
+	ProvisionPublicIps bool `json:"provisionPublicIps"`
 
 	// BYO Resource ID of public IP prefix to be used as outbound.
 	// +optional

api/v1alpha1/staticgatewayconfiguration_types.go

@@ -32,20 +32,32 @@ import (
 // EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
-// GatewayVMSSProfile finds an existing gateway VMSS (virtual machine scale set).
-type GatewayVMSSProfile struct {
+// GatewayVmssProfile finds an existing gateway VMSS (virtual machine scale set).
+type GatewayVmssProfile struct {
 	// Resource group of the VMSS. Must be in the same subscription.
-	VMSSResourceGroup string `json:"vmssResourceGroup,omitempty"`
+	VmssResourceGroup string `json:"vmssResourceGroup,omitempty"`
 
 	// Name of the VMSS
-	VMSSName string `json:"vmssName,omitempty"`
+	VmssName string `json:"vmssName,omitempty"`
 
 	// Public IP prefix size to be applied to this VMSS.
 	//+kubebuilder:validation:Minimum=0
 	//+kubebuilder:validation:Maximum=31
 	PublicIpPrefixSize int32 `json:"publicIpPrefixSize,omitempty"`
 }
 
+// RouteType defines the type of defaultRoute.
+// +kubebuilder:validation:Enum=azureNetworking;staticEgressGateway
+type RouteType string
+
+const (
+	// RouteStaticEgressGateway defines static egress gateway as the default route.
+	RouteStaticEgressGateway RouteType = "staticEgressGateway"
+
+	// RouteAzureNetworking defines azure networking as the default route.
+	RouteAzureNetworking RouteType = "azureNetworking"
+)
+
 // StaticGatewayConfigurationSpec defines the desired state of StaticGatewayConfiguration
 type StaticGatewayConfigurationSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
@@ -57,20 +69,28 @@ type StaticGatewayConfigurationSpec struct {
 
 	// Profile of the gateway VMSS to apply the gateway configuration.
 	// +optional
-	GatewayVMSSProfile `json:"gatewayVmssProfile,omitempty"`
+	GatewayVmssProfile `json:"gatewayVmssProfile,omitempty"`
+
+	// Pod default route, should be either azureNetworking (pod's eth0) or staticEgressGateway (default).
+	//+kubebuilder:default=staticEgressGateway
+	DefaultRoute RouteType `json:"defaultRoute,omitempty"`
+
+	// Whether to provision public IP prefixes for outbound.
+	//+kubebuilder:default=true
+	ProvisionPublicIps bool `json:"provisionPublicIps"`
 
-	// BYO Resource ID of public IP prefix to be used as outbound.
+	// BYO Resource ID of public IP prefix to be used as outbound. This can only be specified when provisionPublicIps is true.
 	// +optional
 	PublicIpPrefixId string `json:"publicIpPrefixId,omitempty"`
 
-	// CIDRs to be excluded from outbound.
-	ExcludeCIDRs []string `json:"excludeCIDRs,omitempty"`
+	// CIDRs to be excluded from the default route.
+	ExcludeCidrs []string `json:"excludeCidrs,omitempty"`
 }
 
 // GatewayWireguardProfile provides details about gateway side wireguard configuration.
 type GatewayWireguardProfile struct {
 	// Gateway IP for wireguard connection.
-	WireguardServerIP string `json:"wireguardServerIP,omitempty"`
+	WireguardServerIp string `json:"wireguardServerIp,omitempty"`
 
 	// Listening port of the gateway side wireguard daemon.
 	WireguardServerPort int32 `json:"wireguardServerPort,omitempty"`
@@ -87,8 +107,8 @@ type StaticGatewayConfigurationStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	// Public IP Prefix CIDR used for this gateway configuration.
-	PublicIpPrefix string `json:"publicIpPrefix,omitempty"`
+	// Egress IP Prefix CIDR used for this gateway configuration.
+	EgressIpPrefix string `json:"egressIpPrefix,omitempty"`
 
 	// Gateway side wireguard profile.
 	GatewayWireguardProfile `json:"gatewayWireguardProfile,omitempty"`

api/v1alpha1/staticgatewayconfiguration_webhook.go

@@ -80,39 +80,45 @@ func (r *StaticGatewayConfiguration) ValidateDelete() (admission.Warnings, error
 }
 
 func (r *StaticGatewayConfiguration) validateSGC() error {
-	// need to validate either GatewayNodepoolName or GatewayVMSSProfile is provided, but not both
+	// need to validate either GatewayNodepoolName or GatewayVmssProfile is provided, but not both
 	var allErrs field.ErrorList
 
 	if r.Spec.GatewayNodepoolName == "" && r.vmssProfileIsEmpty() {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewaynodepoolname"),
-			fmt.Sprintf("GatewayNodepoolName: %s, GatewayVMSSProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVMSSProfile),
-			"Either GatewayNodepoolName or GatewayVMSSProfile must be provided"))
+			fmt.Sprintf("GatewayNodepoolName: %s, GatewayVmssProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVmssProfile),
+			"Either GatewayNodepoolName or GatewayVmssProfile must be provided"))
 	}
 
 	if r.Spec.GatewayNodepoolName != "" && !r.vmssProfileIsEmpty() {
 		allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewaynodepoolname"),
-			fmt.Sprintf("GatewayNodepoolName: %s, GatewayVMSSProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVMSSProfile),
-			"Only one of GatewayNodepoolName and GatewayVMSSProfile should be provided"))
+			fmt.Sprintf("GatewayNodepoolName: %s, GatewayVmssProfile: %#v", r.Spec.GatewayNodepoolName, r.Spec.GatewayVmssProfile),
+			"Only one of GatewayNodepoolName and GatewayVmssProfile should be provided"))
 	}
 
 	if !r.vmssProfileIsEmpty() {
-		if r.Spec.GatewayVMSSProfile.VMSSResourceGroup == "" {
+		if r.Spec.GatewayVmssProfile.VmssResourceGroup == "" {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("vmssresourcegroup"),
-				r.Spec.GatewayVMSSProfile.VMSSResourceGroup,
+				r.Spec.GatewayVmssProfile.VmssResourceGroup,
 				"Gateway vmss resource group is empty"))
 		}
-		if r.Spec.GatewayVMSSProfile.VMSSName == "" {
+		if r.Spec.GatewayVmssProfile.VmssName == "" {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("vmssname"),
-				r.Spec.GatewayVMSSProfile.VMSSName,
+				r.Spec.GatewayVmssProfile.VmssName,
 				"Gateway vmss name is empty"))
 		}
-		if r.Spec.GatewayVMSSProfile.PublicIpPrefixSize < 0 || r.Spec.GatewayVMSSProfile.PublicIpPrefixSize > 31 {
+		if r.Spec.GatewayVmssProfile.PublicIpPrefixSize < 0 || r.Spec.GatewayVmssProfile.PublicIpPrefixSize > 31 {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("gatewayvmssprofile").Child("publicipprefixsize"),
-				r.Spec.GatewayVMSSProfile.PublicIpPrefixSize,
+				r.Spec.GatewayVmssProfile.PublicIpPrefixSize,
 				"Gateway vmss public ip prefix size should be between 0 and 31 inclusively"))
 		}
 	}
 
+	if !r.Spec.ProvisionPublicIps && r.Spec.PublicIpPrefixId != "" {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("publicipprefixid"),
+			r.Spec.PublicIpPrefixId,
+			"PublicIpPrefixId should be empty when ProvisionPublicIps is false"))
+	}
+
 	if len(allErrs) == 0 {
 		return nil
 	}
@@ -122,7 +128,7 @@ func (r *StaticGatewayConfiguration) validateSGC() error {
 }
 
 func (r *StaticGatewayConfiguration) vmssProfileIsEmpty() bool {
-	return r.Spec.GatewayVMSSProfile.VMSSResourceGroup == "" &&
-		r.Spec.GatewayVMSSProfile.VMSSName == "" &&
-		r.Spec.GatewayVMSSProfile.PublicIpPrefixSize == 0
+	return r.Spec.GatewayVmssProfile.VmssResourceGroup == "" &&
+		r.Spec.GatewayVmssProfile.VmssName == "" &&
+		r.Spec.GatewayVmssProfile.PublicIpPrefixSize == 0
 }

api/v1alpha1/staticgatewayconfiguration_webhook_test.go

@@ -29,61 +29,76 @@ import "testing"
 func TestValidateSGC(t *testing.T) {
 	tests := map[string]struct {
 		GatewayNodepoolName string
-		VMSSResourceGroup   string
-		VMSSName            string
+		VmssResourceGroup   string
+		VmssName            string
 		PublicIpPrefixSize  int32
+		ProvisionPublicIps  bool
+		PublicIpPrefixId    string
 		ExpectErr           bool
 	}{
 		"It should pass when only gateway nodepool name is provided": {
 			GatewayNodepoolName: "test",
 		},
 		"It should pass when vmss profile is provided": {
-			VMSSResourceGroup:  "test",
-			VMSSName:           "test",
+			VmssResourceGroup:  "test",
+			VmssName:           "test",
 			PublicIpPrefixSize: 31,
 		},
 		"It should fail when both gateway nodepool name and vmss profile are provided": {
 			GatewayNodepoolName: "test",
-			VMSSResourceGroup:   "test",
+			VmssResourceGroup:   "test",
 			ExpectErr:           true,
 		},
 		"It should fail when no gateway profile is provided": {
 			ExpectErr: true,
 		},
 		"It should fail when vmss resource group is not provided": {
-			VMSSName:           "test",
+			VmssName:           "test",
 			PublicIpPrefixSize: 31,
 			ExpectErr:          true,
 		},
 		"It should fail when vmss name is not provided": {
-			VMSSResourceGroup:  "test",
+			VmssResourceGroup:  "test",
 			PublicIpPrefixSize: 31,
 			ExpectErr:          true,
 		},
 		"It should fail when PublicIpPrefixSize < 0": {
-			VMSSResourceGroup:  "test",
-			VMSSName:           "test",
+			VmssResourceGroup:  "test",
+			VmssName:           "test",
 			PublicIpPrefixSize: -1,
 			ExpectErr:          true,
 		},
 		"It should fail when PublicIpPrefixSize > 31": {
-			VMSSResourceGroup:  "test",
-			VMSSName:           "test",
+			VmssResourceGroup:  "test",
+			VmssName:           "test",
 			PublicIpPrefixSize: 32,
 			ExpectErr:          true,
 		},
+		"It should fail when PublicIPPrefixId is provided but ProvisionPublicIps is false": {
+			GatewayNodepoolName: "test",
+			ProvisionPublicIps:  false,
+			PublicIpPrefixId:    "test",
+			ExpectErr:           true,
+		},
+		"It should pass when PublicIPPrefixId is provided and ProvisionPublicIps is true": {
+			GatewayNodepoolName: "test",
+			ProvisionPublicIps:  true,
+			PublicIpPrefixId:    "test",
+		},
 	}
 
 	for name, test := range tests {
 		t.Run(name, func(t *testing.T) {
 			sgw := &StaticGatewayConfiguration{
 				Spec: StaticGatewayConfigurationSpec{
 					GatewayNodepoolName: test.GatewayNodepoolName,
-					GatewayVMSSProfile: GatewayVMSSProfile{
-						VMSSResourceGroup:  test.VMSSResourceGroup,
-						VMSSName:           test.VMSSName,
+					GatewayVmssProfile: GatewayVmssProfile{
+						VmssResourceGroup:  test.VmssResourceGroup,
+						VmssName:           test.VmssName,
 						PublicIpPrefixSize: test.PublicIpPrefixSize,
 					},
+					ProvisionPublicIps: test.ProvisionPublicIps,
+					PublicIpPrefixId:   test.PublicIpPrefixId,
 				},
 			}
 			err := sgw.validateSGC()

cmd/kube-egress-cni/main.go

@@ -194,8 +194,9 @@ func cmdAdd(args *skel.CmdArgs) error {
 			}
 
 			exceptionsCidrs := append(resp.GetExceptionCidrs(), config.ExcludedCIDRs...)
+			defaultToGateway := resp.GetDefaultRoute() == v1.DefaultRoute_DEFAULT_ROUTE_STATIC_EGRESS_GATEWAY
 			if os.Getenv("IS_UNIT_TEST_ENV") != "true" {
-				if err := routes.SetPodRoutes(consts.WireguardLinkName, exceptionsCidrs, "/proc/sys", result); err != nil {
+				if err := routes.SetPodRoutes(consts.WireguardLinkName, exceptionsCidrs, defaultToGateway, "/proc/sys", result); err != nil {
 					return fmt.Errorf("failed to setup pod routes: %w", err)
 				}
 			}

config/crd/bases/egressgateway.kubernetes.azure.com_gatewaylbconfigurations.yaml

@@ -54,19 +54,25 @@ spec:
                     description: Resource group of the VMSS. Must be in the same subscription.
                     type: string
                 type: object
+              provisionPublicIps:
+                default: true
+                description: Whether to provision public IP prefixes for outbound.
+                type: boolean
               publicIpPrefixId:
                 description: BYO Resource ID of public IP prefix to be used as outbound.
                 type: string
+            required:
+            - provisionPublicIps
             type: object
           status:
             description: GatewayLBConfigurationStatus defines the observed state of
               GatewayLBConfiguration
             properties:
-              frontendIP:
-                description: Gateway IP for wireguard connection.
+              egressIpPrefix:
+                description: Egress IP Prefix CIDR used for this gateway configuration.
                 type: string
-              publicIpPrefix:
-                description: Public IP Prefix CIDR used for this gateway configuration.
+              frontendIp:
+                description: Gateway IP for wireguard connection.
                 type: string
               serverPort:
                 description: Listening port of the gateway side wireguard daemon.

config/crd/bases/egressgateway.kubernetes.azure.com_gatewayvmconfigurations.yaml

@@ -54,9 +54,15 @@ spec:
                     description: Resource group of the VMSS. Must be in the same subscription.
                     type: string
                 type: object
+              provisionPublicIps:
+                default: true
+                description: Whether to provision public IP prefixes for outbound.
+                type: boolean
               publicIpPrefixId:
                 description: BYO Resource ID of public IP prefix to be used as outbound.
                 type: string
+            required:
+            - provisionPublicIps
             type: object
           status:
             description: GatewayVMConfigurationStatus defines the observed state of