parameterize gatewaylbprobe port

.pipeline/e2e.yaml

@@ -57,7 +57,7 @@ jobs:
   - bash: |
       set -euo pipefail
       mv $(pwd)/azure.json $(pwd)/config/azureconfig/azure.json
-      export EXCEPTION_CIDRS=${POD_CIDR}","${SERVICE_CIDR}
+      echo EXCEPTION_CIDRS=${POD_CIDR}","${SERVICE_CIDR} > $(pwd)/config/environment_variables/environment.env
       IMAGE_REGISTRY=$(registry.url) make install
       kubectl wait --for=condition=ready pod -A -l app=kube-egress-gateway --timeout=300s
       kubectl get all -n kube-egress-gateway-system

cmd/kube-egress-gateway-controller/cmd/root.go

@@ -29,6 +29,7 @@ import (
 	controllers "github.com/Azure/kube-egress-gateway/controllers/manager"
 	"github.com/Azure/kube-egress-gateway/pkg/azmanager"
 	"github.com/Azure/kube-egress-gateway/pkg/config"
+	"github.com/Azure/kube-egress-gateway/pkg/consts"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -56,6 +57,7 @@ var (
 	cloudConfig             config.CloudConfig
 	scheme                  = runtime.NewScheme()
 	metricsPort             int
+	gatewayLBProbePort      int
 	enableLeaderElection    bool
 	leaderElectionNamespace string
 	probePort               int
@@ -80,10 +82,11 @@ func init() {
 
 	rootCmd.Flags().IntVar(&metricsPort, "metrics-bind-port", 8080, "The port the metric endpoint binds to.")
 	rootCmd.Flags().IntVar(&probePort, "health-probe-bind-port", 8081, "The port the probe endpoint binds to.")
+	rootCmd.Flags().IntVar(&gatewayLBProbePort, "gateway-lb-probe-port", 8082, "The port the gateway lb health probe endpoint binds to.")
 	rootCmd.Flags().BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
-	rootCmd.Flags().StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "the namespace to create leader election objects")
+	rootCmd.Flags().StringVar(&leaderElectionNamespace, "leader-election-namespace", os.Getenv(consts.PodNamespaceEnvKey), "the namespace to create leader election objects")
 
 	zapOpts.BindFlags(goflag.CommandLine)
 	rootCmd.Flags().AddGoFlagSet(goflag.CommandLine)
@@ -206,6 +209,7 @@ func startControllers(cmd *cobra.Command, args []string) {
 		Client:       mgr.GetClient(),
 		AzureManager: az,
 		Recorder:     mgr.GetEventRecorderFor("gatewayLBConfiguration-controller"),
+		LBProbePort:  gatewayLBProbePort,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "GatewayLBConfiguration")
 		os.Exit(1)

cmd/kube-egress-gateway-daemon/cmd/root.go

@@ -56,13 +56,14 @@ func Execute() {
 }
 
 var (
-	cloudConfigFile string
-	cloudConfig     config.CloudConfig
-	scheme          = runtime.NewScheme()
-	setupLog        = ctrl.Log.WithName("setup")
-	metricsPort     int
-	probePort       int
-	zapOpts         = zap.Options{
+	cloudConfigFile    string
+	cloudConfig        config.CloudConfig
+	scheme             = runtime.NewScheme()
+	setupLog           = ctrl.Log.WithName("setup")
+	metricsPort        int
+	probePort          int
+	gatewayLBProbePort int
+	zapOpts            = zap.Options{
 		Development: true,
 	}
 )
@@ -84,6 +85,7 @@ func init() {
 
 	rootCmd.Flags().IntVar(&metricsPort, "metrics-bind-port", 8080, "The port the metric endpoint binds to.")
 	rootCmd.Flags().IntVar(&probePort, "health-probe-bind-port", 8081, "The port the probe endpoint binds to.")
+	rootCmd.Flags().IntVar(&gatewayLBProbePort, "gateway-lb-probe-port", 8082, "The port the gateway lb probe endpoint binds to.")
 
 	zapOpts.BindFlags(goflag.CommandLine)
 	rootCmd.Flags().AddGoFlagSet(goflag.CommandLine)
@@ -177,11 +179,18 @@ func startControllers(cmd *cobra.Command, args []string) {
 		os.Exit(1)
 	}
 
+	lbProbeServer := healthprobe.NewLBProbeServer(gatewayLBProbePort)
+	if err := mgr.Add(manager.RunnableFunc(lbProbeServer.Start)); err != nil {
+		setupLog.Error(err, "unbaled to set up gateway health probe server")
+		os.Exit(1)
+	}
+
 	netnsCleanupEvents := make(chan event.GenericEvent)
 	if err = (&controllers.StaticGatewayConfigurationReconciler{
-		Client:       mgr.GetClient(),
-		AzureManager: az,
-		TickerEvents: netnsCleanupEvents,
+		Client:        mgr.GetClient(),
+		AzureManager:  az,
+		TickerEvents:  netnsCleanupEvents,
+		LBProbeServer: lbProbeServer,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "StaticGatewayConfiguration")
 		os.Exit(1)
@@ -197,11 +206,6 @@ func startControllers(cmd *cobra.Command, args []string) {
 	}
 	//+kubebuilder:scaffold:builder
 
-	if err := mgr.Add(manager.RunnableFunc(healthprobe.Start)); err != nil {
-		setupLog.Error(err, "unbaled to set up gateway health probe server")
-		os.Exit(1)
-	}
-
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

config/cnimanager/kustomization.yaml

@@ -1,3 +1,3 @@
-bases:
+resources:
   - ./daemon
   - ./rbac

config/crd/kustomization.yaml

@@ -9,10 +9,10 @@ resources:
 - bases/egressgateway.kubernetes.azure.com_gatewaystatuses.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
-patchesStrategicMerge:
+patches:
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
 # patches here are for enabling the conversion webhook for each CRD
-- patches/webhook_in_staticgatewayconfigurations.yaml
+  - path: patches/webhook_in_staticgatewayconfigurations.yaml
 #- patches/webhook_in_podwireguardendpoints.yaml
 #- patches/webhook_in_gatewaylbconfigurations.yaml
 #- patches/webhook_in_gatewayvmconfigurations.yaml
@@ -21,7 +21,7 @@ patchesStrategicMerge:
 
 # [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
 # patches here are for enabling the CA injection for each CRD
-- patches/cainjection_in_staticgatewayconfigurations.yaml
+#- patches/cainjection_in_staticgatewayconfigurations.yaml
 #- patches/cainjection_in_podwireguardendpoints.yaml
 #- patches/cainjection_in_gatewaylbconfigurations.yaml
 #- patches/cainjection_in_gatewayvmconfigurations.yaml

config/daemon/kustomization.yaml

@@ -1,3 +1,3 @@
-bases:
+resources:
 - ./rbac
 - ./manager

config/daemon/manager/cloud_config_patch.yaml

@@ -8,8 +8,6 @@ spec:
     spec:
       containers:
       - name: daemon
-        args:
-        - "--cloud-config=/etc/config/azure.json"
         volumeMounts:
         - name: daemon-config-volume
           mountPath: /etc/config

config/daemon/manager/kustomization.yaml

@@ -1,5 +1,5 @@
 resources:
 - manager.yaml
 
-patchesStrategicMerge:
-  - cloud_config_patch.yaml
+patches:
+  - path: cloud_config_patch.yaml

config/daemon/manager/manager.yaml

@@ -37,6 +37,9 @@ spec:
       containers:
       - command:
         - /kube-egress-gateway-daemon
+        args:
+        - --gateway-lb-probe-port=8082
+        - --cloud-config=/etc/config/azure.json
         image: daemon:latest
         name: daemon
         securityContext:

config/environment_variables/environment.env

@@ -1,2 +1 @@
-# We do not specify actual values. Kustomize will capture the current environment variables into the ConfigMap.
-EXCEPTION_CIDRS
+EXCEPTION_CIDRS=

config/manager/kustomization.yaml

@@ -1,5 +1,5 @@
 
-bases:
+resources:
 - ./rbac
 - ./manager
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
@@ -10,11 +10,11 @@ bases:
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ./prometheus
 
-patchesStrategicMerge:
+patches:
 # Protect the /metrics endpoint by putting it behind auth.
 # If you want your controller-manager to expose the /metrics
 # endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
+  - path: manager_auth_proxy_patch.yaml
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
@@ -26,7 +26,7 @@ patchesStrategicMerge:
 # - webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
-vars:
+# vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
 # - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
 #   objref:

config/manager/manager/kustomization.yaml

@@ -3,7 +3,7 @@ resources:
 generatorOptions:
   disableNameSuffixHash: true
 
-patchesStrategicMerge:
+patches:
   # Mount the controller config file for loading manager configurations
   # through a ComponentConfig type
-  - manager_config_patch.yaml
+  - path: manager_config_patch.yaml

config/manager/manager/manager.yaml

@@ -31,13 +31,21 @@ spec:
         - /kube-egress-gateway-controller
         args:
         - --leader-elect
+        - --leader-election-namespace=$(MY_POD_NAMESPACE)
+        - --cloud-config=/etc/config/azure.json
+        - --gateway-lb-probe-port=8082
         image: controller:latest
         name: manager
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
             drop:
               - "ALL"
+        env:
+        - name: MY_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         livenessProbe:
           httpGet:
             path: /healthz

config/manager/manager/manager_config_patch.yaml

@@ -8,8 +8,6 @@ spec:
     spec:
       containers:
       - name: manager
-        args:
-          - "--cloud-config=/etc/config/azure.json"
         volumeMounts:
         - name: manager-config-volume
           mountPath: /etc/config

controllers/daemon/staticgatewayconfiguration_controller.go

@@ -47,11 +47,12 @@ var _ reconcile.Reconciler = &StaticGatewayConfigurationReconciler{}
 type StaticGatewayConfigurationReconciler struct {
 	client.Client
 	*azmanager.AzureManager
-	TickerEvents chan event.GenericEvent
-	Netlink      netlinkwrapper.Interface
-	NetNS        netnswrapper.Interface
-	IPTables     iptableswrapper.Interface
-	WgCtrl       wgctrlwrapper.Interface
+	TickerEvents  chan event.GenericEvent
+	LBProbeServer *healthprobe.LBProbeServer
+	Netlink       netlinkwrapper.Interface
+	NetNS         netnswrapper.Interface
+	IPTables      iptableswrapper.Interface
+	WgCtrl        wgctrlwrapper.Interface
 }
 
 //+kubebuilder:rbac:groups=egressgateway.kubernetes.azure.com,resources=staticgatewayconfigurations,verbs=get;list;watch
@@ -194,7 +195,7 @@ func (r *StaticGatewayConfigurationReconciler) reconcile(
 		return err
 	}
 
-	if err := healthprobe.AddGateway(string(gwConfig.GetUID())); err != nil {
+	if err := r.LBProbeServer.AddGateway(string(gwConfig.GetUID())); err != nil {
 		return err
 	}
 
@@ -279,7 +280,7 @@ func (r *StaticGatewayConfigurationReconciler) ensureDeleted(ctx context.Context
 	}
 
 	gwUID := getGatewayUIDFromNamespaceName(netns)
-	if err := healthprobe.RemoveGateway(gwUID); err != nil {
+	if err := r.LBProbeServer.RemoveGateway(gwUID); err != nil {
 		return err
 	}
 	return nil

controllers/daemon/staticgatewayconfiguration_controller_test.go

@@ -42,6 +42,7 @@ import (
 	"github.com/Azure/kube-egress-gateway/pkg/azmanager"
 	"github.com/Azure/kube-egress-gateway/pkg/config"
 	"github.com/Azure/kube-egress-gateway/pkg/consts"
+	"github.com/Azure/kube-egress-gateway/pkg/healthprobe"
 	"github.com/Azure/kube-egress-gateway/pkg/imds"
 	"github.com/Azure/kube-egress-gateway/pkg/iptableswrapper/mockiptableswrapper"
 	"github.com/Azure/kube-egress-gateway/pkg/netlinkwrapper/mocknetlinkwrapper"
@@ -82,7 +83,7 @@ var _ = Describe("Daemon StaticGatewayConfiguration controller unit tests", func
 		mctrl := gomock.NewController(GinkgoT())
 		az := getMockAzureManager(mctrl)
 		cl := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(objects...).Build()
-		r = &StaticGatewayConfigurationReconciler{Client: cl, AzureManager: az}
+		r = &StaticGatewayConfigurationReconciler{Client: cl, AzureManager: az, LBProbeServer: healthprobe.NewLBProbeServer(1000)}
 		r.Netlink = mocknetlinkwrapper.NewMockInterface(mctrl)
 		r.NetNS = mocknetnswrapper.NewMockInterface(mctrl)
 		r.IPTables = mockiptableswrapper.NewMockInterface(mctrl)

controllers/manager/gatewaylbconfiguration_controller.go

@@ -33,7 +33,8 @@ import (
 type GatewayLBConfigurationReconciler struct {
 	client.Client
 	*azmanager.AzureManager
-	Recorder record.EventRecorder
+	Recorder    record.EventRecorder
+	LBProbePort int
 }
 
 type lbPropertyNames struct {
@@ -350,7 +351,7 @@ func (r *GatewayLBConfigurationReconciler) reconcileLBRule(
 
 	probeID := r.GetLBProbeID(names.probeName)
 	expectedLBRule := getExpectedLBRule(&names.lbRuleName, frontendID, backendID, probeID)
-	expectedProbe := getExpectedLBProbe(&names.probeName, lbConfig)
+	expectedProbe := getExpectedLBProbe(&names.probeName, r.LBProbePort, lbConfig)
 
 	lbRules := lb.Properties.LoadBalancingRules
 	if needLB {
@@ -537,6 +538,7 @@ func getExpectedLBRule(lbRuleName, frontendID, backendID, probeID *string) *netw
 
 func getExpectedLBProbe(
 	probeName *string,
+	lbProbePort int,
 	lbConfig *egressgatewayv1alpha1.GatewayLBConfiguration,
 ) *network.Probe {
 	gatewayUID := ""
@@ -550,7 +552,7 @@ func getExpectedLBProbe(
 	probeProp := &network.ProbePropertiesFormat{
 		RequestPath: to.Ptr(consts.GatewayHealthProbeEndpoint + gatewayUID),
 		Protocol:    to.Ptr(network.ProbeProtocolHTTP),
-		Port:        to.Ptr(consts.WireguardDaemonServicePort),
+		Port:        to.Ptr(int32(lbProbePort)),
 	}
 	return &network.Probe{
 		Name:       probeName,