Support k8gb behind a reverse proxy

chart/k8gb/templates/validatingadmissionpolicy.yaml

@@ -0,0 +1,32 @@
+{{ if .Values.k8gb.validatingAdmissionPolicy.enabled -}}
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: k8gb-exposed-ip-annotation
+spec:
+  policyName: k8gb-exposed-ip-annotation
+  matchResources:
+    namespaceSelector: {}
+  validationActions:
+    - Deny
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: k8gb-exposed-ip-annotation
+spec:
+  validations:
+    - expression: object.metadata.annotations['k8gb.io/exposed-ip-addresses'].matches('^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$')
+      message: The annotation 'k8gb.io/exposed-ip-addresses' must contain a valid IPv4 address
+  matchConditions:
+    - name: hasExposedIPAddressesAnnotation
+      expression: "has(object.metadata.annotations) && 'k8gb.io/exposed-ip-addresses' in object.metadata.annotations"
+  matchConstraints:
+    resourceRules:
+      - apiGroups: ["k8gb.absa.oss"]
+        apiVersions: ["v1beta1"]
+        operations: ["CREATE", "UPDATE"]
+        resources: ["gslbs"]
+  failurePolicy: Fail
+{{ end -}}

chart/k8gb/values.schema.json

@@ -324,6 +324,9 @@
                 "serviceMonitor": {
                     "$ref": "#/definitions/k8gbServiceMonitor"
                 },
+                "validatingAdmissionPolicy": {
+                    "$ref": "#/definitions/k8gbValidatingAdmissionPolicy"
+                },
                 "podAnnotations": {
                     "type": "object"
                 },
@@ -409,6 +412,15 @@
                 }
             }
         },
+        "k8gbValidatingAdmissionPolicy": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "type": "boolean"
+                }
+            }
+        },
         "Ns1": {
             "type": "object",
             "additionalProperties": false,

chart/k8gb/values.yaml

@@ -56,6 +56,9 @@ k8gb:
   # -- enable ServiceMonitor
   serviceMonitor:
     enabled: false
+  # -- enable validating admission policies
+  validatingAdmissionPolicy:
+    enabled: true
   # -- pod annotations
   podAnnotations: {}
   # -- pod labels

controllers/depresolver/testdata/invalid_omitempty_empty.yaml

@@ -37,5 +37,3 @@ spec:
     type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
     splitBrainThresholdSeconds:
     weight:
-
-

controllers/gslb_controller_reconciliation.go

@@ -178,7 +178,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	}
 	gslb.Status.Servers = servers
 
-	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers)
+	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers)
 	if err != nil {
 		m.IncrementError(gslb)
 		return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err))

controllers/refresolver/ingress/ingress.go

@@ -35,6 +35,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	ingress *netv1.Ingress
 }
@@ -158,9 +163,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return utils.ParseIPAddresses(ingressIPsFromAnnotation)
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

controllers/refresolver/ingress/ingress_test.go

@@ -96,40 +96,80 @@ func TestGetServers(t *testing.T) {
 
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
-		name        string
-		ingressYaml string
-		expectedIPs []string
+		name          string
+		annotations   map[string]string
+		ingressYaml   string
+		expectedIPs   []string
+		expectedError bool
 	}{
 		{
-			name:        "no exposed IPs",
-			ingressYaml: "./testdata/ingress_no_ips.yaml",
-			expectedIPs: []string{},
+			name:          "no exposed IPs",
+			annotations:   map[string]string{},
+			ingressYaml:   "./testdata/ingress_no_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: false,
 		},
 		{
-			name:        "single exposed IP",
-			ingressYaml: "../testdata/ingress_referenced.yaml",
-			expectedIPs: []string{"10.0.0.1"},
+			name:          "single exposed IP",
+			annotations:   map[string]string{},
+			ingressYaml:   "../testdata/ingress_referenced.yaml",
+			expectedIPs:   []string{"10.0.0.1"},
+			expectedError: false,
 		},
 		{
-			name:        "multiple exposed IPs",
-			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
-			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
+			name:          "multiple exposed IPs",
+			annotations:   map[string]string{},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"10.0.0.1", "10.0.0.2"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with no exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
+		},
+		{
+			name:          "annotation with single exposed IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with multiple exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153", "185.199.109.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with invalid IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "192.169.0.test"},
+			ingressYaml:   "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
 		},
 	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
 			// arrange
-			ingress := utils.FileToIngress(test.ingressYaml)
+			ingress := utils.FileToIngress(tt.ingressYaml)
 			resolver := ReferenceResolver{
 				ingress: ingress,
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
-			assert.NoError(t, err)
+			IPs, err := resolver.GetGslbExposedIPs(tt.annotations, []utils.DNSServer{})
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
 
 			// assert
-			assert.Equal(t, test.expectedIPs, IPs)
+			assert.Equal(t, tt.expectedIPs, IPs)
 		})
 	}
 }

controllers/refresolver/istiovirtualservice/istiovirtualservice.go

@@ -37,6 +37,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	virtualService *istio.VirtualService
 	lbService      *corev1.Service
@@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return utils.ParseIPAddresses(ingressIPsFromAnnotation)
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go

@@ -111,40 +111,73 @@ func TestGetServers(t *testing.T) {
 
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
-		name        string
-		serviceYaml string
-		expectedIPs []string
+		name          string
+		annotations   map[string]string
+		serviceYaml   string
+		expectedIPs   []string
+		expectedError bool
 	}{
 		{
-			name:        "no exposed IPs",
-			serviceYaml: "./testdata/istio_service_no_ips.yaml",
-			expectedIPs: []string{},
+			name:          "no exposed IPs",
+			serviceYaml:   "./testdata/istio_service_no_ips.yaml",
+			annotations:   map[string]string{},
+			expectedIPs:   []string{},
+			expectedError: false,
 		},
 		{
-			name:        "single exposed IP",
-			serviceYaml: "../testdata/istio_service.yaml",
-			expectedIPs: []string{"10.0.0.1"},
+			name:          "single exposed IP",
+			annotations:   map[string]string{},
+			serviceYaml:   "../testdata/istio_service.yaml",
+			expectedIPs:   []string{"10.0.0.1"},
+			expectedError: false,
 		},
 		{
-			name:        "multiple exposed IPs",
-			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
-			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
+			name:          "multiple exposed IPs",
+			annotations:   map[string]string{},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{"10.0.0.1", "10.0.0.2"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with no exposed IPs",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
+		},
+		{
+			name:          "annotation with single exposed IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{"185.199.110.153"},
+			expectedError: false,
+		},
+		{
+			name:          "annotation with invalid IP",
+			annotations:   map[string]string{"k8gb.io/exposed-ip-addresses": "192.169.0.test"},
+			serviceYaml:   "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs:   []string{},
+			expectedError: true,
 		},
 	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
 			// arrange
-			svc := utils.FileToService(test.serviceYaml)
+			svc := utils.FileToService(tt.serviceYaml)
 			resolver := ReferenceResolver{
 				lbService: svc,
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
-			assert.NoError(t, err)
+			IPs, err := resolver.GetGslbExposedIPs(tt.annotations, []utils.DNSServer{})
+			if tt.expectedError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
 
 			// assert
-			assert.Equal(t, test.expectedIPs, IPs)
+			assert.Equal(t, tt.expectedIPs, IPs)
 		})
 	}
 }

controllers/refresolver/refresolver.go

@@ -34,7 +34,7 @@ type GslbReferenceResolver interface {
 	// GetServers retrieves GSLB the server configuration
 	GetServers() ([]*k8gbv1beta1.Server, error)
 	// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-	GetGslbExposedIPs(utils.DNSList) ([]string, error)
+	GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error)
 }
 
 // New creates a new GSLBReferenceResolver