Add OVN load-balancer health checks

doc/api-extensions.md

@@ -2562,3 +2562,15 @@ This adds new configuration options to virtual machines to directly issue QMP co
 * `raw.qemu.qmp.early`
 * `raw.qemu.qmp.pre-start`
 * `raw.qemu.qmp.post-start`
+
+## `network_load_balancer_health_check`
+
+This adds the ability to perform health checks for load balancer backends.
+
+The following new configuration options are introduced:
+
+* `healthcheck`
+* `healthcheck.interval`
+* `healthcheck.timeout`
+* `healthcheck.failure_count`
+* `healthcheck.success_count`

doc/config_options.txt

@@ -1557,6 +1557,51 @@ This template gets access to the project name (`projectName`), integration name
 ```
 
 <!-- config group network_integration-ovn end -->
+<!-- config group network_load_balancer-common start -->
+```{config:option} user.* network_load_balancer-common
+:shortdesc: "Free form user key/value storage"
+:type: "string"
+User keys can be used in search.
+```
+
+<!-- config group network_load_balancer-common end -->
+<!-- config group network_load_balancer-healthcheck start -->
+```{config:option} healthcheck network_load_balancer-healthcheck
+:defaultdesc: "`false`"
+:shortdesc: "Whether to perform checks on the backends"
+:type: "bool"
+
+```
+
+```{config:option} healthcheck.failure_count network_load_balancer-healthcheck
+:defaultdesc: "`3`"
+:shortdesc: "Number of failed tests to consider the backend offline"
+:type: "integer"
+
+```
+
+```{config:option} healthcheck.interval network_load_balancer-healthcheck
+:defaultdesc: "`10`"
+:shortdesc: "Interval in seconds between health checks"
+:type: "integer"
+
+```
+
+```{config:option} healthcheck.success_count network_load_balancer-healthcheck
+:defaultdesc: "`3`"
+:shortdesc: "Number of successful tests to consider the backend online"
+:type: "integer"
+
+```
+
+```{config:option} healthcheck.timeout network_load_balancer-healthcheck
+:defaultdesc: "`30`"
+:shortdesc: "Test timeout"
+:type: "integer"
+
+```
+
+<!-- config group network_load_balancer-healthcheck end -->
 <!-- config group network_zone-common start -->
 ```{config:option} dns.nameservers network_zone-common
 :required: "no"

doc/howto/network_load_balancers.md

@@ -34,10 +34,20 @@ Property         | Type         | Required | Description
 :--              | :--          | :--      | :--
 `listen_address` | string       | yes      | IP address to listen on
 `description`    | string       | no       | Description of the network load balancer
-`config`         | string set   | no       | Configuration options as key/value pairs (only `user.*` custom keys supported)
+`config`         | string set   | no       | Configuration options as key/value pairs (see below)
 `backends`       | backend list | no       | List of {ref}`backend specifications <network-load-balancers-backend-specifications>`
 `ports`          | port list    | no       | List of {ref}`port specifications <network-load-balancers-port-specifications>`
 
+### Configuration options
+
+The following configuration options are available for all network integrations:
+
+% Include content from [../config_options.txt](../config_options.txt)
+```{include} ../config_options.txt
+    :start-after: <!-- config group network_load_balancer-common start -->
+    :end-before: <!-- config group network_load_balancer-common end -->
+```
+
 (network-load-balancers-listen-addresses)=
 ### Requirements for listen addresses
 

internal/server/metadata/configuration.json

@@ -1716,6 +1716,63 @@
 				]
 			}
 		},
+		"network_load_balancer": {
+			"common": {
+				"keys": [
+					{
+						"user.*": {
+							"longdesc": "User keys can be used in search.",
+							"shortdesc": "Free form user key/value storage",
+							"type": "string"
+						}
+					}
+				]
+			},
+			"healthcheck": {
+				"keys": [
+					{
+						"healthcheck": {
+							"defaultdesc": "`false`",
+							"longdesc": "",
+							"shortdesc": "Whether to perform checks on the backends",
+							"type": "bool"
+						}
+					},
+					{
+						"healthcheck.failure_count": {
+							"defaultdesc": "`3`",
+							"longdesc": "",
+							"shortdesc": "Number of failed tests to consider the backend offline",
+							"type": "integer"
+						}
+					},
+					{
+						"healthcheck.interval": {
+							"defaultdesc": "`10`",
+							"longdesc": "",
+							"shortdesc": "Interval in seconds between health checks",
+							"type": "integer"
+						}
+					},
+					{
+						"healthcheck.success_count": {
+							"defaultdesc": "`3`",
+							"longdesc": "",
+							"shortdesc": "Number of successful tests to consider the backend online",
+							"type": "integer"
+						}
+					},
+					{
+						"healthcheck.timeout": {
+							"defaultdesc": "`30`",
+							"longdesc": "",
+							"shortdesc": "Test timeout",
+							"type": "integer"
+						}
+					}
+				]
+			}
+		},
 		"network_zone": {
 			"common": {
 				"keys": [

internal/server/network/driver_common.go

@@ -1177,13 +1177,71 @@ func (n *common) loadBalancerValidate(listenAddress net.IP, forward *api.Network
 		}
 	}
 
-	// Look for any unknown config fields.
-	for k := range forward.Config {
+	// Check the configuration.
+	lbOptions := map[string]func(value string) error{
+		// gendoc:generate(entity=network_load_balancer, group=healthcheck, key=healthcheck)
+		//
+		// ---
+		//  type: bool
+		//  defaultdesc: `false`
+		//  shortdesc: Whether to perform checks on the backends
+		"healthcheck": validate.Optional(validate.IsBool),
+
+		// gendoc:generate(entity=network_load_balancer, group=healthcheck, key=healthcheck.interval)
+		//
+		// ---
+		//  type: integer
+		//  shortdesc: Interval in seconds between health checks
+		//  defaultdesc: `10`
+		"healthcheck.interval": validate.IsUint32,
+
+		// gendoc:generate(entity=network_load_balancer, group=healthcheck, key=healthcheck.success_count)
+		//
+		// ---
+		//  type: integer
+		//  shortdesc: Number of successful tests to consider the backend online
+		//  defaultdesc: `3`
+		"healthcheck.success_count": validate.IsUint32,
+
+		// gendoc:generate(entity=network_load_balancer, group=healthcheck, key=healthcheck.failure_count)
+		//
+		// ---
+		//  type: integer
+		//  shortdesc: Number of failed tests to consider the backend offline
+		//  defaultdesc: `3`
+		"healthcheck.failure_count": validate.IsUint32,
+
+		// gendoc:generate(entity=network_load_balancer, group=healthcheck, key=healthcheck.timeout)
+		//
+		// ---
+		//  type: integer
+		//  shortdesc: Test timeout
+		//  defaultdesc: `30`
+		"healthcheck.timeout": validate.IsUint32,
+	}
+
+	for k, v := range forward.Config {
 		// User keys are not validated.
+
+		// gendoc:generate(entity=network_load_balancer, group=common, key=user.*)
+		// User keys can be used in search.
+		// ---
+		//  type: string
+		//  shortdesc: Free form user key/value storage
 		if internalInstance.IsUserConfig(k) {
 			continue
 		}
 
+		checker, ok := lbOptions[k]
+		if ok {
+			err := checker(v)
+			if err != nil {
+				return nil, err
+			}
+
+			continue
+		}
+
 		return nil, fmt.Errorf("Invalid option %q", k)
 	}
 

internal/server/network/driver_ovn.go

@@ -27,6 +27,7 @@ import (
 	"github.com/lxc/incus/v6/internal/server/db"
 	dbCluster "github.com/lxc/incus/v6/internal/server/db/cluster"
 	deviceConfig "github.com/lxc/incus/v6/internal/server/device/config"
+	"github.com/lxc/incus/v6/internal/server/dnsmasq/dhcpalloc"
 	"github.com/lxc/incus/v6/internal/server/instance"
 	"github.com/lxc/incus/v6/internal/server/ip"
 	"github.com/lxc/incus/v6/internal/server/locking"
@@ -1950,15 +1951,14 @@ func (n *ovn) validateUplinkNetwork(p *api.Project, uplinkNetworkName string) (s
 
 // getDHCPv4Reservations returns list DHCP IPv4 reservations from NICs connected to this network.
 func (n *ovn) getDHCPv4Reservations() ([]iprange.Range, error) {
-	routerIntPortIPv4, _, err := n.parseRouterIntPortIPv4Net()
+	routerIntPortIPv4, ipv4Net, err := n.parseRouterIntPortIPv4Net()
 	if err != nil {
 		return nil, fmt.Errorf("Failed parsing router's internal port IPv4 Net for DHCP reservation: %w", err)
 	}
 
 	var dhcpReserveIPv4s []iprange.Range
-
 	if routerIntPortIPv4 != nil {
-		dhcpReserveIPv4s = []iprange.Range{{Start: routerIntPortIPv4}}
+		dhcpReserveIPv4s = []iprange.Range{{Start: routerIntPortIPv4}, {Start: dhcpalloc.GetIP(ipv4Net, -2)}}
 	}
 
 	err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error {
@@ -4744,7 +4744,7 @@ func (n *ovn) ForwardCreate(forward api.NetworkForwardsPost, clientType request.
 
 		vips := n.forwardFlattenVIPs(net.ParseIP(forward.ListenAddress), net.ParseIP(forward.Config["target_address"]), portMaps)
 
-		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(forward.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(forward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 		if err != nil {
 			return fmt.Errorf("Failed applying OVN load balancer: %w", err)
 		}
@@ -4845,7 +4845,7 @@ func (n *ovn) ForwardUpdate(listenAddress string, req api.NetworkForwardPut, cli
 		}
 
 		vips := n.forwardFlattenVIPs(net.ParseIP(newForward.ListenAddress), net.ParseIP(newForward.Config["target_address"]), portMaps)
-		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newForward.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newForward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 		if err != nil {
 			return fmt.Errorf("Failed applying OVN load balancer: %w", err)
 		}
@@ -4855,7 +4855,7 @@ func (n *ovn) ForwardUpdate(listenAddress string, req api.NetworkForwardPut, cli
 			portMaps, err := n.forwardValidate(net.ParseIP(curForward.ListenAddress), &curForward.NetworkForwardPut)
 			if err == nil {
 				vips := n.forwardFlattenVIPs(net.ParseIP(curForward.ListenAddress), net.ParseIP(curForward.Config["target_address"]), portMaps)
-				_ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curForward.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+				_ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curForward.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 				_ = n.forwardBGPSetupPrefixes()
 			}
 		})
@@ -5118,7 +5118,19 @@ func (n *ovn) LoadBalancerCreate(loadBalancer api.NetworkLoadBalancersPost, clie
 
 		vips := n.loadBalancerFlattenVIPs(net.ParseIP(loadBalancer.ListenAddress), portMaps)
 
-		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(loadBalancer.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+		// Look at health checking configuration.
+		healthCheck, err := n.getHealthCheck(loadBalancer.NetworkLoadBalancerPut)
+		if err != nil {
+			return err
+		}
+
+		if healthCheck != nil {
+			for i := range vips {
+				vips[i].HealthCheck = healthCheck
+			}
+		}
+
+		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(loadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 		if err != nil {
 			return fmt.Errorf("Failed applying OVN load balancer: %w", err)
 		}
@@ -5220,7 +5232,19 @@ func (n *ovn) LoadBalancerUpdate(listenAddress string, req api.NetworkLoadBalanc
 
 		vips := n.loadBalancerFlattenVIPs(net.ParseIP(newLoadBalancer.ListenAddress), portMaps)
 
-		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newLoadBalancer.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+		// Look at health checking configuration.
+		healthCheck, err := n.getHealthCheck(newLoadBalancer.NetworkLoadBalancerPut)
+		if err != nil {
+			return err
+		}
+
+		if healthCheck != nil {
+			for i := range vips {
+				vips[i].HealthCheck = healthCheck
+			}
+		}
+
+		err = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(newLoadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 		if err != nil {
 			return fmt.Errorf("Failed applying OVN load balancer: %w", err)
 		}
@@ -5230,7 +5254,7 @@ func (n *ovn) LoadBalancerUpdate(listenAddress string, req api.NetworkLoadBalanc
 			portMaps, err := n.loadBalancerValidate(net.ParseIP(curLoadBalancer.ListenAddress), &curLoadBalancer.NetworkLoadBalancerPut)
 			if err == nil {
 				vips := n.loadBalancerFlattenVIPs(net.ParseIP(curLoadBalancer.ListenAddress), portMaps)
-				_ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curLoadBalancer.ListenAddress), []networkOVN.OVNRouter{n.getRouterName()}, []networkOVN.OVNSwitch{n.getIntSwitchName()}, vips...)
+				_ = n.ovnnb.CreateLoadBalancer(context.TODO(), n.getLoadBalancerName(curLoadBalancer.ListenAddress), n.getRouterName(), n.getIntSwitchName(), vips...)
 				_ = n.forwardBGPSetupPrefixes()
 			}
 		})
@@ -5336,6 +5360,61 @@ func (n *ovn) LoadBalancerDelete(listenAddress string, clientType request.Client
 	return nil
 }
 
+func (n *ovn) getHealthCheck(loadBalancer api.NetworkLoadBalancerPut) (*networkOVN.OVNLoadBalancerHealthCheck, error) {
+	// Check if load-balancer is enabled.
+	if !util.IsTrue(loadBalancer.Config["healthcheck"]) {
+		return nil, nil
+	}
+
+	// Get IPv4 checker.
+	var checkerIPV4 net.IP
+	_, ipv4Net, err := n.parseRouterIntPortIPv4Net()
+	if err == nil {
+		checkerIPV4 = dhcpalloc.GetIP(ipv4Net, -2)
+	}
+
+	// Get IPv6 checker.
+	var checkerIPV6 net.IP
+	_, ipv6Net, err := n.parseRouterIntPortIPv6Net()
+	if err == nil {
+		checkerIPV6 = dhcpalloc.GetIP(ipv6Net, -2)
+	}
+
+	// Parse the healthcheck options.
+	hcInterval, err := strconv.Atoi(loadBalancer.Config["healthcheck.interval"])
+	if err != nil && loadBalancer.Config["healthcheck.interval"] != "" {
+		return nil, err
+	}
+
+	hcTimeout, err := strconv.Atoi(loadBalancer.Config["healthcheck.timeout"])
+	if err != nil && loadBalancer.Config["healthcheck.timeout"] != "" {
+		return nil, err
+	}
+
+	hcFailureCount, err := strconv.Atoi(loadBalancer.Config["healthcheck.failure_count"])
+	if err != nil && loadBalancer.Config["healthcheck.failure_count"] != "" {
+		return nil, err
+	}
+
+	hcSuccessCount, err := strconv.Atoi(loadBalancer.Config["healthcheck.success_count"])
+	if err != nil && loadBalancer.Config["healthcheck.success_count"] != "" {
+		return nil, err
+	}
+
+	// Prepare the load-balancer health check.
+	healthCheck := &networkOVN.OVNLoadBalancerHealthCheck{
+		CheckerIPV4: checkerIPV4,
+		CheckerIPV6: checkerIPV6,
+
+		Interval:     hcInterval,
+		Timeout:      hcTimeout,
+		FailureCount: hcFailureCount,
+		SuccessCount: hcSuccessCount,
+	}
+
+	return healthCheck, nil
+}
+
 // Leases returns a list of leases for the OVN network. Those are directly extracted from the OVN database.
 func (n *ovn) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) {
 	var err error