OCPBUGS-18641: Set dual-stack IPFamilyPriority for vSphere

We have discovered that in dual-stack setups NodeAddresses field of
the instance metadata contains only IPv4 addresses for VMs that do have
both IPv4 and IPv6 addresses assigned (and detected by the VM agent).

It has been traced back to the function responsible for populating this
metadata field. We found out that for our configuration we always filter
only IPv4 addresses even if running in dual-stack. Reason for that is
that `IPFamilyPriority` has a value of `ipv4` even when running in a
dual-stack setup.

This causes an issue because this instance metadata is cross-checked
with addresses provided by the kubelet as part of the
`alpha.kubernetes.io/provided-node-ip` annotation. Without correct value
of `IPFamilyPriority` we are thus removing all the IPv6 addresses.

This PR takes advantage of the Service Networks configured by the user in
install-config and the fact that we only allow 2 networks to be configured
if the setup is dual-stack.

Fixes: OCPBUGS-18641

Author: mkowalski

pkg/cloud/vsphere/assets/cloud-controller-manager-deployment.yaml

@@ -66,6 +66,8 @@ spec:
               value: {{ .globalCredsSecretNamespace }}
             - name: VSPHERE_SECRET_NAME
               value: {{ .globalCredsSecretName }}
+            - name: ENABLE_ALPHA_DUAL_STACK
+              value: "true"
           resources:
             requests:
               cpu: 200m

pkg/cloud/vsphere/vsphere_config_transformer.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	configv1 "github.com/openshift/api/config/v1"
+	"k8s.io/utils/net"
 
 	ccmConfig "github.com/openshift/cluster-cloud-controller-manager-operator/pkg/cloud/vsphere/vsphere_cloud_config"
 )
@@ -26,7 +27,7 @@ const (
 // Currently, CloudConfigTransformer is responsible to populate vcenters, labels, and node networking parameters from
 // the Infrastructure resource.
 // Also, this function converts legacy deprecated INI configuration format to a YAML-based one.
-func CloudConfigTransformer(source string, infra *configv1.Infrastructure, _ *configv1.Network) (string, error) {
+func CloudConfigTransformer(source string, infra *configv1.Infrastructure, network *configv1.Network) (string, error) {
 	if infra.Status.PlatformStatus == nil ||
 		infra.Status.PlatformStatus.Type != configv1.VSpherePlatformType {
 		return "", fmt.Errorf("invalid platform, expected to be %s", configv1.VSpherePlatformType)
@@ -42,6 +43,7 @@ func CloudConfigTransformer(source string, infra *configv1.Infrastructure, _ *co
 	// https://github.com/openshift/enhancements/blob/f6b33eb0cd4ba060af71fee6192297cf6bc31e5a/enhancements/installer/vsphere-ipi-zonal.md
 	// https://github.com/openshift/api/pull/1278
 	if infra.Spec.PlatformSpec.VSphere != nil {
+		setDualStack(cpiCfg, infra.Status.PlatformStatus.VSphere, &infra.Spec.PlatformSpec.VSphere.NodeNetworking, network)
 		setNodes(cpiCfg, &infra.Spec.PlatformSpec.VSphere.NodeNetworking)
 		setVirtualCenters(cpiCfg, infra.Spec.PlatformSpec.VSphere)
 
@@ -99,3 +101,41 @@ func setVirtualCenters(cfg *ccmConfig.CPIConfig, vSphereSpec *configv1.VSpherePl
 		}
 	}
 }
+
+// setDualStack updates the configuration required by the cloud-provider-vsphere to explicitly set
+// value of IPFamilyPriority instead of using the default which is IPv4. This is needed by the
+// cloud provider in order to properly filter IP addresses that feed the instance metadata.
+//
+// We rely on the Service Networks configuration that initially comes from o/installer and later
+// from the Cluster Network Operator as those two components take care of validating that clusters
+// with dual-stack configuration have exactly 2 of them and that they match the required order.
+//
+// We are mangling with the ExcludeNetworkSubnetCIDR param here because VM agent by default detects
+// also IP addresses that are used by us internally and which should never be exposed as node IPs
+// (i.e. API VIP and Ingress VIP for IPI installations and fd69::2 which is internal to OVN-K8s).
+//
+// Ref.: https://issues.redhat.com/browse/OCPBUGS-18641
+func setDualStack(cfg *ccmConfig.CPIConfig, status *configv1.VSpherePlatformStatus, nodeNetworking *configv1.VSpherePlatformNodeNetworking, network *configv1.Network) {
+	if network != nil && len(network.Spec.ServiceNetwork) == 2 {
+		if net.IsIPv4String(network.Spec.ServiceNetwork[0]) {
+			cfg.Global.IPFamilyPriority = []string{"ipv4", "ipv6"}
+		} else {
+			cfg.Global.IPFamilyPriority = []string{"ipv6", "ipv4"}
+		}
+
+		if status != nil {
+			for _, addr := range append(status.APIServerInternalIPs, status.IngressIPs...) {
+				if net.IsIPv4String(addr) {
+					addr = addr + "/32"
+				} else {
+					addr = addr + "/128"
+				}
+				nodeNetworking.External.ExcludeNetworkSubnetCIDR = append(nodeNetworking.External.ExcludeNetworkSubnetCIDR, addr)
+				nodeNetworking.Internal.ExcludeNetworkSubnetCIDR = append(nodeNetworking.Internal.ExcludeNetworkSubnetCIDR, addr)
+			}
+		}
+
+		nodeNetworking.External.ExcludeNetworkSubnetCIDR = append(nodeNetworking.External.ExcludeNetworkSubnetCIDR, "fd69::2/128")
+		nodeNetworking.Internal.ExcludeNetworkSubnetCIDR = append(nodeNetworking.Internal.ExcludeNetworkSubnetCIDR, "fd69::2/128")
+	}
+}