[Windows] Use Windows management vnic for OVS bridge

Rename the vnic created by Windows host when creating the HNS Network as
the OVS bridge name, and add OVS internal port for the bridge port. Then
OVS uses the vnic created by Windows host directly, and IP/MAC/route
migrations is not needed. With this workflow, the time of losing IP on
the Windows host is cut off.

Signed-off-by: wenying <wenyingd@vmware.com>

docs/design/windows-design.md

@@ -20,9 +20,10 @@ forwarded correctly.
 
 <img src="../assets/hns_integration.svg" width="600" alt="HNS Integration">
 
-SNAT based on OpenFlow is needed to make sure the containers can access the external address.
-The SNATed address is using the IP configured on the OVS bridge. Some additional OpenFlow entries
-are installed to assist in identifying and forwarding the external traffic.
+Windows NetNat is configured to make sure the Pods can access external addresses. The packet
+from a Pod to an external address is firstly output to antrea-gw0, and then SNAT is performed on the
+Windows host. The SNATed packet enters OVS from the OVS bridge interface and leaves the Windows host
+from the uplink interface directly.
 
 Antrea implements the Kubernetes ClusterIP Service leveraging OVS. Pod-to-ClusterIP-Service traffic
 is load-balanced and forwarded directly inside the OVS pipeline. And kube-proxy is running
@@ -99,10 +100,12 @@ port (provided with `local_ip` option). This local address is the one configured
 ### OVS bridge interface configuration
 
 Since OVS is also responsible for taking charge of the network of the host, an interface for the OVS bridge
-is required on which the host network settings are configured. It is created and enabled when creating
-the OVS bridge, and the MAC address should be changed to be the same as the uplink interface. Then the IP
-address and the route entries originally configured on the uplink interface should also be migrated to
-the interface.
+is required on which the host network settings are configured. The virtual network adapter which is created
+when creating the HNS Network is used as the OVS bridge interface. The virtual network adapter is renamed as
+the expected OVS bridge name, then the OVS bridge port is created. Hence, OVS can find the virtual network
+adapter with the name and attach it directly. Windows host has configured the virtual network adapter with
+IP, MAC and route entries which were originally on the uplink interface when creating the HNSNetwork, as a
+result, no extra manual IP/MAC/Route configurations on the OVS bridge are needed.
 
 The packets that are sent to/from the Windows host should be forwarded on this interface. So the OVS bridge
 is also a valid entry point into the OVS pipeline. A special ofport number 65534 (named as LOCAL) for the

hack/windows/Clean-AntreaNetwork.ps1

@@ -18,6 +18,7 @@ $OVS_DB_SCHEMA_PATH = "$OVSInstallDir\usr\share\openvswitch\vswitch.ovsschema"
 # Replace the path using the actual path where OVS conf.db locates. It is always under path OVSInstallDir\etc\openvswitch.
 $OVSDB_CONF_DIR = "$OVSInstallDir\etc\openvswitch"
 $OVS_DB_PATH = "$OVSDB_CONF_DIR\conf.db"
+$OVS_BR_ADAPTER = "vEthernet (br-int)"
 
 function GetHnsnetworkId($NetName) {
     $NetList= $(Get-HnsNetwork -ErrorAction SilentlyContinue)
@@ -80,7 +81,7 @@ function ResetOVSService() {
 }
 
 function RemoveNetworkAdapter($adapterName) {
-    $adapter = $(Get-NetAdapter $adapterName -ErrorAction Ignore)
+    $adapter = $(Get-NetAdapter "$adapterName" -ErrorAction Ignore)
     if ($adapter -ne $null) {
         Remove-NetIPAddress -IfAlias $adapterName -Confirm:$false
         Write-Host "Network adapter $adapter.Name is left on the Windows host with status $adapter.Status, please remove it manually."
@@ -105,7 +106,7 @@ function clearOVSBridge() {
         $BrIntDeleted = $false
         foreach ($RetryCount in $RetryCountRange) {
             Write-Host "Waiting for OVS bridge deletion complete ($RetryCount/$MaxRetryCount)..."
-            $BrIntAdapter = $(Get-NetAdapter br-int -ErrorAction SilentlyContinue)
+            $BrIntAdapter = $(Get-NetAdapter "$OVS_BR_ADAPTER" -ErrorAction SilentlyContinue)
             if ($BrIntAdapter -eq $null) {
                 $BrIntDeleted = $true
                 break
@@ -122,7 +123,7 @@ function clearOVSBridge() {
     }
 }
 
-$BrIntDeleted = $(Get-NetAdapter br-int) -Eq $null
+$BrIntDeleted = $(Get-NetAdapter "$OVS_BR_ADAPTER") -Eq $null
 if ($BrIntDeleted -eq $false) {
     clearOVSBridge
 }
@@ -139,7 +140,7 @@ if ($NetId -ne $null) {
 # This might happen after the Windows host is restarted abnormally, in which case some stale configurations block
 # ovs-vswitchd running, like the pid file and the misconfigurations in OVSDB.
 ResetOVSService "ovs-vswitchd"
-RemoveNetworkAdapter "br-int"
+RemoveNetworkAdapter $OVS_BR_ADAPTER
 RemoveNetworkAdapter "antrea-gw0"
 ClearHyperVBindingOnAdapter($uplink)
 if ($RenewIPConfig) {

pkg/agent/agent_windows.go

@@ -130,8 +130,8 @@ func (i *Initializer) prepareOVSBridge() error {
 	// and break the OpenFlow channel.
 	// The length of datapathID is 64 bits, the lower 48-bits are for a MAC address, while the upper 16-bits are
 	// implementer-defined. Antrea uses "0x0000" for the upper 16-bits.
-	datapathID := strings.Replace(hnsNetwork.SourceMac, ":", "", -1)
-	datapathID = "0000" + datapathID
+	uplinkMACStr := strings.Replace(hnsNetwork.SourceMac, ":", "", -1)
+	datapathID := "0000" + uplinkMACStr
 	if err = i.ovsBridgeClient.SetDatapathID(datapathID); err != nil {
 		klog.Errorf("Failed to set datapath_id %s: %v", datapathID, err)
 		return err
@@ -142,6 +142,10 @@ func (i *Initializer) prepareOVSBridge() error {
 	if _, err = i.ovsBridgeClient.GetOFPort(brName, false); err == nil {
 		klog.Infof("OVS bridge local port %s already exists, skip the configuration", brName)
 	} else {
+		// Rename the vnic created by Windows host with the OVS bridge name, then it can be used by OVS directly.
+		if err := util.RenameVMNetworkAdapter(hnsNetwork.Name, uplinkMACStr, brName); err != nil {
+			return err
+		}
 		// OVS does not receive "ofport_request" param when creating local port, so here use config.AutoAssignedOFPort=0
 		// to ignore this param.
 		if _, err = i.ovsBridgeClient.CreateInternalPort(brName, config.AutoAssignedOFPort, nil); err != nil {
@@ -167,34 +171,6 @@ func (i *Initializer) prepareOVSBridge() error {
 	i.ifaceStore.AddInterface(uplinkInterface)
 	ovsCtlClient := ovsctl.NewClient(i.ovsBridge)
 
-	// Move network configuration of uplink interface to OVS bridge local interface.
-	// - The net configuration of uplink will be restored by OS if the attached HNS network is deleted.
-	// - When ovs-vswitchd is down, antrea-agent will disable OVS Extension. The OVS bridge local interface will work
-	//   like a normal interface on host and is responsible for forwarding host traffic.
-	if err = util.EnableHostInterface(brName); err != nil {
-		return err
-	}
-	if err = util.SetAdapterMACAddress(brName, &uplinkNetConfig.MAC); err != nil {
-		return err
-	}
-	// TODO: Configure IPv6 Address.
-	if err = util.ConfigureInterfaceAddressWithDefaultGateway(brName, uplinkNetConfig.IP, uplinkNetConfig.Gateway); err != nil {
-		if !strings.Contains(err.Error(), "Instance MSFT_NetIPAddress already exists") {
-			return err
-		}
-		err = nil
-		klog.V(4).Infof("Address: %s already exists when configuring IP on interface %s", uplinkNetConfig.IP.String(), brName)
-	}
-	// Restore the host routes which are lost when moving the network configuration of the uplink interface to OVS bridge interface.
-	if err = i.restoreHostRoutes(); err != nil {
-		return err
-	}
-
-	if uplinkNetConfig.DNSServers != "" {
-		if err = util.SetAdapterDNSServers(brName, uplinkNetConfig.DNSServers); err != nil {
-			return err
-		}
-	}
 	// Set the uplink with "no-flood" config, so that the IP of local Pods and "antrea-gw0" will not be leaked to the
 	// underlay network by the "normal" flow entry.
 	if err := ovsCtlClient.SetPortNoFlood(config.UplinkOFPort); err != nil {
@@ -276,7 +252,7 @@ func (i *Initializer) restoreHostRoutes() error {
 func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
 	// Find transport Interface in the order: ifaceName -> "vEthernet (ifaceName)" -> br-int. Return immediately if
 	// an interface using the specified name exists. Using "vEthernet (ifaceName)" or br-int is for restart agent case.
-	for _, name := range []string{ifaceName, fmt.Sprintf("%s (%s)", util.ContainerVNICPrefix, ifaceName), ovsBridgeName} {
+	for _, name := range []string{ifaceName, util.VirtualAdapterName(ifaceName), ovsBridgeName} {
 		ipNet, _, link, err := util.GetIPNetDeviceByName(name)
 		if err == nil {
 			return ipNet, nil, link, nil

pkg/agent/cniserver/interface_configuration_windows.go

@@ -162,7 +162,7 @@ func (ic *ifConfigurator) configureContainerLink(
 		// CmdAdd request is returned; 2) for Docker runtime, the interface is created after hcsshim.HotAttachEndpoint,
 		// and the hcsshim call is not synchronized from the observation.
 		return ic.addPostInterfaceCreateHook(infraContainerID, epName, containerAccess, func() error {
-			ifaceName := fmt.Sprintf("%s (%s)", util.ContainerVNICPrefix, epName)
+			ifaceName := util.VirtualAdapterName(epName)
 			if err := util.SetInterfaceMTU(ifaceName, mtu); err != nil {
 				return fmt.Errorf("failed to configure MTU on container interface '%s': %v", ifaceName, err)
 			}
@@ -345,7 +345,7 @@ func (ic *ifConfigurator) checkContainerInterface(
 			containerIface.Sandbox, sandboxID)
 	}
 	hnsEP := strings.Split(containerIface.Name, "_")[0]
-	containerIfaceName := fmt.Sprintf("%s (%s)", util.ContainerVNICPrefix, hnsEP)
+	containerIfaceName := util.VirtualAdapterName(hnsEP)
 	intf, err := net.InterfaceByName(containerIfaceName)
 	if err != nil {
 		klog.Errorf("Failed to get container %s interface: %v", containerID, err)

pkg/agent/cniserver/pod_configuration_windows.go

@@ -71,7 +71,7 @@ func (pc *podConfigurator) connectInterfaceToOVS(
 	// Use the outer veth interface name as the OVS port name.
 	ovsPortName := hostIface.Name
 	containerConfig := buildContainerConfig(ovsPortName, containerID, podName, podNameSpace, containerIface, ips)
-	hostIfAlias := fmt.Sprintf("%s (%s)", util.ContainerVNICPrefix, ovsPortName)
+	hostIfAlias := util.VirtualAdapterName(ovsPortName)
 	// - For Containerd runtime, the container interface is created after CNI replying the network setup result.
 	//   So for such case we need to use asynchronous way to wait for interface to be created: we create the OVS port
 	//   and set the OVS Interface type "" first, and change the OVS Interface type to "internal" to connect to the

pkg/agent/interfacestore/types.go

@@ -31,6 +31,8 @@ const (
 	TunnelInterface
 	// UplinkInterface is used to mark current interface is for uplink port
 	UplinkInterface
+	// HostInterface is used to mark current interface is for host
+	HostInterface
 )
 
 type InterfaceType uint8
@@ -140,6 +142,10 @@ func NewUplinkInterface(uplinkName string) *InterfaceConfig {
 	return uplinkConfig
 }
 
+func NewHostInterface(hostInterfaceName string) *InterfaceConfig {
+	return &InterfaceConfig{InterfaceName: hostInterfaceName, Type: HostInterface}
+}
+
 // TODO: remove this method after IPv4/IPv6 dual-stack is supported completely.
 func (c *InterfaceConfig) GetIPv4Addr() net.IP {
 	return util.GetIPv4Addr(c.IPs)

pkg/agent/route/route_windows.go

@@ -44,6 +44,9 @@ const (
 
 var (
 	virtualServiceIPv4Net = util.NewIPNet(config.VirtualServiceIPv4)
+
+	// This is for test.
+	getOVSBridgeInterfaceNameFunc = util.VirtualAdapterName
 )
 
 type Client struct {
@@ -71,9 +74,10 @@ func NewClient(networkConfig *config.NetworkConfig, noSNAT, proxyAll, connectUpl
 // Service LoadBalancing is provided by OpenFlow.
 func (c *Client) Initialize(nodeConfig *config.NodeConfig, done func()) error {
 	c.nodeConfig = nodeConfig
-	bridgeInf, err := net.InterfaceByName(nodeConfig.OVSBridge)
+	bridgeInterfaceName := getOVSBridgeInterfaceNameFunc(nodeConfig.OVSBridge)
+	bridgeInf, err := net.InterfaceByName(bridgeInterfaceName)
 	if err != nil {
-		return fmt.Errorf("failed to find the interface %s: %v", nodeConfig.OVSBridge, err)
+		return fmt.Errorf("failed to find the interface %s: %v", bridgeInterfaceName, err)
 	}
 	c.bridgeInfIndex = bridgeInf.Index
 	if err := c.initFwRules(); err != nil {

pkg/agent/route/route_windows_test.go

@@ -67,6 +67,9 @@ func TestRouteOperation(t *testing.T) {
 		},
 	}
 	called := false
+	getOVSBridgeInterfaceNameFunc = func(bridge string) string {
+		return bridge
+	}
 	err = client.Initialize(nodeConfig, func() { called = true })
 	require.Nil(t, err)
 	require.True(t, called)

pkg/agent/util/net_windows.go

@@ -189,6 +189,12 @@ func RemoveManagementInterface(networkName string) error {
 	return err
 }
 
+func RenameVMNetworkAdapter(networkName string, macStr, newName string) error {
+	cmd := fmt.Sprintf(`get-vmnetworkadapter -managementOS -SwitchName "%s" | ? MacAddress -EQ "%s" | Rename-VMNetworkAdapter -NewName "%s"`, networkName, macStr, newName)
+	_, err := ps.RunCommand(cmd)
+	return err
+}
+
 // ConfigureMACAddress set specified MAC address on interface.
 func SetAdapterMACAddress(adapterName string, macConfig *net.HardwareAddr) error {
 	macAddr := strings.Replace(macConfig.String(), ":", "", -1)
@@ -382,7 +388,7 @@ func PrepareHNSNetwork(subnetCIDR *net.IPNet, nodeIPNet *net.IPNet, uplinkAdapte
 	}
 
 	// Enable OVS Extension on the HNS Network. If an error occurs, delete the HNS Network and return the error.
-	if err = enableHNSOnOVS(hnsNet); err != nil {
+	if err = EnableHNSNetworkExtension(hnsNet.Id, OVSExtensionID); err != nil {
 		hnsNet.Delete()
 		return err
 	}
@@ -425,26 +431,6 @@ func EnableRSCOnVSwitch(vSwitch string) error {
 	return nil
 }
 
-func enableHNSOnOVS(hnsNet *hcsshim.HNSNetwork) error {
-	// Release OS management for HNS Network if Hyper-V is enabled.
-	hypervEnabled, err := WindowsHyperVEnabled()
-	if err != nil {
-		return err
-	}
-	if hypervEnabled {
-		if err := RemoveManagementInterface(LocalHNSNetwork); err != nil {
-			klog.Errorf("Failed to remove the interface managed by OS for HNSNetwork %s", LocalHNSNetwork)
-			return err
-		}
-	}
-
-	// Enable the HNS Network with OVS extension.
-	if err := EnableHNSNetworkExtension(hnsNet.Id, OVSExtensionID); err != nil {
-		return err
-	}
-	return err
-}
-
 // GetDefaultGatewayByInterfaceIndex returns the default gateway configured on the specified interface.
 func GetDefaultGatewayByInterfaceIndex(ifIndex int) (string, error) {
 	cmd := fmt.Sprintf("$(Get-NetRoute -InterfaceIndex %d -DestinationPrefix 0.0.0.0/0 ).NextHop", ifIndex)
@@ -789,3 +775,7 @@ func ReplaceNetNeighbor(neighbor *Neighbor) error {
 	}
 	return NewNetNeighbor(neighbor)
 }
+
+func VirtualAdapterName(ovsPortName string) string {
+	return fmt.Sprintf("%s (%s)", ContainerVNICPrefix, ovsPortName)
+}