Attach host interface to secondary bridge after clearing flow-restore…

…-wait flag

Antrea uses a global configuration flow-restore-wait='true' to ensure that OVS
OpenFlow entries can start working after the dependencies are ready. A
connectivity issue exists if a setup uses a Node's primary interface as the uplink
of the secondary network OVS bridge, before the flow-restore-wait option of
OVS is removed by antrea-agent.
This commit ensures agent attaches a host interface to the secondary OVS
bridge after the global flow-restore-wait option is removed.

Signed-off-by: Wenying Dong <wenyingd@vmware.com>

cmd/antrea-agent/agent.go

@@ -731,14 +731,15 @@ func run(o *Options) error {
 		go ipamController.Run(stopCh)
 	}
 
+	var secondaryNetworkController *secondarynetwork.Controller
 	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
-		defer secondarynetwork.RestoreHostInterfaceConfiguration(&o.config.SecondaryNetwork)
-		if err := secondarynetwork.Initialize(
+		secondaryNetworkController, err = secondarynetwork.NewController(
 			o.config.ClientConnection, o.config.KubeAPIServerOverride,
-			k8sClient, localPodInformer.Get(), nodeConfig.Name,
-			podUpdateChannel, stopCh,
-			&o.config.SecondaryNetwork, ovsdbConnection); err != nil {
-			return fmt.Errorf("failed to initialize secondary network: %v", err)
+			k8sClient, localPodInformer.Get(),
+			podUpdateChannel,
+			&o.config.SecondaryNetwork, ovsdbConnection)
+		if err != nil {
+			return fmt.Errorf("failed to create secondary network controller: %w", err)
 		}
 	}
 
@@ -864,6 +865,15 @@ func run(o *Options) error {
 			return fmt.Errorf("failed to connect uplink to OVS bridge: %w", err)
 		}
 	}
+	// secondaryNetworkController Initialize must be run after FlowRestoreComplete for the case that Node
+	// IPs are moved to the secondary OVS bridge
+	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
+		defer secondaryNetworkController.Restore()
+		if err = secondaryNetworkController.Initialize(); err != nil {
+			return fmt.Errorf("failed to initialize secondary network: %v", err)
+		}
+		go secondaryNetworkController.Run(stopCh)
+	}
 
 	// statsCollector collects stats and reports to the antrea-controller periodically. For now it's only used for
 	// NetworkPolicy stats and Multicast stats.

pkg/agent/secondarynetwork/init.go

@@ -0,0 +1,109 @@
+// Copyright 2024 Antrea Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package secondarynetwork
+
+import (
+	"fmt"
+
+	"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
+	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/cache"
+	componentbaseconfig "k8s.io/component-base/config"
+	"k8s.io/klog/v2"
+
+	"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
+	agentconfig "antrea.io/antrea/pkg/config/agent"
+	"antrea.io/antrea/pkg/ovs/ovsconfig"
+	"antrea.io/antrea/pkg/util/channel"
+	"antrea.io/antrea/pkg/util/k8s"
+)
+
+var (
+	newOVSBridgeFn = ovsconfig.NewOVSBridge
+)
+
+type Controller struct {
+	ovsBridgeClient ovsconfig.OVSBridgeClient
+	secNetConfig    *agentconfig.SecondaryNetworkConfig
+	podController   *podwatch.PodController
+}
+
+func NewController(
+	clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
+	kubeAPIServerOverride string,
+	k8sClient clientset.Interface,
+	podInformer cache.SharedIndexInformer,
+	podUpdateSubscriber channel.Subscriber,
+	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB,
+) (*Controller, error) {
+	ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
+	// definition from the API Server.
+	netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
+	if err != nil {
+		return nil, fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
+	}
+
+	// Create podController to handle secondary network configuration for Pods with
+	// k8s.v1.cni.cncf.io/networks Annotation defined.
+	podWatchController, err := podwatch.NewPodController(
+		k8sClient, netAttachDefClient, podInformer,
+		podUpdateSubscriber, ovsBridgeClient)
+	if err != nil {
+		return nil, err
+	}
+	return &Controller{
+		ovsBridgeClient: ovsBridgeClient,
+		secNetConfig:    secNetConfig,
+		podController:   podWatchController}, nil
+}
+
+// Run starts the Pod controller for secondary networks.
+func (c *Controller) Run(stopCh <-chan struct{}) {
+	c.podController.Run(stopCh)
+}
+
+// CreateNetworkAttachDefClient creates net-attach-def client handle from the given config.
+func createNetworkAttachDefClient(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (netdefclient.K8sCniCncfIoV1Interface, error) {
+	kubeConfig, err := k8s.CreateRestConfig(config, kubeAPIServerOverride)
+	if err != nil {
+		return nil, err
+	}
+
+	netAttachDefClient, err := netdefclient.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, err
+	}
+	return netAttachDefClient, nil
+}
+
+func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) {
+	if len(bridges) == 0 {
+		return nil, nil
+	}
+	// Only one OVS bridge is supported.
+	bridgeConfig := bridges[0]
+	ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb)
+	if err := ovsBridgeClient.Create(); err != nil {
+		return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err)
+	}
+	klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName)
+	return ovsBridgeClient, nil
+}

pkg/agent/secondarynetwork/init_linux.go

@@ -21,52 +21,28 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
-	netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/cache"
-	componentbaseconfig "k8s.io/component-base/config"
 	"k8s.io/klog/v2"
 
 	"antrea.io/antrea/pkg/agent/interfacestore"
-	"antrea.io/antrea/pkg/agent/secondarynetwork/podwatch"
 	"antrea.io/antrea/pkg/agent/util"
-	agentconfig "antrea.io/antrea/pkg/config/agent"
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
-	"antrea.io/antrea/pkg/util/channel"
-	"antrea.io/antrea/pkg/util/k8s"
 )
 
 var (
 	// Funcs which will be overridden with mock funcs in tests.
 	interfaceByNameFn = net.InterfaceByName
-	newOVSBridgeFn    = ovsconfig.NewOVSBridge
 )
 
-// Initialize sets up OVS bridges and starts the Pod controller for secondary networks.
-func Initialize(
-	clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
-	kubeAPIServerOverride string,
-	k8sClient clientset.Interface,
-	podInformer cache.SharedIndexInformer,
-	nodeName string,
-	podUpdateSubscriber channel.Subscriber,
-	stopCh <-chan struct{},
-	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
-
-	ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
-	if err != nil {
-		return err
-	}
-
+// Initialize sets up OVS bridges.
+func (c *Controller) Initialize() error {
 	// We only support moving and restoring of interface configuration to OVS Bridge for the single physical interface case.
-	if len(secNetConfig.OVSBridges) != 0 {
-		phyInterfaces := make([]string, len(secNetConfig.OVSBridges[0].PhysicalInterfaces))
-		copy(phyInterfaces, secNetConfig.OVSBridges[0].PhysicalInterfaces)
+	if len(c.secNetConfig.OVSBridges) != 0 {
+		phyInterfaces := make([]string, len(c.secNetConfig.OVSBridges[0].PhysicalInterfaces))
+		copy(phyInterfaces, c.secNetConfig.OVSBridges[0].PhysicalInterfaces)
 		if len(phyInterfaces) == 1 {
 
 			bridgedName, _, err := util.PrepareHostInterfaceConnection(
-				ovsBridgeClient,
+				c.ovsBridgeClient,
 				phyInterfaces[0],
 				0,
 				map[string]interface{}{
@@ -78,49 +54,18 @@ func Initialize(
 			}
 			phyInterfaces[0] = bridgedName
 		}
-		if err = connectPhyInterfacesToOVSBridge(ovsBridgeClient, phyInterfaces); err != nil {
+		if err := connectPhyInterfacesToOVSBridge(c.ovsBridgeClient, phyInterfaces); err != nil {
 			return err
 		}
 	}
-
-	// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
-	// definition from the API Server.
-	netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
-	if err != nil {
-		return fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
-	}
-
-	// Create podController to handle secondary network configuration for Pods with
-	// k8s.v1.cni.cncf.io/networks Annotation defined.
-	if podWatchController, err := podwatch.NewPodController(
-		k8sClient, netAttachDefClient, podInformer,
-		podUpdateSubscriber, ovsBridgeClient); err != nil {
-		return err
-	} else {
-		go podWatchController.Run(stopCh)
-	}
 	return nil
 }
 
-// RestoreHostInterfaceConfiguration restores interface configuration from secondary-bridge back to host-interface.
-func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {
-	if len(secNetConfig.OVSBridges) != 0 && len(secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {
-		util.RestoreHostInterfaceConfiguration(secNetConfig.OVSBridges[0].BridgeName, secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
-	}
-}
-
-func createOVSBridge(bridges []agentconfig.OVSBridgeConfig, ovsdb *ovsdb.OVSDB) (ovsconfig.OVSBridgeClient, error) {
-	if len(bridges) == 0 {
-		return nil, nil
+// Restore restores interface configuration from secondary-bridge back to host-interface.
+func (c *Controller) Restore() {
+	if len(c.secNetConfig.OVSBridges) != 0 && len(c.secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {
+		util.RestoreHostInterfaceConfiguration(c.secNetConfig.OVSBridges[0].BridgeName, c.secNetConfig.OVSBridges[0].PhysicalInterfaces[0])
 	}
-	// Only one OVS bridge is supported.
-	bridgeConfig := bridges[0]
-	ovsBridgeClient := newOVSBridgeFn(bridgeConfig.BridgeName, ovsconfig.OVSDatapathSystem, ovsdb)
-	if err := ovsBridgeClient.Create(); err != nil {
-		return nil, fmt.Errorf("failed to create OVS bridge %s: %v", bridgeConfig.BridgeName, err)
-	}
-	klog.InfoS("OVS bridge created", "bridge", bridgeConfig.BridgeName)
-	return ovsBridgeClient, nil
 }
 
 func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient, phyInterfaces []string) error {
@@ -146,17 +91,3 @@ func connectPhyInterfacesToOVSBridge(ovsBridgeClient ovsconfig.OVSBridgeClient,
 	}
 	return nil
 }
-
-// CreateNetworkAttachDefClient creates net-attach-def client handle from the given config.
-func createNetworkAttachDefClient(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) (netdefclient.K8sCniCncfIoV1Interface, error) {
-	kubeConfig, err := k8s.CreateRestConfig(config, kubeAPIServerOverride)
-	if err != nil {
-		return nil, err
-	}
-
-	netAttachDefClient, err := netdefclient.NewForConfig(kubeConfig)
-	if err != nil {
-		return nil, err
-	}
-	return netAttachDefClient, nil
-}

pkg/agent/secondarynetwork/init_windows.go

@@ -17,30 +17,10 @@
 
 package secondarynetwork
 
-import (
-	"errors"
-
-	"github.com/TomCodeLV/OVSDB-golang-lib/pkg/ovsdb"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/tools/cache"
-	componentbaseconfig "k8s.io/component-base/config"
-
-	agentconfig "antrea.io/antrea/pkg/config/agent"
-	"antrea.io/antrea/pkg/util/channel"
-)
-
-func Initialize(
-	clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
-	kubeAPIServerOverride string,
-	k8sClient clientset.Interface,
-	podInformer cache.SharedIndexInformer,
-	nodeName string,
-	podUpdateSubscriber channel.Subscriber,
-	stopCh <-chan struct{},
-	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
-	return errors.New("not supported on Windows")
+func (c *Controller) Initialize() error {
+	return nil
 }
 
-func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {
+func (c *Controller) Restore() {
 	// Not supported on Windows.
 }