[SecondaryNetwork] Attach OVS uplink for after FlowRestoreComplete

Physcial network interface is attached on the OVS bridge with SecondaryNetwork
feature enabled. Antrea uses a global configuration flow-restore-wait='true'
to ensure that OVS OpenFlow entries can start working after the dependencies are
ready. A connectivity issue exists if a setup uses the Node NIC as secondary
network interface and connects the NIC to OVS bridge before removing the
flow-restore-wait option.

This change ensures agent attaches the physical network interface to the secondary
OVS bridge after the global flow-restore-wait option is removed.

Signed-off-by: Wenying Dong <wenyingd@vmware.com>

cmd/antrea-agent/agent.go

@@ -731,13 +731,14 @@ func run(o *Options) error {
 		go ipamController.Run(stopCh)
 	}
 
+	var secondaryNetworkController *secondarynetwork.Controller
 	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
-		defer secondarynetwork.RestoreHostInterfaceConfiguration(&o.config.SecondaryNetwork)
-		if err := secondarynetwork.Initialize(
+		secondaryNetworkController, err = secondarynetwork.NewController(
 			o.config.ClientConnection, o.config.KubeAPIServerOverride,
-			k8sClient, localPodInformer.Get(), nodeConfig.Name,
-			podUpdateChannel, stopCh,
-			&o.config.SecondaryNetwork, ovsdbConnection); err != nil {
+			k8sClient, localPodInformer.Get(),
+			podUpdateChannel,
+			&o.config.SecondaryNetwork, ovsdbConnection)
+		if err != nil {
 			return fmt.Errorf("failed to initialize secondary network: %v", err)
 		}
 	}
@@ -864,6 +865,14 @@ func run(o *Options) error {
 			return fmt.Errorf("failed to connect uplink to OVS bridge: %w", err)
 		}
 	}
+	// secondaryNetworkController Initialize must be run after FlowRestoreComplete.
+	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
+		defer secondarynetwork.RestoreHostInterfaceConfiguration(&o.config.SecondaryNetwork)
+		if err = secondaryNetworkController.Initialize(&o.config.SecondaryNetwork); err != nil {
+			return fmt.Errorf("failed to initialize secondary network: %v", err)
+		}
+		go secondaryNetworkController.Run(stopCh)
+	}
 
 	// statsCollector collects stats and reports to the antrea-controller periodically. For now it's only used for
 	// NetworkPolicy stats and Multicast stats.

pkg/agent/secondarynetwork/init_linux.go

@@ -43,30 +43,52 @@ var (
 	newOVSBridgeFn    = ovsconfig.NewOVSBridge
 )
 
-// Initialize sets up OVS bridges and starts the Pod controller for secondary networks.
-func Initialize(
+type Controller struct {
+	ovsBridgeClient    ovsconfig.OVSBridgeClient
+	podWatchController podwatch.PodController
+}
+
+func NewController(
 	clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
 	kubeAPIServerOverride string,
 	k8sClient clientset.Interface,
 	podInformer cache.SharedIndexInformer,
-	nodeName string,
 	podUpdateSubscriber channel.Subscriber,
-	stopCh <-chan struct{},
-	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
-
+	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB,
+) (*Controller, error) {
 	ovsBridgeClient, err := createOVSBridge(secNetConfig.OVSBridges, ovsdb)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
+	// definition from the API Server.
+	netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
+	if err != nil {
+		return nil, fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
 	}
 
+	// Create podController to handle secondary network configuration for Pods with
+	// k8s.v1.cni.cncf.io/networks Annotation defined.
+	podWatchController, err := podwatch.NewPodController(
+		k8sClient, netAttachDefClient, podInformer,
+		podUpdateSubscriber, ovsBridgeClient)
+	if err != nil {
+		return nil, err
+	}
+	return &Controller{ovsBridgeClient: ovsBridgeClient, podWatchController: podWatchController}, nil
+}
+
+// Initialize sets up OVS bridges.
+func (c *Controller) Initialize(secNetConfig *agentconfig.SecondaryNetworkConfig) error {
 	// We only support moving and restoring of interface configuration to OVS Bridge for the single physical interface case.
 	if len(secNetConfig.OVSBridges) != 0 {
 		phyInterfaces := make([]string, len(secNetConfig.OVSBridges[0].PhysicalInterfaces))
 		copy(phyInterfaces, secNetConfig.OVSBridges[0].PhysicalInterfaces)
 		if len(phyInterfaces) == 1 {
 
 			bridgedName, _, err := util.PrepareHostInterfaceConnection(
-				ovsBridgeClient,
+				c.ovsBridgeClient,
 				phyInterfaces[0],
 				0,
 				map[string]interface{}{
@@ -78,30 +100,18 @@ func Initialize(
 			}
 			phyInterfaces[0] = bridgedName
 		}
-		if err = connectPhyInterfacesToOVSBridge(ovsBridgeClient, phyInterfaces); err != nil {
+		if err := connectPhyInterfacesToOVSBridge(c.ovsBridgeClient, phyInterfaces); err != nil {
 			return err
 		}
 	}
-
-	// Create the NetworkAttachmentDefinition client, which handles access to secondary network object
-	// definition from the API Server.
-	netAttachDefClient, err := createNetworkAttachDefClient(clientConnectionConfig, kubeAPIServerOverride)
-	if err != nil {
-		return fmt.Errorf("NetworkAttachmentDefinition client creation failed: %v", err)
-	}
-
-	// Create podController to handle secondary network configuration for Pods with
-	// k8s.v1.cni.cncf.io/networks Annotation defined.
-	if podWatchController, err := podwatch.NewPodController(
-		k8sClient, netAttachDefClient, podInformer,
-		podUpdateSubscriber, ovsBridgeClient); err != nil {
-		return err
-	} else {
-		go podWatchController.Run(stopCh)
-	}
 	return nil
 }
 
+// Run starts the Pod controller for secondary networks.
+func (c *Controller) Run(stopCh <-chan struct{}) {
+	c.podWatchController.Run(stopCh)
+}
+
 // RestoreHostInterfaceConfiguration restores interface configuration from secondary-bridge back to host-interface.
 func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {
 	if len(secNetConfig.OVSBridges) != 0 && len(secNetConfig.OVSBridges[0].PhysicalInterfaces) == 1 {

pkg/agent/secondarynetwork/init_windows.go

@@ -29,16 +29,25 @@ import (
 	"antrea.io/antrea/pkg/util/channel"
 )
 
-func Initialize(
+type Controller struct {
+}
+
+func NewController(
 	clientConnectionConfig componentbaseconfig.ClientConnectionConfiguration,
 	kubeAPIServerOverride string,
 	k8sClient clientset.Interface,
 	podInformer cache.SharedIndexInformer,
-	nodeName string,
 	podUpdateSubscriber channel.Subscriber,
-	stopCh <-chan struct{},
-	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB) error {
-	return errors.New("not supported on Windows")
+	secNetConfig *agentconfig.SecondaryNetworkConfig, ovsdb *ovsdb.OVSDB,
+) (*Controller, error) {
+	return nil, errors.New("not supported on Windows")
+}
+
+func (c *Controller) Initialize(secNetConfig *agentconfig.SecondaryNetworkConfig) error {
+	return nil
+}
+
+func (c *Controller) Run(stopCh <-chan struct{}) {
 }
 
 func RestoreHostInterfaceConfiguration(secNetConfig *agentconfig.SecondaryNetworkConfig) {

pkg/agent/secondarynetwork/podwatch/controller.go

@@ -77,6 +77,10 @@ type IPAMAllocator interface {
 	SecondaryNetworkRelease(podOwner *crdv1b1.PodOwner) error
 }
 
+type PodController interface {
+	Run(stopCh <-chan struct{})
+}
+
 type podCNIInfo struct {
 	containerID string
 	netNS       string