[ExternalNode] Support role configuration on antrea agent (antrea-io#3542)

1. Add feature gate for ExternalNode which enables running Agent on a
   VM or BM.
2. Support Agent running APIServer only on localhost if it is not
   running on cluster worker Node.
3. CNIServer is loaded only when Agent is running on cluster worker
   Node.
4. Use a seperate build directory to generate agent configrations for
   ExternalNode.

Signed-off-by: wenyingd <wenyingd@vmware.com>

Author: wenyingd

build/yamls/externalnode/conf/antrea-agent.conf

@@ -0,0 +1,43 @@
+# FeatureGates is a map of feature names to bools that enable or disable experimental features.
+featureGates:
+# Enable running agent on an unmanaged VM/BM.
+  ExternalNode: true
+
+# Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins
+# to define security policies which apply to the entire cluster, and Antrea NetworkPolicy
+# feature that supports priorities, rule actions and externalEntities in the future.
+  AntreaPolicy: true
+
+# Enable collecting and exposing NetworkPolicy statistics.
+  NetworkPolicyStats: true
+
+# Name of the OpenVSwitch bridge antrea-agent will create and use.
+# Make sure it doesn't conflict with your existing OpenVSwitch bridges.
+#ovsBridge: br-int
+
+# Datapath type to use for the OpenVSwitch bridge created by Antrea. Supported values are:
+# - system
+# - netdev
+# 'system' is the default value and corresponds to the kernel datapath. Use 'netdev' to run
+# OVS in userspace mode (not fully supported yet). Userspace mode requires the tun device driver to
+# be available.
+#ovsDatapathType: system
+
+# The port for the antrea-agent APIServer to serve on.
+# Note that if it's set to another value, the `containerPort` of the `api` port of the
+# `antrea-agent` container must be set to the same value.
+#apiPort: 10350
+
+# NodeType is type of the Node where Antrea Agent is running.
+# Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode".
+nodeType: externalNode
+
+# The path to access the kubeconfig file used in the connection to K8s APIServer. The file contains the K8s
+# APIServer endpoint and the token of ServiceAccount required in the connection.
+clientConnection:
+  kubeconfig: antrea-agent.kubeconfig
+
+# The path to access the kubeconfig file used in the connection to Antrea Controller. The file contains the
+# antrea-controller APIServer endpoint and the token of ServiceAccount required in the connection.
+antreaClientConnection:
+  kubeconfig: antrea-agent.antrea.kubeconfig

cmd/antrea-agent/agent.go

@@ -87,6 +87,8 @@ const resyncPeriodDisabled = 0 * time.Minute
 // The devices that should be excluded from NodePort.
 var excludeNodePortDevices = []string{"antrea-egress0", "antrea-ingress0", "kube-ipvs0"}
 
+var ipv4Localhost = net.ParseIP("127.0.0.1")
+
 // run starts Antrea agent with the given options and waits for termination signal.
 func run(o *Options) error {
 	klog.Infof("Starting Antrea agent (version %s)", version.GetFullVersion())
@@ -147,7 +149,10 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.Multicluster),
 	)
 
-	_, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR)
+	var serviceCIDRNet *net.IPNet
+	if o.nodeType == config.K8sNode {
+		_, serviceCIDRNet, _ = net.ParseCIDR(o.config.ServiceCIDR)
+	}
 	var serviceCIDRNetv6 *net.IPNet
 	if o.config.ServiceCIDRv6 != "" {
 		_, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6)
@@ -234,6 +239,7 @@ func run(o *Options) error {
 		serviceConfig,
 		networkReadyCh,
 		stopCh,
+		o.nodeType,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),
 		o.config.AntreaProxy.ProxyAll,
 		connectUplinkToBridge)
@@ -250,19 +256,22 @@ func run(o *Options) error {
 		ipsecCertController = ipseccertificate.NewIPSecCertificateController(k8sClient, ovsBridgeClient, nodeConfig.Name)
 	}
 
-	nodeRouteController := noderoute.NewNodeRouteController(
-		k8sClient,
-		informerFactory,
-		ofClient,
-		ovsBridgeClient,
-		routeClient,
-		ifaceStore,
-		networkConfig,
-		nodeConfig,
-		agentInitializer.GetWireGuardClient(),
-		o.config.AntreaProxy.ProxyAll,
-		ipsecCertController,
-	)
+	var nodeRouteController *noderoute.Controller
+	if o.nodeType == config.K8sNode {
+		nodeRouteController = noderoute.NewNodeRouteController(
+			k8sClient,
+			informerFactory,
+			ofClient,
+			ovsBridgeClient,
+			routeClient,
+			ifaceStore,
+			networkConfig,
+			nodeConfig,
+			agentInitializer.GetWireGuardClient(),
+			o.config.AntreaProxy.ProxyAll,
+			ipsecCertController,
+		)
+	}
 
 	var mcRouteController *mcroute.MCRouteController
 	var mcInformerFactory mcinformers.SharedInformerFactory
@@ -403,33 +412,36 @@ func run(o *Options) error {
 		}
 	}
 
-	isChaining := false
-	if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
-		isChaining = true
-	}
-	cniServer := cniserver.New(
-		o.config.CNISocket,
-		o.config.HostProcPathPrefix,
-		nodeConfig,
-		k8sClient,
-		routeClient,
-		isChaining,
-		enableBridgingMode,
-		enableAntreaIPAM,
-		o.config.DisableTXChecksumOffload,
-		networkReadyCh)
-
+	var cniServer *cniserver.CNIServer
 	var cniPodInfoStore cnipodcache.CNIPodInfoStore
-	if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
-		cniPodInfoStore = cnipodcache.NewCNIPodInfoStore()
-		err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, cniPodInfoStore)
-		if err != nil {
-			return fmt.Errorf("error initializing CNI server with cniPodInfoStore cache: %v", err)
+	if o.nodeType == config.K8sNode {
+		isChaining := false
+		if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
+			isChaining = true
 		}
-	} else {
-		err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, nil)
-		if err != nil {
-			return fmt.Errorf("error initializing CNI server: %v", err)
+		cniServer = cniserver.New(
+			o.config.CNISocket,
+			o.config.HostProcPathPrefix,
+			nodeConfig,
+			k8sClient,
+			routeClient,
+			isChaining,
+			enableBridgingMode,
+			enableAntreaIPAM,
+			o.config.DisableTXChecksumOffload,
+			networkReadyCh)
+
+		if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) {
+			cniPodInfoStore = cnipodcache.NewCNIPodInfoStore()
+			err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, cniPodInfoStore)
+			if err != nil {
+				return fmt.Errorf("error initializing CNI server with cniPodInfoStore cache: %v", err)
+			}
+		} else {
+			err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, nil)
+			if err != nil {
+				return fmt.Errorf("error initializing CNI server: %v", err)
+			}
 		}
 	}
 
@@ -519,11 +531,17 @@ func run(o *Options) error {
 
 	log.StartLogFileNumberMonitor(stopCh)
 
-	go podUpdateChannel.Run(stopCh)
-
-	go routeClient.Run(stopCh)
+	if o.nodeType == config.K8sNode {
+		go routeClient.Run(stopCh)
+		go podUpdateChannel.Run(stopCh)
+		go cniServer.Run(stopCh)
+		go nodeRouteController.Run(stopCh)
+	}
 
-	go cniServer.Run(stopCh)
+	if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec &&
+		networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert {
+		go ipsecCertController.Run(stopCh)
+	}
 
 	go antreaClientProvider.Run(ctx)
 
@@ -532,8 +550,6 @@ func run(o *Options) error {
 		go ipsecCertController.Run(stopCh)
 	}
 
-	go nodeRouteController.Run(stopCh)
-
 	go networkPolicyController.Run(stopCh)
 	// Initialize the NPL agent.
 	if enableNodePortLocal {
@@ -691,11 +707,16 @@ func run(o *Options) error {
 	if err != nil {
 		return fmt.Errorf("error generating Cipher Suite list: %v", err)
 	}
+	bindAddress := net.IPv4zero
+	if o.nodeType == config.ExternalNode {
+		bindAddress = ipv4Localhost
+	}
 	apiServer, err := apiserver.New(
 		agentQuerier,
 		networkPolicyController,
 		mcastController,
 		externalIPController,
+		bindAddress,
 		o.config.APIPort,
 		*o.config.EnablePrometheusMetrics,
 		o.config.ClientConnection.Kubeconfig,