Add global except list for egress to avoid SNAT (antrea-io#2707)

For some environment, some destination(not podCIDR/svcCIDR) can be
communicate with each other directly for better network performance,
we should avoid SNAT for such destination.

Signed-off-by: Yang Li yang.li@transwarp.io

build/yamls/antrea-aks.yml

@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

build/yamls/antrea-eks.yml

@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

build/yamls/antrea-gke.yml

@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.

build/yamls/antrea-ipsec.yml

@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

build/yamls/antrea.yml

@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

build/yamls/base/conf/antrea-agent.conf

@@ -93,6 +93,10 @@ wireGuard:
 #  The port for WireGuard to receive traffic.
 #  port: 51820
 
+egressExcept:
+#  The cidrs will ignore SNAT action when Egress enabled
+#  cidrs: []
+
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
 # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

cmd/antrea-agent/agent.go

@@ -129,6 +129,14 @@ func run(o *Options) error {
 	wireguardConfig := &config.WireGuardConfig{
 		Port: o.config.WireGuard.Port,
 	}
+	excidrs := []net.IPNet{}
+	for _, cidr := range o.config.EgressExcept.CIDRs {
+		_, excidr, _ := net.ParseCIDR(cidr)
+		excidrs = append(excidrs, *excidr)
+	}
+	egressExceptConfig := &config.EgressExceptConfig{
+		CIDRs: excidrs,
+	}
 	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
@@ -158,6 +166,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		wireguardConfig,
+		egressExceptConfig,
 		networkReadyCh,
 		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))

cmd/antrea-agent/config.go

@@ -162,9 +162,15 @@ type AgentConfig struct {
 	// If there are multiple IP addresses configured on the interface, the first one is used.
 	// The interface configured with Node IP is used if this parameter is not set.
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// Egress related configurations.
+	EgressExcept EgressExceptConfig `yaml:"egressExcept"`
 }
 
 type WireGuardConfig struct {
 	// The port for the WireGuard to receive traffic. Defaults to 51820.
 	Port int `yaml:"port,omitempty"`
 }
+
+type EgressExceptConfig struct {
+	CIDRs []string `yaml:"cidrs,omitempty"`
+}

cmd/antrea-agent/options.go

@@ -153,6 +153,14 @@ func (o *Options) validate(args []string) error {
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
+	if features.DefaultFeatureGate.Enabled(features.Egress) {
+		for _, cidr := range o.config.EgressExcept.CIDRs {
+			_, _, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("Egress Except CIDR %s is invalid", cidr)
+			}
+		}
+	}
 	return nil
 }
 

pkg/agent/agent.go

@@ -69,21 +69,22 @@ var getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName
 
 // Initializer knows how to setup host networking, OpenVSwitch, and Openflow.
 type Initializer struct {
-	client          clientset.Interface
-	ovsBridgeClient ovsconfig.OVSBridgeClient
-	ofClient        openflow.Client
-	routeClient     route.Interface
-	wireGuardClient wireguard.Interface
-	ifaceStore      interfacestore.InterfaceStore
-	ovsBridge       string
-	hostGateway     string // name of gateway port on the OVS bridge
-	mtu             int
-	serviceCIDR     *net.IPNet // K8s Service ClusterIP CIDR
-	serviceCIDRv6   *net.IPNet // K8s Service ClusterIP CIDR in IPv6
-	networkConfig   *config.NetworkConfig
-	nodeConfig      *config.NodeConfig
-	wireGuardConfig *config.WireGuardConfig
-	enableProxy     bool
+	client             clientset.Interface
+	ovsBridgeClient    ovsconfig.OVSBridgeClient
+	ofClient           openflow.Client
+	routeClient        route.Interface
+	wireGuardClient    wireguard.Interface
+	ifaceStore         interfacestore.InterfaceStore
+	ovsBridge          string
+	hostGateway        string // name of gateway port on the OVS bridge
+	mtu                int
+	serviceCIDR        *net.IPNet // K8s Service ClusterIP CIDR
+	serviceCIDRv6      *net.IPNet // K8s Service ClusterIP CIDR in IPv6
+	networkConfig      *config.NetworkConfig
+	nodeConfig         *config.NodeConfig
+	wireGuardConfig    *config.WireGuardConfig
+	egressExceptConfig *config.EgressExceptConfig
+	enableProxy        bool
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
 	networkReadyCh chan<- struct{}
@@ -103,25 +104,27 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	wireGuardConfig *config.WireGuardConfig,
+	egressExceptConfig *config.EgressExceptConfig,
 	networkReadyCh chan<- struct{},
 	stopCh <-chan struct{},
 	enableProxy bool) *Initializer {
 	return &Initializer{
-		ovsBridgeClient: ovsBridgeClient,
-		client:          k8sClient,
-		ifaceStore:      ifaceStore,
-		ofClient:        ofClient,
-		routeClient:     routeClient,
-		ovsBridge:       ovsBridge,
-		hostGateway:     hostGateway,
-		mtu:             mtu,
-		serviceCIDR:     serviceCIDR,
-		serviceCIDRv6:   serviceCIDRv6,
-		networkConfig:   networkConfig,
-		wireGuardConfig: wireGuardConfig,
-		networkReadyCh:  networkReadyCh,
-		stopCh:          stopCh,
-		enableProxy:     enableProxy,
+		ovsBridgeClient:    ovsBridgeClient,
+		client:             k8sClient,
+		ifaceStore:         ifaceStore,
+		ofClient:           ofClient,
+		routeClient:        routeClient,
+		ovsBridge:          ovsBridge,
+		hostGateway:        hostGateway,
+		mtu:                mtu,
+		serviceCIDR:        serviceCIDR,
+		serviceCIDRv6:      serviceCIDRv6,
+		networkConfig:      networkConfig,
+		wireGuardConfig:    wireGuardConfig,
+		egressExceptConfig: egressExceptConfig,
+		networkReadyCh:     networkReadyCh,
+		stopCh:             stopCh,
+		enableProxy:        enableProxy,
 	}
 }
 
@@ -355,7 +358,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 	roundInfo := getRoundInfo(i.ovsBridgeClient)
 
 	// Set up all basic flows.
-	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig)
+	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig, i.egressExceptConfig)
 	if err != nil {
 		klog.Errorf("Failed to initialize openflow client: %v", err)
 		return err

pkg/agent/config/node_config.go

@@ -80,6 +80,10 @@ type WireGuardConfig struct {
 	MTU int
 }
 
+type EgressExceptConfig struct {
+	CIDRs []net.IPNet
+}
+
 // Local Node configurations retrieved from K8s API or host networking state.
 type NodeConfig struct {
 	// The Node's name used in Kubernetes.
@@ -114,7 +118,8 @@ type NodeConfig struct {
 	// The config of the OVS bridge uplink interface. Only for Windows Node.
 	UplinkNetConfig *AdapterNetConfig
 	// The config of the WireGuard interface.
-	WireGuardConfig *WireGuardConfig
+	WireGuardConfig    *WireGuardConfig
+	EgressExceptConfig *EgressExceptConfig
 }
 
 func (n *NodeConfig) String() string {

pkg/agent/openflow/client.go

@@ -40,7 +40,7 @@ type Client interface {
 	// be called to ensure that the set of OVS flows is correct. All flows programmed in the
 	// switch which match the current round number will be deleted before any new flow is
 	// installed.
-	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig) (<-chan struct{}, error)
+	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error)
 
 	// InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist.
 	InstallGatewayFlows() error
@@ -729,9 +729,10 @@ func (c *client) initialize() error {
 	return nil
 }
 
-func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig) (<-chan struct{}, error) {
+func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error) {
 	c.nodeConfig = nodeConfig
 	c.networkConfig = networkConfig
+	c.egressExceptConfig = egressExceptConfig
 
 	if config.IsIPv4Enabled(nodeConfig, c.networkConfig.TrafficEncapMode) {
 		c.ipProtocols = append(c.ipProtocols, binding.ProtocolIP)
@@ -778,11 +779,20 @@ func (c *client) InstallExternalFlows() error {
 	localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
 
 	var flows []binding.Flow
+	var ipv4CIDRs []net.IPNet
+	var ipv6CIDRs []net.IPNet
+	for _, cidr := range c.egressExceptConfig.CIDRs {
+		if cidr.IP.To4() == nil {
+			ipv6CIDRs = append(ipv6CIDRs, cidr)
+		} else {
+			ipv4CIDRs = append(ipv4CIDRs, cidr)
+		}
+	}
 	if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil {
-		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
+		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs)
 	}
 	if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil {
-		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...)
 	}
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return fmt.Errorf("failed to install flows for external communication: %v", err)

pkg/agent/openflow/pipeline.go

@@ -319,10 +319,11 @@ type client struct {
 	// conjMatchFlowContext.
 	globalConjMatchFlowCache map[string]*conjMatchFlowContext
 	// replayMutex provides exclusive access to the OFSwitch to the ReplayFlows method.
-	replayMutex   sync.RWMutex
-	nodeConfig    *config.NodeConfig
-	networkConfig *config.NetworkConfig
-	gatewayOFPort uint32
+	replayMutex        sync.RWMutex
+	nodeConfig         *config.NodeConfig
+	networkConfig      *config.NetworkConfig
+	egressExceptConfig *config.EgressExceptConfig
+	gatewayOFPort      uint32
 	// ovsDatapathType is the type of the datapath used by the bridge.
 	ovsDatapathType ovsconfig.OVSDatapathType
 	// ovsMetersAreSupported indicates whether the OVS datapath supports OpenFlow meters.
@@ -1857,7 +1858,7 @@ func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Catego
 // snatCommonFlows installs the default flows for performing SNAT for traffic to
 // the external network. The flows identify the packets to external, and send
 // them to snatTable, where SNAT IPs are looked up for the packets.
-func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow {
+func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet, category cookie.Category) []binding.Flow {
 	l3FwdTable := c.pipeline[l3ForwardingTable]
 	nextTable := l3FwdTable.GetNext()
 	ipProto := getIPProtocol(localSubnet.IP)
@@ -1910,6 +1911,15 @@ func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGate
 			Cookie(c.cookieAllocator.Request(category).Raw()).
 			Done(),
 	}
+	for _, cidr := range exceptCIDRs {
+		flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).
+			MatchProtocol(ipProto).
+			MatchRegMark(FromLocalRegMark).
+			MatchDstIPNet(cidr).
+			Action().GotoTable(nextTable).
+			Cookie(c.cookieAllocator.Request(category).Raw()).
+			Done())
+	}
 	return flows
 }
 
@@ -2174,11 +2184,11 @@ func (c *client) decTTLFlows(category cookie.Category) []binding.Flow {
 }
 
 // externalFlows returns the flows needed to enable SNAT for external traffic.
-func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow {
+func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, cidrs []net.IPNet) []binding.Flow {
 	if !c.enableEgress {
 		return nil
 	}
-	return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT)
+	return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cidrs, cookie.SNAT)
 }
 
 // policyConjKeyFuncKeyFunc knows how to get key of a *policyRuleConjunction.