Add global except list for egress to avoid SNAT (antrea-io#2707)

For some environment, some destination(not podCIDR/svcCIDR) can be communicate with each other directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li <yang.li@transwarp.io>
leonstack · Sep 13, 2021 · 00a59f8 · 00a59f8
1 parent dded211
commit 00a59f8
Show file tree

Hide file tree

Showing 15 changed files with 112 additions and 37 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -3996,7 +4000,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-99c875tk88
+  name: antrea-config-md98g8d8md
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4067,7 +4071,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-99c875tk88
+          value: antrea-config-md98g8d8md
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4118,7 +4122,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-99c875tk88
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4399,7 +4403,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-99c875tk88
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -3996,7 +4000,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-99c875tk88
+  name: antrea-config-md98g8d8md
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4067,7 +4071,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-99c875tk88
+          value: antrea-config-md98g8d8md
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4118,7 +4122,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-99c875tk88
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4401,7 +4405,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-99c875tk88
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -3996,7 +4000,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dbmkcb65c8
+  name: antrea-config-cckd925724
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4067,7 +4071,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dbmkcb65c8
+          value: antrea-config-cckd925724
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4118,7 +4122,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dbmkcb65c8
+          name: antrea-config-cckd925724
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4402,7 +4406,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-dbmkcb65c8
+          name: antrea-config-cckd925724
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4001,7 +4005,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-tthkbhb7k5
+  name: antrea-config-hc46gctkhm
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4081,7 +4085,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-tthkbhb7k5
+          value: antrea-config-hc46gctkhm
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4132,7 +4136,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-tthkbhb7k5
+          name: antrea-config-hc46gctkhm
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4448,7 +4452,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-tthkbhb7k5
+          name: antrea-config-hc46gctkhm
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3848,6 +3848,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egressExcept:
+    #  The cidrs will ignore SNAT action when Egress enabled
+    #  cidrs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4001,7 +4005,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-mc8h75hbgg
+  name: antrea-config-72bf28b52t
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4072,7 +4076,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-mc8h75hbgg
+          value: antrea-config-72bf28b52t
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4123,7 +4127,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-mc8h75hbgg
+          name: antrea-config-72bf28b52t
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4404,7 +4408,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-mc8h75hbgg
+          name: antrea-config-72bf28b52t
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -93,6 +93,10 @@ wireGuard:
 #  The port for WireGuard to receive traffic.
 #  port: 51820
 
+egressExcept:
+#  The cidrs will ignore SNAT action when Egress enabled
+#  cidrs: []
+
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
 # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -129,6 +129,14 @@ func run(o *Options) error {
 	wireguardConfig := &config.WireGuardConfig{
 		Port: o.config.WireGuard.Port,
 	}
+	excidrs := []net.IPNet{}
+	for _, cidr := range o.config.EgressExcept.CIDRs {
+		_, excidr, _ := net.ParseCIDR(cidr)
+		excidrs = append(excidrs, *excidr)
+	}
+	egressExceptConfig := &config.EgressExceptConfig{
+		CIDRs: excidrs,
+	}
 	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
@@ -158,6 +166,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		wireguardConfig,
+		egressExceptConfig,
 		networkReadyCh,
 		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -162,9 +162,15 @@ type AgentConfig struct {
 	// If there are multiple IP addresses configured on the interface, the first one is used.
 	// The interface configured with Node IP is used if this parameter is not set.
 	TransportInterface string `yaml:"transportInterface,omitempty"`
+	// Egress related configurations.
+	EgressExcept EgressExceptConfig `yaml:"egressExcept"`
 }
 
 type WireGuardConfig struct {
 	// The port for the WireGuard to receive traffic. Defaults to 51820.
 	Port int `yaml:"port,omitempty"`
 }
+
+type EgressExceptConfig struct {
+	CIDRs []string `yaml:"cidrs,omitempty"`
+}
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -153,6 +153,14 @@ func (o *Options) validate(args []string) error {
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
+	if features.DefaultFeatureGate.Enabled(features.Egress) {
+		for _, cidr := range o.config.EgressExcept.CIDRs {
+			_, _, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("Egress Except CIDR %s is invalid", cidr)
+			}
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -83,6 +83,7 @@ type Initializer struct {
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	wireGuardConfig *config.WireGuardConfig
+	egressExceptConfig *config.EgressExceptConfig
 	enableProxy     bool
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
@@ -103,6 +104,7 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	wireGuardConfig *config.WireGuardConfig,
+	egressExceptConfig *config.EgressExceptConfig,
 	networkReadyCh chan<- struct{},
 	stopCh <-chan struct{},
 	enableProxy bool) *Initializer {
@@ -119,6 +121,7 @@ func NewInitializer(
 		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		wireGuardConfig: wireGuardConfig,
+		egressExceptConfig: egressExceptConfig,
 		networkReadyCh:  networkReadyCh,
 		stopCh:          stopCh,
 		enableProxy:     enableProxy,
@@ -355,7 +358,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 	roundInfo := getRoundInfo(i.ovsBridgeClient)
 
 	// Set up all basic flows.
-	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig)
+	ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig, i.egressExceptConfig)
 	if err != nil {
 		klog.Errorf("Failed to initialize openflow client: %v", err)
 		return err

diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go
@@ -80,6 +80,10 @@ type WireGuardConfig struct {
 	MTU int
 }
 
+type EgressExceptConfig struct {
+	CIDRs []net.IPNet
+}
+
 // Local Node configurations retrieved from K8s API or host networking state.
 type NodeConfig struct {
 	// The Node's name used in Kubernetes.
@@ -115,6 +119,7 @@ type NodeConfig struct {
 	UplinkNetConfig *AdapterNetConfig
 	// The config of the WireGuard interface.
 	WireGuardConfig *WireGuardConfig
+	EgressExceptConfig *EgressExceptConfig
 }
 
 func (n *NodeConfig) String() string {

diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go
@@ -40,7 +40,7 @@ type Client interface {
 	// be called to ensure that the set of OVS flows is correct. All flows programmed in the
 	// switch which match the current round number will be deleted before any new flow is
 	// installed.
-	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig) (<-chan struct{}, error)
+	Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error)
 
 	// InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist.
 	InstallGatewayFlows() error
@@ -729,9 +729,10 @@ func (c *client) initialize() error {
 	return nil
 }
 
-func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig) (<-chan struct{}, error) {
+func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error) {
 	c.nodeConfig = nodeConfig
 	c.networkConfig = networkConfig
+	c.egressExceptConfig = egressExceptConfig
 
 	if config.IsIPv4Enabled(nodeConfig, c.networkConfig.TrafficEncapMode) {
 		c.ipProtocols = append(c.ipProtocols, binding.ProtocolIP)
@@ -778,11 +779,20 @@ func (c *client) InstallExternalFlows() error {
 	localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
 
 	var flows []binding.Flow
+	var ipv4CIDRs []net.IPNet
+	var ipv6CIDRs []net.IPNet
+	for _, cidr := range c.egressExceptConfig.CIDRs{
+		if cidr.IP.To4() == nil {
+			ipv6CIDRs = append(ipv6CIDRs, cidr)
+		} else {
+			ipv4CIDRs = append(ipv4CIDRs, cidr)
+		}
+	}
 	if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil {
-		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
+		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs)
 	}
 	if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil {
-		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...)
 	}
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return fmt.Errorf("failed to install flows for external communication: %v", err)