Add global except list for egress to avoid SNAT (antrea-io#2707)

For some environment, Pod can communicate with some destination (not podCIDR/svcCIDR) directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li <yang.li@transwarp.io>
leonstack · Sep 17, 2021 · 9a5337f · 9a5337f
1 parent e32664e
commit 9a5337f
Show file tree

Hide file tree

Showing 15 changed files with 118 additions and 37 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -3856,6 +3856,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4054,7 +4058,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dtc759g79k
+  name: antrea-config-md98g8d8md
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4129,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dtc759g79k
+          value: antrea-config-md98g8d8md
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4180,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dtc759g79k
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4457,7 +4461,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-dtc759g79k
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -3856,6 +3856,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4054,7 +4058,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dtc759g79k
+  name: antrea-config-md98g8d8md
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4129,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dtc759g79k
+          value: antrea-config-md98g8d8md
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4180,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dtc759g79k
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4459,7 +4463,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-dtc759g79k
+          name: antrea-config-md98g8d8md
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -3856,6 +3856,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack
     # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by
     # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed.
@@ -4054,7 +4058,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-65f7gf8456
+  name: antrea-config-cckd925724
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4125,7 +4129,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-65f7gf8456
+          value: antrea-config-cckd925724
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4176,7 +4180,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-65f7gf8456
+          name: antrea-config-cckd925724
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4460,7 +4464,7 @@ spec:
           path: /home/kubernetes/bin
         name: host-cni-bin
       - configMap:
-          name: antrea-config-65f7gf8456
+          name: antrea-config-cckd925724
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -3856,6 +3856,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4059,7 +4063,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-fcd8c2h5b5
+  name: antrea-config-hc46gctkhm
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4139,7 +4143,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-fcd8c2h5b5
+          value: antrea-config-hc46gctkhm
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4190,7 +4194,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-fcd8c2h5b5
+          name: antrea-config-hc46gctkhm
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4506,7 +4510,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-fcd8c2h5b5
+          name: antrea-config-hc46gctkhm
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -3856,6 +3856,10 @@ data:
     #  The port for WireGuard to receive traffic.
     #  port: 51820
 
+    egress:
+    #  exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+    #  exceptCIDRs: []
+
     # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
     # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
     # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
@@ -4059,7 +4063,7 @@ metadata:
   annotations: {}
   labels:
     app: antrea
-  name: antrea-config-dhb74b822t
+  name: antrea-config-72bf28b52t
   namespace: kube-system
 ---
 apiVersion: v1
@@ -4130,7 +4134,7 @@ spec:
             fieldRef:
               fieldPath: spec.serviceAccountName
         - name: ANTREA_CONFIG_MAP_NAME
-          value: antrea-config-dhb74b822t
+          value: antrea-config-72bf28b52t
         image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest
         imagePullPolicy: IfNotPresent
         livenessProbe:
@@ -4181,7 +4185,7 @@ spec:
         key: node-role.kubernetes.io/master
       volumes:
       - configMap:
-          name: antrea-config-dhb74b822t
+          name: antrea-config-72bf28b52t
         name: antrea-config
       - name: antrea-controller-tls
         secret:
@@ -4462,7 +4466,7 @@ spec:
         operator: Exists
       volumes:
       - configMap:
-          name: antrea-config-dhb74b822t
+          name: antrea-config-72bf28b52t
         name: antrea-config
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf
@@ -93,6 +93,10 @@ wireGuard:
 #  The port for WireGuard to receive traffic.
 #  port: 51820
 
+egress:
+#  The cidrs will ignore SNAT action when Egress enabled
+#  exceptCIDRs: []
+
 # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
 # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
 # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -134,6 +134,14 @@ func run(o *Options) error {
 	wireguardConfig := &config.WireGuardConfig{
 		Port: o.config.WireGuard.Port,
 	}
+	exceptCIDRs := []net.IPNet{}
+	for _, cidr := range o.config.Egress.ExceptCIDRs {
+		_, exceptCIDR, _ := net.ParseCIDR(cidr)
+		exceptCIDRs = append(exceptCIDRs, *exceptCIDR)
+	}
+	egressConfig := &config.EgressConfig{
+		ExceptCIDRs: exceptCIDRs,
+	}
 	routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll)
 	if err != nil {
 		return fmt.Errorf("error creating route client: %v", err)
@@ -173,6 +181,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		wireguardConfig,
+		egressConfig,
 		networkReadyCh,
 		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy),

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -176,6 +176,8 @@ type AgentConfig struct {
 	TransportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"`
 	// AntreaProxy contains AntreaProxy related configuration options.
 	AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"`
+	// Egress related configurations.
+	Egress EgressConfig `yaml:"egress"`
 }
 
 type AntreaProxyConfig struct {
@@ -192,3 +194,8 @@ type WireGuardConfig struct {
 	// The port for the WireGuard to receive traffic. Defaults to 51820.
 	Port int `yaml:"port,omitempty"`
 }
+
+type EgressConfig struct {
+	// exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress.
+	ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"`
+}
diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -156,6 +156,14 @@ func (o *Options) validate(args []string) error {
 	if err := o.validateFlowExporterConfig(); err != nil {
 		return fmt.Errorf("failed to validate flow exporter config: %v", err)
 	}
+	if features.DefaultFeatureGate.Enabled(features.Egress) {
+		for _, cidr := range o.config.Egress.ExceptCIDRs {
+			_, _, err := net.ParseCIDR(cidr)
+			if err != nil {
+				return fmt.Errorf("Egress Except CIDR %s is invalid", cidr)
+			}
+		}
+	}
 	return nil
 }
 

diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
@@ -88,6 +88,7 @@ type Initializer struct {
 	networkConfig   *config.NetworkConfig
 	nodeConfig      *config.NodeConfig
 	wireGuardConfig *config.WireGuardConfig
+	egressConfig    *config.EgressConfig
 	enableProxy     bool
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
@@ -111,6 +112,7 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	wireGuardConfig *config.WireGuardConfig,
+	egressConfig *config.EgressConfig,
 	networkReadyCh chan<- struct{},
 	stopCh <-chan struct{},
 	enableProxy bool,
@@ -132,6 +134,7 @@ func NewInitializer(
 		serviceCIDRv6:         serviceCIDRv6,
 		networkConfig:         networkConfig,
 		wireGuardConfig:       wireGuardConfig,
+		egressConfig:          egressConfig,
 		networkReadyCh:        networkReadyCh,
 		stopCh:                stopCh,
 		enableProxy:           enableProxy,
@@ -385,7 +388,7 @@ func (i *Initializer) initOpenFlowPipeline() error {
 
 	// Install OpenFlow entries to enable Pod traffic to external IP
 	// addresses.
-	if err := i.ofClient.InstallExternalFlows(); err != nil {
+	if err := i.ofClient.InstallExternalFlows(i.egressConfig.ExceptCIDRs); err != nil {
 		klog.Errorf("Failed to install openflow entries for external connectivity: %v", err)
 		return err
 	}

diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go
@@ -92,6 +92,10 @@ type WireGuardConfig struct {
 	MTU int
 }
 
+type EgressConfig struct {
+	ExceptCIDRs []net.IPNet
+}
+
 // Local Node configurations retrieved from K8s API or host networking state.
 type NodeConfig struct {
 	// The Node's name used in Kubernetes.
@@ -127,6 +131,8 @@ type NodeConfig struct {
 	UplinkNetConfig *AdapterNetConfig
 	// The config of the WireGuard interface.
 	WireGuardConfig *WireGuardConfig
+	// The config of the Egress feature.
+	EgressConfig *EgressConfig
 }
 
 func (n *NodeConfig) String() string {

diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go
@@ -155,7 +155,7 @@ type Client interface {
 	// Pods to the external IP address, and mark the packets to be SNAT'd
 	// with the configured SNAT IPs. On Windows Node, the flows also perform
 	// SNAT with the Openflow NAT action.
-	InstallExternalFlows() error
+	InstallExternalFlows(exceptCIDRs []net.IPNet) error
 
 	// InstallSNATMarkFlows installs flows for a local SNAT IP. On Linux, a
 	// single flow is added to mark the packets tunnelled from remote Nodes
@@ -791,15 +791,24 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo
 	return connCh, c.initialize()
 }
 
-func (c *client) InstallExternalFlows() error {
+func (c *client) InstallExternalFlows(exceptCIDRs []net.IPNet) error {
 	localGatewayMAC := c.nodeConfig.GatewayConfig.MAC
 
 	var flows []binding.Flow
+	var ipv4CIDRs []net.IPNet
+	var ipv6CIDRs []net.IPNet
+	for _, cidr := range exceptCIDRs {
+		if cidr.IP.To4() == nil {
+			ipv6CIDRs = append(ipv6CIDRs, cidr)
+		} else {
+			ipv4CIDRs = append(ipv4CIDRs, cidr)
+		}
+	}
 	if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil {
-		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC)
+		flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs)
 	}
 	if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil {
-		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...)
+		flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...)
 	}
 	if err := c.ofEntryOperations.AddAll(flows); err != nil {
 		return fmt.Errorf("failed to install flows for external communication: %v", err)