From 9a5337f8ed046d57dee6979efcfbe8c52e3bca24 Mon Sep 17 00:00:00 2001 From: Yang Li Date: Mon, 13 Sep 2021 11:08:20 +0800 Subject: [PATCH] Add global except list for egress to avoid SNAT (#2707) For some environment, Pod can communicate with some destination (not podCIDR/svcCIDR) directly for better network performance, we should avoid SNAT for such destination. Signed-off-by: Yang Li --- build/yamls/antrea-aks.yml | 12 ++++++++---- build/yamls/antrea-eks.yml | 12 ++++++++---- build/yamls/antrea-gke.yml | 12 ++++++++---- build/yamls/antrea-ipsec.yml | 12 ++++++++---- build/yamls/antrea.yml | 12 ++++++++---- build/yamls/base/conf/antrea-agent.conf | 4 ++++ cmd/antrea-agent/agent.go | 9 +++++++++ cmd/antrea-agent/config.go | 7 +++++++ cmd/antrea-agent/options.go | 8 ++++++++ pkg/agent/agent.go | 5 ++++- pkg/agent/config/node_config.go | 6 ++++++ pkg/agent/openflow/client.go | 17 +++++++++++++---- pkg/agent/openflow/pipeline.go | 16 ++++++++++++---- pkg/agent/openflow/testing/mock_openflow.go | 8 ++++---- test/integration/agent/openflow_test.go | 15 +++++++++++---- 15 files changed, 118 insertions(+), 37 deletions(-) diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 1fa4bd36db4..054c9bc2976 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3856,6 +3856,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4054,7 +4058,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md namespace: kube-system --- apiVersion: v1 @@ -4125,7 +4129,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-dtc759g79k + value: antrea-config-md98g8d8md image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4176,7 +4180,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md name: antrea-config - name: antrea-controller-tls secret: @@ -4457,7 +4461,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index b9c08c2b8e9..56efd5a70f4 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3856,6 +3856,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4054,7 +4058,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md namespace: kube-system --- apiVersion: v1 @@ -4125,7 +4129,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-dtc759g79k + value: antrea-config-md98g8d8md image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4176,7 +4180,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md name: antrea-config - name: antrea-controller-tls secret: @@ -4459,7 +4463,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-dtc759g79k + name: antrea-config-md98g8d8md name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 54acac03fd4..2cbc0607d52 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3856,6 +3856,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. @@ -4054,7 +4058,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-65f7gf8456 + name: antrea-config-cckd925724 namespace: kube-system --- apiVersion: v1 @@ -4125,7 +4129,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-65f7gf8456 + value: antrea-config-cckd925724 image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4176,7 +4180,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-65f7gf8456 + name: antrea-config-cckd925724 name: antrea-config - name: antrea-controller-tls secret: @@ -4460,7 +4464,7 @@ spec: path: /home/kubernetes/bin name: host-cni-bin - configMap: - name: antrea-config-65f7gf8456 + name: antrea-config-cckd925724 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index ec166343f91..6d9b48dcdf3 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3856,6 +3856,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. @@ -4059,7 +4063,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-fcd8c2h5b5 + name: antrea-config-hc46gctkhm namespace: kube-system --- apiVersion: v1 @@ -4139,7 +4143,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-fcd8c2h5b5 + value: antrea-config-hc46gctkhm image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4190,7 +4194,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-fcd8c2h5b5 + name: antrea-config-hc46gctkhm name: antrea-config - name: antrea-controller-tls secret: @@ -4506,7 +4510,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-fcd8c2h5b5 + name: antrea-config-hc46gctkhm name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 070a2150d60..5a642fac50c 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3856,6 +3856,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egress: + # exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + # exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. @@ -4059,7 +4063,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-dhb74b822t + name: antrea-config-72bf28b52t namespace: kube-system --- apiVersion: v1 @@ -4130,7 +4134,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-dhb74b822t + value: antrea-config-72bf28b52t image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -4181,7 +4185,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-dhb74b822t + name: antrea-config-72bf28b52t name: antrea-config - name: antrea-controller-tls secret: @@ -4462,7 +4466,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-dhb74b822t + name: antrea-config-72bf28b52t name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 76d6d72b613..b91356d4588 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -93,6 +93,10 @@ wireGuard: # The port for WireGuard to receive traffic. # port: 51820 +egress: +# The cidrs will ignore SNAT action when Egress enabled +# exceptCIDRs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 40ed56f3360..ccb906a4177 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -134,6 +134,14 @@ func run(o *Options) error { wireguardConfig := &config.WireGuardConfig{ Port: o.config.WireGuard.Port, } + exceptCIDRs := []net.IPNet{} + for _, cidr := range o.config.Egress.ExceptCIDRs { + _, exceptCIDR, _ := net.ParseCIDR(cidr) + exceptCIDRs = append(exceptCIDRs, *exceptCIDR) + } + egressConfig := &config.EgressConfig{ + ExceptCIDRs: exceptCIDRs, + } routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT, o.config.AntreaProxy.ProxyAll) if err != nil { return fmt.Errorf("error creating route client: %v", err) @@ -173,6 +181,7 @@ func run(o *Options) error { serviceCIDRNetv6, networkConfig, wireguardConfig, + egressConfig, networkReadyCh, stopCh, features.DefaultFeatureGate.Enabled(features.AntreaProxy), diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index fd1c9fe6054..5eda3a92955 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -176,6 +176,8 @@ type AgentConfig struct { TransportInterfaceCIDRs []string `yaml:"transportInterfaceCIDRs,omitempty"` // AntreaProxy contains AntreaProxy related configuration options. AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"` + // Egress related configurations. + Egress EgressConfig `yaml:"egress"` } type AntreaProxyConfig struct { @@ -192,3 +194,8 @@ type WireGuardConfig struct { // The port for the WireGuard to receive traffic. Defaults to 51820. Port int `yaml:"port,omitempty"` } + +type EgressConfig struct { + // exceptCIDRs is the CIDR ranges to which outbound Pod traffic will not be SNATed by Egress. + ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"` +} diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 077a22e16b4..de5677fcab8 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -156,6 +156,14 @@ func (o *Options) validate(args []string) error { if err := o.validateFlowExporterConfig(); err != nil { return fmt.Errorf("failed to validate flow exporter config: %v", err) } + if features.DefaultFeatureGate.Enabled(features.Egress) { + for _, cidr := range o.config.Egress.ExceptCIDRs { + _, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("Egress Except CIDR %s is invalid", cidr) + } + } + } return nil } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 23a4aee9650..0f1ee69bf80 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -88,6 +88,7 @@ type Initializer struct { networkConfig *config.NetworkConfig nodeConfig *config.NodeConfig wireGuardConfig *config.WireGuardConfig + egressConfig *config.EgressConfig enableProxy bool // networkReadyCh should be closed once the Node's network is ready. // The CNI server will wait for it before handling any CNI Add requests. @@ -111,6 +112,7 @@ func NewInitializer( serviceCIDRv6 *net.IPNet, networkConfig *config.NetworkConfig, wireGuardConfig *config.WireGuardConfig, + egressConfig *config.EgressConfig, networkReadyCh chan<- struct{}, stopCh <-chan struct{}, enableProxy bool, @@ -132,6 +134,7 @@ func NewInitializer( serviceCIDRv6: serviceCIDRv6, networkConfig: networkConfig, wireGuardConfig: wireGuardConfig, + egressConfig: egressConfig, networkReadyCh: networkReadyCh, stopCh: stopCh, enableProxy: enableProxy, @@ -385,7 +388,7 @@ func (i *Initializer) initOpenFlowPipeline() error { // Install OpenFlow entries to enable Pod traffic to external IP // addresses. - if err := i.ofClient.InstallExternalFlows(); err != nil { + if err := i.ofClient.InstallExternalFlows(i.egressConfig.ExceptCIDRs); err != nil { klog.Errorf("Failed to install openflow entries for external connectivity: %v", err) return err } diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 189f57f6947..4073542b81e 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -92,6 +92,10 @@ type WireGuardConfig struct { MTU int } +type EgressConfig struct { + ExceptCIDRs []net.IPNet +} + // Local Node configurations retrieved from K8s API or host networking state. type NodeConfig struct { // The Node's name used in Kubernetes. @@ -127,6 +131,8 @@ type NodeConfig struct { UplinkNetConfig *AdapterNetConfig // The config of the WireGuard interface. WireGuardConfig *WireGuardConfig + // The config of the Egress feature. + EgressConfig *EgressConfig } func (n *NodeConfig) String() string { diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index eaea01da8eb..0593797f0f1 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -155,7 +155,7 @@ type Client interface { // Pods to the external IP address, and mark the packets to be SNAT'd // with the configured SNAT IPs. On Windows Node, the flows also perform // SNAT with the Openflow NAT action. - InstallExternalFlows() error + InstallExternalFlows(exceptCIDRs []net.IPNet) error // InstallSNATMarkFlows installs flows for a local SNAT IP. On Linux, a // single flow is added to mark the packets tunnelled from remote Nodes @@ -791,15 +791,24 @@ func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeCo return connCh, c.initialize() } -func (c *client) InstallExternalFlows() error { +func (c *client) InstallExternalFlows(exceptCIDRs []net.IPNet) error { localGatewayMAC := c.nodeConfig.GatewayConfig.MAC var flows []binding.Flow + var ipv4CIDRs []net.IPNet + var ipv6CIDRs []net.IPNet + for _, cidr := range exceptCIDRs { + if cidr.IP.To4() == nil { + ipv6CIDRs = append(ipv6CIDRs, cidr) + } else { + ipv4CIDRs = append(ipv4CIDRs, cidr) + } + } if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs) } if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...) } if err := c.ofEntryOperations.AddAll(flows); err != nil { return fmt.Errorf("failed to install flows for external communication: %v", err) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index e64e8cc3a80..d77d9c0cc75 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -1859,7 +1859,6 @@ func (c *client) rejectBypassNetworkpolicyFlows(category cookie.Category) (flows Done() apFlows = append(apFlows, apIngressRejFlow) } - } allRejFlows = append(allRejFlows, apFlows...) return allRejFlows @@ -2018,7 +2017,7 @@ func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Catego // snatCommonFlows installs the default flows for performing SNAT for traffic to // the external network. The flows identify the packets to external, and send // them to snatTable, where SNAT IPs are looked up for the packets. -func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { +func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] nextTable := l3FwdTable.GetNext() ipProto := getIPProtocol(localSubnet.IP) @@ -2071,6 +2070,15 @@ func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGate Cookie(c.cookieAllocator.Request(category).Raw()). Done(), } + for _, cidr := range exceptCIDRs { + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal). + MatchProtocol(ipProto). + MatchRegMark(FromLocalRegMark). + MatchDstIPNet(cidr). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } return flows } @@ -2406,11 +2414,11 @@ func (c *client) decTTLFlows(category cookie.Category) []binding.Flow { } // externalFlows returns the flows needed to enable SNAT for external traffic. -func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow { +func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet) []binding.Flow { if !c.enableEgress { return nil } - return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT) + return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, exceptCIDRs, cookie.SNAT) } // policyConjKeyFuncKeyFunc knows how to get key of a *policyRuleConjunction. diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 971a2129a47..37712898905 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -337,17 +337,17 @@ func (mr *MockClientMockRecorder) InstallEndpointFlows(arg0, arg1 interface{}) * } // InstallExternalFlows mocks base method -func (m *MockClient) InstallExternalFlows() error { +func (m *MockClient) InstallExternalFlows(arg0 []net.IPNet) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallExternalFlows") + ret := m.ctrl.Call(m, "InstallExternalFlows", arg0) ret0, _ := ret[0].(error) return ret0 } // InstallExternalFlows indicates an expected call of InstallExternalFlows -func (mr *MockClientMockRecorder) InstallExternalFlows() *gomock.Call { +func (mr *MockClientMockRecorder) InstallExternalFlows(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallExternalFlows", reflect.TypeOf((*MockClient)(nil).InstallExternalFlows)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallExternalFlows", reflect.TypeOf((*MockClient)(nil).InstallExternalFlows), arg0) } // InstallGatewayFlows mocks base method diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 3e7be68e871..8c244a71eb1 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -101,6 +101,8 @@ type testConfig struct { var ( _, podIPv4CIDR, _ = net.ParseCIDR("192.168.1.0/24") _, podIPv6CIDR, _ = net.ParseCIDR("fd74:ca9b:172:19::/64") + _, exceptIPv4CIDR, _ = net.ParseCIDR("192.168.2.0/24") + _, exceptIPv6CIDR, _ = net.ParseCIDR("fd74:ca9b:172:20::/64") ) func TestConnectivityFlows(t *testing.T) { @@ -242,18 +244,19 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { } func testExternalFlows(t *testing.T, config *testConfig) { - if err := c.InstallExternalFlows(); err != nil { + ExceptCIDRs := []net.IPNet{*exceptIPv4CIDR, *exceptIPv6CIDR} + if err := c.InstallExternalFlows(ExceptCIDRs); err != nil { t.Errorf("Failed to install OpenFlow entries to allow Pod to communicate to the external addresses: %v", err) } gwMAC := config.nodeConfig.GatewayConfig.MAC if config.nodeConfig.NodeIPv4Addr != nil && config.nodeConfig.PodIPv4CIDR != nil { - for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv4Addr.IP, config.nodeConfig.PodIPv4CIDR, gwMAC) { + for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv4Addr.IP, config.nodeConfig.PodIPv4CIDR, gwMAC, exceptIPv4CIDR) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } if config.nodeConfig.NodeIPv6Addr != nil && config.nodeConfig.PodIPv6CIDR != nil { - for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv6Addr.IP, config.nodeConfig.PodIPv6CIDR, gwMAC) { + for _, tableFlow := range expectedExternalFlows(config.nodeConfig.NodeIPv6Addr.IP, config.nodeConfig.PodIPv6CIDR, gwMAC, exceptIPv6CIDR) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } } @@ -1324,7 +1327,7 @@ func prepareIPNetAddresses(addresses []string) []types.Address { return ipAddresses } -func expectedExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.HardwareAddr) []expectTableFlows { +func expectedExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.HardwareAddr, exceptCIDR *net.IPNet) []expectTableFlows { var ipProtoStr, nwDstFieldName string if localSubnet.IP.To4() != nil { ipProtoStr = "ip" @@ -1346,6 +1349,10 @@ func expectedExternalFlows(nodeIP net.IP, localSubnet *net.IPNet, gwMAC net.Hard MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x2/0xf,%s=%s", ipProtoStr, nwDstFieldName, nodeIP.String()), ActStr: "goto_table:80", }, + { + MatchStr: fmt.Sprintf("priority=200,%s,reg0=0x2/0xf,%s=%s", ipProtoStr, nwDstFieldName, exceptCIDR.String()), + ActStr: "goto_table:80", + }, { MatchStr: fmt.Sprintf("priority=190,%s,reg0=0x2/0xf", ipProtoStr), ActStr: "goto_table:71",