diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 6a823bbe332..ad14d6e673c 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3848,6 +3848,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egressExcept: + # The cidrs will ignore SNAT action when Egress enabled + # cidrs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 6d246c3fa7c..c7235aa8243 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3848,6 +3848,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egressExcept: + # The cidrs will ignore SNAT action when Egress enabled + # cidrs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index c0f0f8f0fc4..24e9aea7eec 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3848,6 +3848,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egressExcept: + # The cidrs will ignore SNAT action when Egress enabled + # cidrs: [] + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index c299f230e81..75ceb77a181 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3848,6 +3848,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egressExcept: + # The cidrs will ignore SNAT action when Egress enabled + # cidrs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index d95ecbbc555..907172cfd36 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3848,6 +3848,10 @@ data: # The port for WireGuard to receive traffic. # port: 51820 + egressExcept: + # The cidrs will ignore SNAT action when Egress enabled + # cidrs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 43488921cd2..36b2e93f71e 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -93,6 +93,10 @@ wireGuard: # The port for WireGuard to receive traffic. # port: 51820 +egressExcept: +# The cidrs will ignore SNAT action when Egress enabled +# cidrs: [] + # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index f63c1554219..61b630bac0c 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -129,6 +129,14 @@ func run(o *Options) error { wireguardConfig := &config.WireGuardConfig{ Port: o.config.WireGuard.Port, } + excidrs := []net.IPNet{} + for _, cidr := range o.config.EgressExcept.CIDRs { + _, excidr, _ := net.ParseCIDR(cidr) + excidrs = append(excidrs, *excidr) + } + egressExceptConfig := &config.EgressExceptConfig{ + CIDRs: excidrs, + } routeClient, err := route.NewClient(serviceCIDRNet, networkConfig, o.config.NoSNAT) if err != nil { return fmt.Errorf("error creating route client: %v", err) @@ -158,6 +166,7 @@ func run(o *Options) error { serviceCIDRNetv6, networkConfig, wireguardConfig, + egressExceptConfig, networkReadyCh, stopCh, features.DefaultFeatureGate.Enabled(features.AntreaProxy)) diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 4ae77758b92..98ebbcdcac7 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -162,9 +162,15 @@ type AgentConfig struct { // If there are multiple IP addresses configured on the interface, the first one is used. // The interface configured with Node IP is used if this parameter is not set. TransportInterface string `yaml:"transportInterface,omitempty"` + // Egress related configurations. + EgressExcept EgressExceptConfig `yaml:"egressExcept"` } type WireGuardConfig struct { // The port for the WireGuard to receive traffic. Defaults to 51820. Port int `yaml:"port,omitempty"` } + +type EgressExceptConfig struct { + CIDRs []string `yaml:"cidrs,omitempty"` +} diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 7c4f0973b2b..7423863d289 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -153,6 +153,14 @@ func (o *Options) validate(args []string) error { if err := o.validateFlowExporterConfig(); err != nil { return fmt.Errorf("failed to validate flow exporter config: %v", err) } + if features.DefaultFeatureGate.Enabled(features.Egress) { + for _, cidr := range o.config.EgressExcept.CIDRs { + _, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("Egress Except CIDR %s is invalid", cidr) + } + } + } return nil } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index e237a56910b..c1526ef4c77 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -69,21 +69,22 @@ var getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName // Initializer knows how to setup host networking, OpenVSwitch, and Openflow. type Initializer struct { - client clientset.Interface - ovsBridgeClient ovsconfig.OVSBridgeClient - ofClient openflow.Client - routeClient route.Interface - wireGuardClient wireguard.Interface - ifaceStore interfacestore.InterfaceStore - ovsBridge string - hostGateway string // name of gateway port on the OVS bridge - mtu int - serviceCIDR *net.IPNet // K8s Service ClusterIP CIDR - serviceCIDRv6 *net.IPNet // K8s Service ClusterIP CIDR in IPv6 - networkConfig *config.NetworkConfig - nodeConfig *config.NodeConfig - wireGuardConfig *config.WireGuardConfig - enableProxy bool + client clientset.Interface + ovsBridgeClient ovsconfig.OVSBridgeClient + ofClient openflow.Client + routeClient route.Interface + wireGuardClient wireguard.Interface + ifaceStore interfacestore.InterfaceStore + ovsBridge string + hostGateway string // name of gateway port on the OVS bridge + mtu int + serviceCIDR *net.IPNet // K8s Service ClusterIP CIDR + serviceCIDRv6 *net.IPNet // K8s Service ClusterIP CIDR in IPv6 + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + wireGuardConfig *config.WireGuardConfig + egressExceptConfig *config.EgressExceptConfig + enableProxy bool // networkReadyCh should be closed once the Node's network is ready. // The CNI server will wait for it before handling any CNI Add requests. networkReadyCh chan<- struct{} @@ -103,25 +104,27 @@ func NewInitializer( serviceCIDRv6 *net.IPNet, networkConfig *config.NetworkConfig, wireGuardConfig *config.WireGuardConfig, + egressExceptConfig *config.EgressExceptConfig, networkReadyCh chan<- struct{}, stopCh <-chan struct{}, enableProxy bool) *Initializer { return &Initializer{ - ovsBridgeClient: ovsBridgeClient, - client: k8sClient, - ifaceStore: ifaceStore, - ofClient: ofClient, - routeClient: routeClient, - ovsBridge: ovsBridge, - hostGateway: hostGateway, - mtu: mtu, - serviceCIDR: serviceCIDR, - serviceCIDRv6: serviceCIDRv6, - networkConfig: networkConfig, - wireGuardConfig: wireGuardConfig, - networkReadyCh: networkReadyCh, - stopCh: stopCh, - enableProxy: enableProxy, + ovsBridgeClient: ovsBridgeClient, + client: k8sClient, + ifaceStore: ifaceStore, + ofClient: ofClient, + routeClient: routeClient, + ovsBridge: ovsBridge, + hostGateway: hostGateway, + mtu: mtu, + serviceCIDR: serviceCIDR, + serviceCIDRv6: serviceCIDRv6, + networkConfig: networkConfig, + wireGuardConfig: wireGuardConfig, + egressExceptConfig: egressExceptConfig, + networkReadyCh: networkReadyCh, + stopCh: stopCh, + enableProxy: enableProxy, } } @@ -355,7 +358,7 @@ func (i *Initializer) initOpenFlowPipeline() error { roundInfo := getRoundInfo(i.ovsBridgeClient) // Set up all basic flows. - ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig) + ofConnCh, err := i.ofClient.Initialize(roundInfo, i.nodeConfig, i.networkConfig, i.egressExceptConfig) if err != nil { klog.Errorf("Failed to initialize openflow client: %v", err) return err diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index d5f8e0499a8..c301c4f0861 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -80,6 +80,10 @@ type WireGuardConfig struct { MTU int } +type EgressExceptConfig struct { + CIDRs []net.IPNet +} + // Local Node configurations retrieved from K8s API or host networking state. type NodeConfig struct { // The Node's name used in Kubernetes. @@ -114,7 +118,8 @@ type NodeConfig struct { // The config of the OVS bridge uplink interface. Only for Windows Node. UplinkNetConfig *AdapterNetConfig // The config of the WireGuard interface. - WireGuardConfig *WireGuardConfig + WireGuardConfig *WireGuardConfig + EgressExceptConfig *EgressExceptConfig } func (n *NodeConfig) String() string { diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 5803ba56813..cd3f3ae4fde 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -40,7 +40,7 @@ type Client interface { // be called to ensure that the set of OVS flows is correct. All flows programmed in the // switch which match the current round number will be deleted before any new flow is // installed. - Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig) (<-chan struct{}, error) + Initialize(roundInfo types.RoundInfo, config *config.NodeConfig, networkconfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error) // InstallGatewayFlows sets up flows related to an OVS gateway port, the gateway must exist. InstallGatewayFlows() error @@ -729,9 +729,10 @@ func (c *client) initialize() error { return nil } -func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig) (<-chan struct{}, error) { +func (c *client) Initialize(roundInfo types.RoundInfo, nodeConfig *config.NodeConfig, networkConfig *config.NetworkConfig, egressExceptConfig *config.EgressExceptConfig) (<-chan struct{}, error) { c.nodeConfig = nodeConfig c.networkConfig = networkConfig + c.egressExceptConfig = egressExceptConfig if config.IsIPv4Enabled(nodeConfig, c.networkConfig.TrafficEncapMode) { c.ipProtocols = append(c.ipProtocols, binding.ProtocolIP) @@ -778,11 +779,20 @@ func (c *client) InstallExternalFlows() error { localGatewayMAC := c.nodeConfig.GatewayConfig.MAC var flows []binding.Flow + var ipv4CIDRs []net.IPNet + var ipv6CIDRs []net.IPNet + for _, cidr := range c.egressExceptConfig.CIDRs { + if cidr.IP.To4() == nil { + ipv6CIDRs = append(ipv6CIDRs, cidr) + } else { + ipv4CIDRs = append(ipv4CIDRs, cidr) + } + } if c.nodeConfig.NodeIPv4Addr != nil && c.nodeConfig.PodIPv4CIDR != nil { - flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC) + flows = c.externalFlows(c.nodeConfig.NodeIPv4Addr.IP, *c.nodeConfig.PodIPv4CIDR, localGatewayMAC, ipv4CIDRs) } if c.nodeConfig.NodeIPv6Addr != nil && c.nodeConfig.PodIPv6CIDR != nil { - flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC)...) + flows = append(flows, c.externalFlows(c.nodeConfig.NodeIPv6Addr.IP, *c.nodeConfig.PodIPv6CIDR, localGatewayMAC, ipv6CIDRs)...) } if err := c.ofEntryOperations.AddAll(flows); err != nil { return fmt.Errorf("failed to install flows for external communication: %v", err) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 7adf2cc0a77..f82fdc9a1fa 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -319,10 +319,11 @@ type client struct { // conjMatchFlowContext. globalConjMatchFlowCache map[string]*conjMatchFlowContext // replayMutex provides exclusive access to the OFSwitch to the ReplayFlows method. - replayMutex sync.RWMutex - nodeConfig *config.NodeConfig - networkConfig *config.NetworkConfig - gatewayOFPort uint32 + replayMutex sync.RWMutex + nodeConfig *config.NodeConfig + networkConfig *config.NetworkConfig + egressExceptConfig *config.EgressExceptConfig + gatewayOFPort uint32 // ovsDatapathType is the type of the datapath used by the bridge. ovsDatapathType ovsconfig.OVSDatapathType // ovsMetersAreSupported indicates whether the OVS datapath supports OpenFlow meters. @@ -1857,7 +1858,7 @@ func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Catego // snatCommonFlows installs the default flows for performing SNAT for traffic to // the external network. The flows identify the packets to external, and send // them to snatTable, where SNAT IPs are looked up for the packets. -func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, category cookie.Category) []binding.Flow { +func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, exceptCIDRs []net.IPNet, category cookie.Category) []binding.Flow { l3FwdTable := c.pipeline[l3ForwardingTable] nextTable := l3FwdTable.GetNext() ipProto := getIPProtocol(localSubnet.IP) @@ -1910,6 +1911,15 @@ func (c *client) snatCommonFlows(nodeIP net.IP, localSubnet net.IPNet, localGate Cookie(c.cookieAllocator.Request(category).Raw()). Done(), } + for _, cidr := range exceptCIDRs { + flows = append(flows, l3FwdTable.BuildFlow(priorityNormal). + MatchProtocol(ipProto). + MatchRegMark(FromLocalRegMark). + MatchDstIPNet(cidr). + Action().GotoTable(nextTable). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } return flows } @@ -2174,11 +2184,11 @@ func (c *client) decTTLFlows(category cookie.Category) []binding.Flow { } // externalFlows returns the flows needed to enable SNAT for external traffic. -func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr) []binding.Flow { +func (c *client) externalFlows(nodeIP net.IP, localSubnet net.IPNet, localGatewayMAC net.HardwareAddr, cidrs []net.IPNet) []binding.Flow { if !c.enableEgress { return nil } - return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cookie.SNAT) + return c.snatCommonFlows(nodeIP, localSubnet, localGatewayMAC, cidrs, cookie.SNAT) } // policyConjKeyFuncKeyFunc knows how to get key of a *policyRuleConjunction. diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index abf0bfe9f2e..51df4cd0c51 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -251,18 +251,18 @@ func (mr *MockClientMockRecorder) InitialTLVMap() *gomock.Call { } // Initialize mocks base method -func (m *MockClient) Initialize(arg0 types.RoundInfo, arg1 *config.NodeConfig, arg2 *config.NetworkConfig) (<-chan struct{}, error) { +func (m *MockClient) Initialize(arg0 types.RoundInfo, arg1 *config.NodeConfig, arg2 *config.NetworkConfig, arg3 *config.EgressExceptConfig) (<-chan struct{}, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Initialize", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "Initialize", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(<-chan struct{}) ret1, _ := ret[1].(error) return ret0, ret1 } // Initialize indicates an expected call of Initialize -func (mr *MockClientMockRecorder) Initialize(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) Initialize(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Initialize", reflect.TypeOf((*MockClient)(nil).Initialize), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Initialize", reflect.TypeOf((*MockClient)(nil).Initialize), arg0, arg1, arg2, arg3) } // InstallBridgeUplinkFlows mocks base method