antrea-io · antoninbas · Oct 26, 2023 · Aug 19, 2023 · Oct 25, 2023
diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf
@@ -76,6 +76,9 @@ featureGates:
 # Allow users to specify the load balancer mode as DSR (Direct Server Return).
 {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "LoadBalancerModeDSR" "default" false) }}
 
+# Enable Egress traffic shaping.
+{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "EgressTrafficShaping" "default" false) }}
+
 # Name of the OpenVSwitch bridge antrea-agent will create and use.
 # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
 ovsBridge: {{ .Values.ovs.bridgeName | quote }}

diff --git a/build/charts/antrea/crds/egress.yaml b/build/charts/antrea/crds/egress.yaml
@@ -239,6 +239,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:

diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -2375,6 +2375,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:
@@ -5561,6 +5571,9 @@ data:
     # Allow users to specify the load balancer mode as DSR (Direct Server Return).
     #  LoadBalancerModeDSR: false
 
+    # Enable Egress traffic shaping.
+    #  EgressTrafficShaping: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6853,7 +6866,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 77b5f317f3faa10adebca604e145675d41d73631984cc8fa075069b70f9f0419
+        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
       labels:
         app: antrea
         component: antrea-agent
@@ -7094,7 +7107,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 77b5f317f3faa10adebca604e145675d41d73631984cc8fa075069b70f9f0419
+        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml
@@ -2366,6 +2366,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -2375,6 +2375,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:
@@ -5561,6 +5571,9 @@ data:
     # Allow users to specify the load balancer mode as DSR (Direct Server Return).
     #  LoadBalancerModeDSR: false
 
+    # Enable Egress traffic shaping.
+    #  EgressTrafficShaping: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6853,7 +6866,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 77b5f317f3faa10adebca604e145675d41d73631984cc8fa075069b70f9f0419
+        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
       labels:
         app: antrea
         component: antrea-agent
@@ -7095,7 +7108,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 77b5f317f3faa10adebca604e145675d41d73631984cc8fa075069b70f9f0419
+        checksum/config: e59e0431902646d46cba490279184fea2bdd3c8b486b5a7b1d3ece9a91614634
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -2375,6 +2375,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:
@@ -5561,6 +5571,9 @@ data:
     # Allow users to specify the load balancer mode as DSR (Direct Server Return).
     #  LoadBalancerModeDSR: false
 
+    # Enable Egress traffic shaping.
+    #  EgressTrafficShaping: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6853,7 +6866,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 0b761fc6deaf2ebde722c4d34a9898d9e9370e3c99467d40a28009909011b9e9
+        checksum/config: 3b1758664de8044af1aa7454c64bd1a4911750e562e1ae9375c9c16a335a469d
       labels:
         app: antrea
         component: antrea-agent
@@ -7092,7 +7105,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 0b761fc6deaf2ebde722c4d34a9898d9e9370e3c99467d40a28009909011b9e9
+        checksum/config: 3b1758664de8044af1aa7454c64bd1a4911750e562e1ae9375c9c16a335a469d
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -2375,6 +2375,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:
@@ -5574,6 +5584,9 @@ data:
     # Allow users to specify the load balancer mode as DSR (Direct Server Return).
     #  LoadBalancerModeDSR: false
 
+    # Enable Egress traffic shaping.
+    #  EgressTrafficShaping: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6866,7 +6879,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 55b17484eb9e47c7af06d7a9367348b851d9de4ad0cdc0e1a3f0b328b08df2d2
+        checksum/config: a34de3efa658ac40c9bde28e08832dd897259fdcf639beab9d4e47531d7da948
         checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4
       labels:
         app: antrea
@@ -7151,7 +7164,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 55b17484eb9e47c7af06d7a9367348b851d9de4ad0cdc0e1a3f0b328b08df2d2
+        checksum/config: a34de3efa658ac40c9bde28e08832dd897259fdcf639beab9d4e47531d7da948
       labels:
         app: antrea
         component: antrea-controller

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -2375,6 +2375,16 @@ spec:
                   type: array
                   items:
                     type: string
+                bandwidth:
+                  type: object
+                  required:
+                    - rate
+                    - burst
+                  properties:
+                    rate:
+                      type: string
+                    burst:
+                      type: string
             status:
               type: object
               properties:
@@ -5561,6 +5571,9 @@ data:
     # Allow users to specify the load balancer mode as DSR (Direct Server Return).
     #  LoadBalancerModeDSR: false
 
+    # Enable Egress traffic shaping.
+    #  EgressTrafficShaping: false
+
     # Name of the OpenVSwitch bridge antrea-agent will create and use.
     # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
     ovsBridge: "br-int"
@@ -6853,7 +6866,7 @@ spec:
         kubectl.kubernetes.io/default-container: antrea-agent
         # Automatically restart Pods with a RollingUpdate if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5f88b901b90e9499d36fc38364a673d34b6fd6e79344fb63770d65ae3544470a
+        checksum/config: aa947bf5c403412b9c8cfcbcc335659992f19bd428886e80f43bafa052bac1e6
       labels:
         app: antrea
         component: antrea-agent
@@ -7092,7 +7105,7 @@ spec:
       annotations:
         # Automatically restart Pod if the ConfigMap changes
         # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments
-        checksum/config: 5f88b901b90e9499d36fc38364a673d34b6fd6e79344fb63770d65ae3544470a
+        checksum/config: aa947bf5c403412b9c8cfcbcc335659992f19bd428886e80f43bafa052bac1e6
       labels:
         app: antrea
         component: antrea-controller

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -160,6 +160,7 @@ func run(o *Options) error {
 		features.DefaultFeatureGate.Enabled(features.AntreaPolicy),
 		l7NetworkPolicyEnabled,
 		o.enableEgress,
+		features.DefaultFeatureGate.Enabled(features.EgressTrafficShaping),
 		enableFlowExporter,
 		o.config.AntreaProxy.ProxyAll,
 		features.DefaultFeatureGate.Enabled(features.LoadBalancerModeDSR),
@@ -514,6 +515,7 @@ func run(o *Options) error {
 		egressController, err = egress.NewEgressController(
 			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeTransportInterfaceName,
 			memberlistCluster, egressInformer, nodeInformer, podUpdateChannel, serviceCIDRProvider, o.config.Egress.MaxEgressIPsPerNode,
+			features.DefaultFeatureGate.Enabled(features.EgressTrafficShaping),
 		)
 		if err != nil {
 			return fmt.Errorf("error creating new Egress controller: %v", err)

diff --git a/docs/egress.md b/docs/egress.md
@@ -9,6 +9,7 @@
   - [AppliedTo](#appliedto)
   - [EgressIP](#egressip)
   - [ExternalIPPool](#externalippool)
+  - [Bandwidth](#bandwidth)
 - [The ExternalIPPool resource](#the-externalippool-resource)
   - [IPRanges](#ipranges)
   - [NodeSelector](#nodeselector)
@@ -127,6 +128,46 @@ The `externalIPPool` field specifies the name of the `ExternalIPPool` that the
 be assigned to. It can be empty, which means users should assign the `egressIP`
 to one Node manually.
 
+### Bandwidth
+
+The `bandwidth` field enables traffic shaping for an Egress, by limiting the
+bandwidth for all egress traffic belonging to this Egress. `rate` specifies
+the maximum transmission rate. `burst` specifies the maximum burst size when
+traffic exceeds the rate. The user-provided values for `rate` and `burst` must
+follow the Kubernetes [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/) format,
+e.g. 300k, 100M, 2G. All backend workloads selected by a rate-limited Egress share the
+same bandwidth while sending egress traffic via this Egress. If these limits are exceeded,
+the traffic will be dropped.
+
+**Note**: Traffic shaping is currently in alpha version. To use this feature, users should
+enable the `EgressTrafficShaping` feature gate. Each Egress IP can be applied one bandwidth only.
+If multiple Egresses use the same IP but configure different bandwidths, the effective
+bandwidth will be selected randomly from the set of configured bandwidths. The effective use of the `bandwidth`
+function requires the OVS datapath to support meters.
+
+An Egress with traffic shaping example:
+
+```yaml
+apiVersion: crd.antrea.io/v1beta1
+kind: Egress
+metadata:
+  name: egress-prod-web
+spec:
+  appliedTo:
+    namespaceSelector:
+      matchLabels:
+        env: prod
+    podSelector:
+      matchLabels:
+        role: web
+  egressIP: 10.10.0.8
+  bandwidth:
+    rate: 800M
+    burst: 2G
+status:
+  egressNode: node01
+```
+
 ## The ExternalIPPool resource
 
 ExternalIPPool defines one or multiple IP ranges that can be used in the

diff --git a/docs/feature-gates.md b/docs/feature-gates.md
@@ -55,6 +55,7 @@ edit the Agent configuration in the
 | `SupportBundleCollection`     | Agent + Controller | `false` | Alpha | v1.10         | N/A          | N/A        | Yes                |                                               |
 | `L7NetworkPolicy`             | Agent + Controller | `false` | Alpha | v1.10         | N/A          | N/A        | Yes                |                                               |
 | `AdminNetworkPolicy`          | Controller         | `false` | Alpha | v1.13         | N/A          | N/A        | Yes                |                                               |
+| `EgressTrafficShaping`        | Agent              | `false` | Alpha | v1.14         | N/A          | N/A        | Yes                | OVS meters should be supported                |
 
 ## Description and Requirements of Features
 
@@ -402,3 +403,13 @@ this [document](antrea-l7-network-policy.md#prerequisites) for more information
 
 The `AdminNetworkPolicy` API (which currently includes the AdminNetworkPolicy and BaselineAdminNetworkPolicy objects)
 complements the Antrea-native policies and help cluster administrators to set security postures in a portable manner.
+
+### EgressTrafficShaping
+
+The `EgressTrafficShaping` feature gate of Antrea Agent enables traffic shaping of Egress, which could limit the
+bandwidth for all egress traffic belonging to an Egress. Refer to this [document](egress.md#trafficshaping)
+
+#### Requirements for this Feature
+
+This feature leverages OVS meters to do the actual rate-limiting, therefore this feature requires OVS meters
+to be supported in the datapath.