Support EgressIP assigning and failover in antrea-agent

1.Add memberlist cluster in antrea-agent A cluster will be created in the background when the egress feature is turned on. And the local Node will join all the other K8s Nodes in a memberlist cluster. Each Node in the cluster holds the same consistent hash ring for each ExternalIPPool, in order to distribute egress IPs equally among the selected Nodes (which are part of the memberlist cluster). When a Node leaves the cluster, its IPs are redistributed. When a Node joins the cluster, it's added to the hash ring and a small fraction of IPs are re-assigned to that Node. 2.Add selecting egress's owner node and assign egress-ip to node Assign a owner node for egress which with a valid externalIPPool. Add egress status of api, when egress has assigned a owner node and egressIP has assigned, the egress status will updated, egress status is the owner node name. Signed-off-by: wenqiq <wenqiq@vmware.com>
antrea-io · Jul 7, 2021 · adfca93 · adfca93
1 parent d84da6c
commit adfca93
Show file tree

Hide file tree

Showing 31 changed files with 1,956 additions and 37 deletions.
diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml
@@ -1345,6 +1345,10 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
     name: v1alpha2
     schema:
       openAPIV3Schema:
@@ -1415,11 +1419,18 @@ spec:
             required:
             - appliedTo
             type: object
+          status:
+            properties:
+              egressNode:
+                type: string
+            type: object
         required:
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -3282,6 +3293,20 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - egresses/status
+  verbs:
+  - update
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - externalippools
+  verbs:
+  - get
+  - watch
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -3497,6 +3522,7 @@ rules:
   - watch
   - list
   - update
+  - patch
 - apiGroups:
   - crd.antrea.io
   resources:

diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -1345,6 +1345,10 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
     name: v1alpha2
     schema:
       openAPIV3Schema:
@@ -1415,11 +1419,18 @@ spec:
             required:
             - appliedTo
             type: object
+          status:
+            properties:
+              egressNode:
+                type: string
+            type: object
         required:
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -3282,6 +3293,20 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - egresses/status
+  verbs:
+  - update
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - externalippools
+  verbs:
+  - get
+  - watch
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -3497,6 +3522,7 @@ rules:
   - watch
   - list
   - update
+  - patch
 - apiGroups:
   - crd.antrea.io
   resources:

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -1345,6 +1345,10 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
     name: v1alpha2
     schema:
       openAPIV3Schema:
@@ -1415,11 +1419,18 @@ spec:
             required:
             - appliedTo
             type: object
+          status:
+            properties:
+              egressNode:
+                type: string
+            type: object
         required:
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -3282,6 +3293,20 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - egresses/status
+  verbs:
+  - update
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - externalippools
+  verbs:
+  - get
+  - watch
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -3497,6 +3522,7 @@ rules:
   - watch
   - list
   - update
+  - patch
 - apiGroups:
   - crd.antrea.io
   resources:

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -1345,6 +1345,10 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
     name: v1alpha2
     schema:
       openAPIV3Schema:
@@ -1415,11 +1419,18 @@ spec:
             required:
             - appliedTo
             type: object
+          status:
+            properties:
+              egressNode:
+                type: string
+            type: object
         required:
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -3282,6 +3293,20 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - egresses/status
+  verbs:
+  - update
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - externalippools
+  verbs:
+  - get
+  - watch
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -3497,6 +3522,7 @@ rules:
   - watch
   - list
   - update
+  - patch
 - apiGroups:
   - crd.antrea.io
   resources:

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -1345,6 +1345,10 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
     name: v1alpha2
     schema:
       openAPIV3Schema:
@@ -1415,11 +1419,18 @@ spec:
             required:
             - appliedTo
             type: object
+          status:
+            properties:
+              egressNode:
+                type: string
+            type: object
         required:
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -3282,6 +3293,20 @@ rules:
   - get
   - watch
   - list
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - egresses/status
+  verbs:
+  - update
+- apiGroups:
+  - crd.antrea.io
+  resources:
+  - externalippools
+  verbs:
+  - get
+  - watch
+  - list
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -3497,6 +3522,7 @@ rules:
   - watch
   - list
   - update
+  - patch
 - apiGroups:
   - crd.antrea.io
   resources:

diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml
@@ -149,6 +149,20 @@ rules:
       - get
       - watch
       - list
+  - apiGroups:
+      - crd.antrea.io
+    resources:
+      - egresses/status
+    verbs:
+      - update
+  - apiGroups:
+      - crd.antrea.io
+    resources:
+      - externalippools
+    verbs:
+      - get
+      - watch
+      - list
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

diff --git a/build/yamls/base/controller-rbac.yml b/build/yamls/base/controller-rbac.yml
@@ -207,6 +207,7 @@ rules:
       - watch
       - list
       - update
+      - patch
   - apiGroups:
       - crd.antrea.io
     resources:

diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml
@@ -81,6 +81,11 @@ spec:
                 - format: ipv6
               externalIPPool:
                 type: string
+          status:
+            type: object
+            properties:
+              egressNode:
+                type: string
     additionalPrinterColumns:
     - description: Specifies the SNAT IP address for the selected workloads.
       jsonPath: .spec.egressIP
@@ -89,6 +94,12 @@ spec:
     - jsonPath: .metadata.creationTimestamp
       name: Age
       type: date
+    - description: The Owner Node of egress IP
+      jsonPath: .status.egressNode
+      name: Node
+      type: string
+    subresources:
+      status: {}
   scope: Cluster
   names:
     plural: egresses

diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -77,6 +77,8 @@ func run(o *Options) error {
 	crdInformerFactory := crdinformers.NewSharedInformerFactory(crdClient, informerDefaultResync)
 	traceflowInformer := crdInformerFactory.Crd().V1alpha1().Traceflows()
 	egressInformer := crdInformerFactory.Crd().V1alpha2().Egresses()
+	nodeInformer := informerFactory.Core().V1().Nodes()
+	externalIPPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools()
 
 	// Create Antrea Clientset for the given config.
 	antreaClientProvider := agent.NewAntreaClientProvider(o.config.AntreaClientConnection, k8sClient)
@@ -223,7 +225,13 @@ func run(o *Options) error {
 
 	var egressController *egress.EgressController
 	if features.DefaultFeatureGate.Enabled(features.Egress) {
-		egressController = egress.NewEgressController(ofClient, egressInformer, antreaClientProvider, ifaceStore, routeClient, nodeConfig.Name)
+		egressController, err = egress.NewEgressController(
+			ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP,
+			o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer,
+		)
+		if err != nil {
+			return fmt.Errorf("error creating new Egress controller: %v", err)
+		}
 	}
 
 	isChaining := false

diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go
@@ -98,6 +98,11 @@ type AgentConfig struct {
 	// APIPort is the port for the antrea-agent APIServer to serve on.
 	// Defaults to 10350.
 	APIPort int `yaml:"apiPort,omitempty"`
+
+	// ClusterMembershipPort is the server port used by the antrea-agent to run a gossip-based cluster membership protocol. Currently it's used only when the Egress feature is enabled.
+	// Defaults to 10351.
+	ClusterMembershipPort int `yaml:"clusterPort,omitempty"`
+
 	// Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener
 	// Defaults to true.
 	EnablePrometheusMetrics bool `yaml:"enablePrometheusMetrics,omitempty"`

diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go
@@ -193,6 +193,10 @@ func (o *Options) setDefaults() {
 		o.config.APIPort = apis.AntreaAgentAPIPort
 	}
 
+	if o.config.ClusterMembershipPort == 0 {
+		o.config.ClusterMembershipPort = apis.AntreaAgentClusterMembershipPort
+	}
+
 	if features.DefaultFeatureGate.Enabled(features.FlowExporter) {
 		if o.config.FlowCollectorAddr == "" {
 			o.config.FlowCollectorAddr = defaultFlowCollectorAddress
-Original file line number
+Diff line change
@@ Expand Up / @@ -207,6 +207,7 @@ rules: @@
           - watch
           - list
           - update
+          - patch
       - apiGroups:
           - crd.antrea.io
         resources:
@@ Expand Down @@