From a3de615d98c7d034cbc9f309153bf736cd21b120 Mon Sep 17 00:00:00 2001 From: wenqiq Date: Wed, 26 May 2021 21:19:06 +0800 Subject: [PATCH] Add select egress's owner node and assign egress-ip to node for egress failover feature Add egress status of api and Update codegen of egress status Signed-off-by: wenqiq --- build/yamls/antrea-aks.yml | 19 +++ build/yamls/antrea-eks.yml | 19 +++ build/yamls/antrea-gke.yml | 19 +++ build/yamls/antrea-ipsec.yml | 19 +++ build/yamls/antrea.yml | 19 +++ build/yamls/base/agent-rbac.yml | 3 + build/yamls/base/crds.yml | 16 +++ go.mod | 1 + go.sum | 2 + .../controller/egress/egress_controller.go | 89 ++++++++++++++ .../controller/egress/ip_assign_advertise.go | 20 +++ .../egress/ip_assign_advertise_linux.go | 115 ++++++++++++++++++ .../egress/ip_assign_advertise_windows.go | 45 +++++++ pkg/agent/memberlist/cluster.go | 6 +- pkg/agent/memberlist/cluster_test.go | 2 +- pkg/agent/util/ndp/ndp_test.go | 12 +- pkg/apis/crd/v1alpha2/types.go | 26 +++- .../crd/v1alpha2/zz_generated.deepcopy.go | 17 +++ .../versioned/typed/crd/v1alpha2/egress.go | 16 +++ .../typed/crd/v1alpha2/fake/fake_egress.go | 11 ++ pkg/controller/metrics/prometheus.go | 7 ++ plugins/octant/go.sum | 1 + 22 files changed, 473 insertions(+), 11 deletions(-) create mode 100644 pkg/agent/controller/egress/ip_assign_advertise.go create mode 100644 pkg/agent/controller/egress/ip_assign_advertise_linux.go create mode 100644 pkg/agent/controller/egress/ip_assign_advertise_windows.go diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 183d0617362..61ab2cddc82 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3280,10 +3296,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index eb1b98e72e8..6fce8f2dc43 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3280,10 +3296,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 831b516c95e..a632df4d9f3 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3280,10 +3296,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 8fff8dfc0bc..3cccc2a28eb 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3280,10 +3296,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index f1a724df9bc..70ce1f96d5a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3280,10 +3296,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml index 13c94107750..73b3531a459 100644 --- a/build/yamls/base/agent-rbac.yml +++ b/build/yamls/base/agent-rbac.yml @@ -146,10 +146,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index 38a1867b21c..382ab41ad0c 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -81,6 +81,16 @@ spec: - format: ipv6 externalIPPool: type: string + failoverPolicy: + type: string + enum: + - Auto + - None + status: + type: object + properties: + nodeName: + type: string additionalPrinterColumns: - description: Specifies the SNAT IP address for the selected workloads. jsonPath: .spec.egressIP @@ -89,6 +99,12 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string + subresources: + status: {} scope: Cluster names: plural: egresses diff --git a/go.mod b/go.mod index 44131667a54..23705d99eb3 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module antrea.io/antrea go 1.15 require ( + bou.ke/monkey v1.0.2 github.com/Mellanox/sriovnet v1.0.2 github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331 github.com/Microsoft/hcsshim v0.8.9 diff --git a/go.sum b/go.sum index 9d74c5f3f40..a3d79d64282 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= diff --git a/pkg/agent/controller/egress/egress_controller.go b/pkg/agent/controller/egress/egress_controller.go index 0f8d16cf44c..9175cee90d2 100644 --- a/pkg/agent/controller/egress/egress_controller.go +++ b/pkg/agent/controller/egress/egress_controller.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" @@ -40,8 +41,10 @@ import ( "antrea.io/antrea/pkg/agent/route" cpv1b2 "antrea.io/antrea/pkg/apis/controlplane/v1beta2" crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" + clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/controller/metrics" "antrea.io/antrea/pkg/util/k8s" ) @@ -102,6 +105,7 @@ type egressBinding struct { type EgressController struct { ofClient openflow.Client routeClient route.Interface + crdClient clientsetversioned.Interface antreaClientProvider agent.AntreaClientProvider egressInformer cache.SharedIndexInformer @@ -179,6 +183,7 @@ func NewEgressController( resyncPeriod, ) localIPDetector.AddEventHandler(c.onLocalIPUpdate) + c.cluster.AddEventHandler(c.onClusterNodeUpdate) return c } @@ -214,6 +219,39 @@ func (c *EgressController) onLocalIPUpdate(ip string, added bool) { } } +func (c *EgressController) onClusterNodeUpdate(nodeName string, join bool) { + var egresses []*crdv1a2.Egress + if join { + // list all egress of local node, and move not hit egress + c.egressStatesMutex.RLock() + defer c.egressStatesMutex.RUnlock() + for egressName := range c.egressStates { + if c.cluster.ShouldSelect(egressName) { + continue + } + egress, _ := c.egressLister.Get(egressName) + egresses = append(egresses, egress) + } + klog.V(0).Infof("Detected cluster node (%s) joined, and egress should unbind", nodeName) + } else { + // list egress owned by left node and handler the egress if hit by local node + egressesOfLeftNode, _ := c.egressLister.List(labels.Everything()) + for _, egress := range egressesOfLeftNode { + if egress.Status.NodeName != nodeName { + continue + } + if c.cluster.ShouldSelect(egress.Name) { + egresses = append(egresses, egress) + } + } + klog.V(0).Infof("Detected cluster node (%s) left, and egress should bind", nodeName) + } + for _, egress := range egresses { + c.enqueueEgress(egress) + } + klog.V(0).Infof("%d egress should move and enqueue egress worker", len(egresses)) +} + // Run will create defaultWorkers workers (go routines) which will process the Egress events from the // workqueue. func (c *EgressController) Run(stopCh <-chan struct{}) { @@ -232,6 +270,18 @@ func (c *EgressController) Run(stopCh <-chan struct{}) { go wait.NonSlidingUntil(c.watchEgressGroup, 5*time.Second, stopCh) + // if node support egress failover + if c.cluster.LocalNodeJoined() { + // when a new node join gossip cluster, it should handle the egress that hit itself(moved from other nodes) + egresses, _ := c.egressLister.List(labels.Everything()) + + for _, e := range egresses { + if e.Spec.EgressFailoverPolicyType == crdv1a2.EgressFailoverPolicyTypeAuto && c.cluster.ShouldSelect(e.Name) { + c.enqueueEgress(e) + } + } + } + for i := 0; i < defaultWorkers; i++ { go wait.Until(c.worker, time.Second, stopCh) } @@ -450,6 +500,22 @@ func (c *EgressController) unbindPodEgress(pod, egress string) (string, bool) { return "", false } +func (c *EgressController) updateEgressStatus(egress *crdv1a2.Egress, nodeName string) error { + klog.V(0).Infof("Egress status : %#v, update to : %s", egress.Status, nodeName) + if egress.Status.NodeName == nodeName { + return nil + } + toUpdate := egress.DeepCopy() + toUpdate.Status.NodeName = nodeName + if _, err := c.crdClient.CrdV1alpha2().Egresses().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}); err != nil { + klog.Warningf("egress update error: %s", err) + return err + } + klog.V(0).Infof("update egress %s status success", egress.Name) + metrics.AntreaEgressStatusUpdates.Inc() + return nil +} + func (c *EgressController) syncEgress(egressName string) error { startTime := time.Now() defer func() { @@ -473,6 +539,29 @@ func (c *EgressController) syncEgress(egressName string) error { return err } + if egress.Spec.EgressFailoverPolicyType == crdv1a2.EgressFailoverPolicyTypeAuto && c.cluster.LocalNodeJoined() { + if c.cluster.ShouldSelect(egressName) { + if !c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.cluster.NodeConfig) + if err := a.AssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + if err := c.updateEgressStatus(egress, c.nodeName); err != nil { + return fmt.Errorf("update egress status error:%s", err.Error()) + } + klog.V(0).Infof("Assigned owner node %s for egress %s and update status", + c.nodeName, egressName) + } else { + if c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.cluster.NodeConfig) + if err := a.UnAssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + } + } + eState, exist := c.getEgressState(egressName) // If the EgressIP changes, uninstalls this Egress first. if exist && eState.egressIP != egress.Spec.EgressIP { diff --git a/pkg/agent/controller/egress/ip_assign_advertise.go b/pkg/agent/controller/egress/ip_assign_advertise.go new file mode 100644 index 00000000000..73396eef7d2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise.go @@ -0,0 +1,20 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +type INodeEgressAssigner interface { + AssignOwnerNodeEgressIP(egressIp string) error + UnAssignOwnerNodeEgressIP(egressIp string) error +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_linux.go b/pkg/agent/controller/egress/ip_assign_advertise_linux.go new file mode 100644 index 00000000000..275096d0eb7 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_linux.go @@ -0,0 +1,115 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "fmt" + "net" + + "github.com/vishvananda/netlink" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/util/arping" + "antrea.io/antrea/pkg/agent/util/ndp" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig) + if err != nil { + return err + } + if err := assignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig) + if err != nil { + return err + } + if err := unassignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func getNodeLinkAddr(egressIP string, nodeConfig *config.NodeConfig) (*netlink.Addr, netlink.Link, error) { + egressSpecIP := net.ParseIP(egressIP) + egressIPMask := nodeConfig.NodeIPAddr.Mask + addr := netlink.Addr{IPNet: &net.IPNet{IP: egressSpecIP, Mask: egressIPMask}} + gwLinkIndex := nodeConfig.GatewayConfig.LinkIndex + link, err := netlink.LinkByIndex(gwLinkIndex) + if err != nil { + return nil, nil, fmt.Errorf("get netlink by gw index(%d) error: %+v", gwLinkIndex, err) + } + klog.V(2).Infof("get node iface netlink and address: %+v, %+v", link, addr) + return &addr, link, nil +} + +func assignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // assign IP + ifaceName := link.Attrs().Name + ifaceH, _ := net.InterfaceByName(ifaceName) + klog.V(2).Infof("Adding address %+v to interface %s", addr, ifaceName) + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("failed to add address %v to interface %s: %v", addr, ifaceName, err) + } + egressSpecIP := addr.IP + isIPv4 := egressSpecIP.To4() + if isIPv4 != nil { + if err := arping.GratuitousARPOverIface(isIPv4, ifaceH); err != nil { + klog.Warningf("Failed to send gratuitous ARP: %v", err) + return err + } + klog.V(0).Infof("Send gratuitous ARP: %+v", isIPv4) + } else if isIPv6 := egressSpecIP.To16(); isIPv6 != nil { + if err := ndp.NDPNeighborAdvertiseOverIface(isIPv6, ifaceH); err != nil { + klog.Warningf("Failed to send Advertisement: %v", err) + return err + } + } + return nil +} + +func unassignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // check ip, if existed, uninstall ip + ifaceName := link.Attrs().Name + klog.V(2).Infof("Deleting address %v to interface %s", addr, ifaceName) + if err := netlink.AddrDel(link, addr); err != nil { + return fmt.Errorf("failed to delete address %v to interface %s: %v", addr, ifaceName, err) + } + return nil +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_windows.go b/pkg/agent/controller/egress/ip_assign_advertise_windows.go new file mode 100644 index 00000000000..88f657ddde2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_windows.go @@ -0,0 +1,45 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "antrea.io/antrea/pkg/agent/config" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index 40a4a79245a..bb8ff289750 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -465,7 +465,7 @@ func (c *Cluster) updateNodeConsistentHash(nodeEvent *memberlist.NodeEvent) { func (c *Cluster) updateLocalNodeStatus() { shouldJoin := c.shouldJoinCluster(c.nodeName) - joined := c.localNodeJoined() + joined := c.LocalNodeJoined() if joined && !shouldJoin { c.localNodeEventCh <- true } else if !joined && shouldJoin { @@ -483,8 +483,8 @@ func (c *Cluster) nodeList() []string { return nodes } -// localNodeJoined if merbers num in cluster is 1 means local node not joined in other clusters -func (c *Cluster) localNodeJoined() bool { +// LocalNodeJoined if merbers num in cluster is 1 means local node not joined in other clusters +func (c *Cluster) LocalNodeJoined() bool { // while node joined cluster, num of member will >= 1; if leave cluster, num of member will be 0 return c.mList.NumMembers() >= 1 } diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index acbbd83fe0f..5b17cce5202 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -285,7 +285,7 @@ func TestCluster_Run(t *testing.T) { cache.WaitForCacheSync(stopCh, ipPoolInformer.Informer().HasSynced) s.AddEventHandler(func(nodeName string, added bool) { - t.Logf("notified node %s added (%t) node event handler", nodeName, added) + t.Logf("Notified node %s added (%t) node event handler", nodeName, added) }) go s.Run(stopCh) diff --git a/pkg/agent/util/ndp/ndp_test.go b/pkg/agent/util/ndp/ndp_test.go index f8d07b09a4c..1369799d62d 100644 --- a/pkg/agent/util/ndp/ndp_test.go +++ b/pkg/agent/util/ndp/ndp_test.go @@ -66,7 +66,7 @@ func TestAdvertiserMarshalMessage(t *testing.T) { b, err = marshalMessage(msg) return }(tt.ipv6Addr, tt.hardwareAddr); err != nil || !reflect.DeepEqual(got, tt.want) { - t.Errorf("NDPmarshalMessage() = %v, want %v", got, tt.want) + t.Errorf("NDPmarshalMessage() = %v, want %v, error:%v", got, tt.want, err) } }) } @@ -84,7 +84,7 @@ func MustIPv6(s string) (net.IP, error) { func TestNDPNeighborAdvertiseOverIface(t *testing.T) { ip, err := MustIPv6("fe80::250:56ff:fea7:e29d") if err != nil { - fmt.Println("invalid ipv6: ", err) + t.Logf("Invalid ipv6: %v", err) return } @@ -102,7 +102,7 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { conICMP, err := icmp.ListenPacket("ip6:ipv6-icmp", "fe80::1%lo0") if err != nil { - t.Logf("listen icmp ipv6 local error: %q", err.Error()) + t.Logf("Listen icmp ipv6 local error: %q", err.Error()) monkey.Patch(NewNDPAdvertiser, func(iface *net.Interface) (INDPAdvertiser, error) { s := &Advertiser{ fakeIntFace.HardwareAddr, @@ -115,7 +115,7 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { } return s, nil }) - t.Skipf("cannot test ICMPv6 NDP: %q", err.Error()) + t.Skipf("Cannot test ICMPv6 NDP: %q", err.Error()) } monkey.Patch(icmp.ListenPacket, func(network, address string) (*icmp.PacketConn, error) { return conICMP, nil @@ -125,11 +125,11 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { ifaceName := "ens192" iface, err := net.InterfaceByName(ifaceName) if err != nil { - fmt.Println("interface not found", ifaceName) + t.Errorf("Interface %s not found", ifaceName) return } if err := NDPNeighborAdvertiseOverIface(ip, iface); err != nil { - t.Errorf("new NDP gratuitous advertisement error:%q", err.Error()) + t.Errorf("New NDP gratuitous advertisement error:%q", err.Error()) } } diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 6393886a43b..4963c113739 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -190,7 +190,6 @@ type AppliedTo struct { // +genclient // +genclient:nonNamespaced -// +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // Egress defines which egress (SNAT) IP the traffic from the selected Pods to @@ -202,6 +201,15 @@ type Egress struct { // Specification of the desired behavior of Egress. Spec EgressSpec `json:"spec"` + + // Egress status show the owner node name of egress, if failoverPolicy set auto. + Status EgressStatus `json:"status"` +} + +// EgressStatus owner node name of Egress +type EgressStatus struct { + // The name of the Node that holds the Egress IP. + NodeName string `json:"nodeName"` } // EgressSpec defines the desired state for Egress. @@ -218,8 +226,24 @@ type EgressSpec struct { // If it is non-empty, the EgressIP will be assigned to a Node specified by the pool automatically and will failover // to a different Node when the Node becomes unreachable. ExternalIPPool string `json:"externalIPPool"` + // FailoverPolicy specifies if the Egress IP should failover to another Node + // when the Node that holds it becomes unavailable. + // The default value is "None". + EgressFailoverPolicyType `json:"failoverPolicy"` } +type EgressFailoverPolicyType string + +const ( + // EgressFailoverPolicyTypeNone disables failover for this Egress. The + // Egress IP must be assigned to an arbitrary interface of one Node, and be + // reassigned to another Node manually in case of Node failure. + EgressFailoverPolicyTypeNone EgressFailoverPolicyType = "None" + // EgressFailoverPolicyTypeAuto enables auto failover for this Egress. The + // Egress IP must not be assigned to any interface of any Node in advance. + EgressFailoverPolicyTypeAuto EgressFailoverPolicyType = "Auto" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type EgressList struct { diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index cdf7e23580f..bb2587fb0c7 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -122,6 +122,7 @@ func (in *Egress) DeepCopyInto(out *Egress) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status return } @@ -193,6 +194,22 @@ func (in *EgressSpec) DeepCopy() *EgressSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressStatus) DeepCopyInto(out *EgressStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressStatus. +func (in *EgressStatus) DeepCopy() *EgressStatus { + if in == nil { + return nil + } + out := new(EgressStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go index 784d5c1aec8..52326a2f0ce 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go @@ -38,6 +38,7 @@ type EgressesGetter interface { type EgressInterface interface { Create(ctx context.Context, egress *v1alpha2.Egress, opts v1.CreateOptions) (*v1alpha2.Egress, error) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) + UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha2.Egress, error) @@ -126,6 +127,21 @@ func (c *egresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1. return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *egresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (result *v1alpha2.Egress, err error) { + result = &v1alpha2.Egress{} + err = c.client.Put(). + Resource("egresses"). + Name(egress.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(egress). + Do(ctx). + Into(result) + return +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *egresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { return c.client.Delete(). diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go index 6e142d760b1..9c463ebf406 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go @@ -94,6 +94,17 @@ func (c *FakeEgresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts return obj.(*v1alpha2.Egress), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeEgresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(egressesResource, "status", egress), &v1alpha2.Egress{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.Egress), err +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *FakeEgresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. diff --git a/pkg/controller/metrics/prometheus.go b/pkg/controller/metrics/prometheus.go index 1d8735c0c9b..d346ab67efa 100644 --- a/pkg/controller/metrics/prometheus.go +++ b/pkg/controller/metrics/prometheus.go @@ -96,6 +96,13 @@ var ( Help: "The total number of actual status updates performed for Antrea NetworkPolicy Custom Resources", StabilityLevel: metrics.ALPHA, }) + AntreaEgressStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ + Namespace: metricNamespaceAntrea, + Subsystem: metricSubsystemController, + Name: "eg_status_updates", + Help: "The total number of actual status updates performed for Antrea Egress Custom Resources", + StabilityLevel: metrics.ALPHA, + }) AntreaClusterNetworkPolicyStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ Namespace: metricNamespaceAntrea, Subsystem: metricSubsystemController, diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index dc61f2ce0a0..73c27182a7b 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -1,3 +1,4 @@ +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=