diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 660a6951af1..9a942a6ccdd 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1405,15 +1409,27 @@ spec: - format: ipv4 - format: ipv6 type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo - egressIP type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3195,10 +3211,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 557dc28df22..0d32cfbf21f 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1405,15 +1409,27 @@ spec: - format: ipv4 - format: ipv6 type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo - egressIP type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3195,10 +3211,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index ab4331fa55e..6614fd170f2 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1405,15 +1409,27 @@ spec: - format: ipv4 - format: ipv6 type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo - egressIP type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3195,10 +3211,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 1d8972afe81..30e5b5761af 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1405,15 +1409,27 @@ spec: - format: ipv4 - format: ipv6 type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo - egressIP type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3195,10 +3211,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 339707c7876..534b4269647 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1405,15 +1409,27 @@ spec: - format: ipv4 - format: ipv6 type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo - egressIP type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3195,10 +3211,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml index 13c94107750..73b3531a459 100644 --- a/build/yamls/base/agent-rbac.yml +++ b/build/yamls/base/agent-rbac.yml @@ -146,10 +146,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index fa9eca30fb3..9143f2431ae 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -75,6 +75,16 @@ spec: oneOf: - format: ipv4 - format: ipv6 + failoverPolicy: + type: string + enum: + - Auto + - None + status: + type: object + properties: + nodeName: + type: string additionalPrinterColumns: - description: Specifies the SNAT IP address for the selected workloads. jsonPath: .spec.egressIP @@ -83,6 +93,12 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string + subresources: + status: {} scope: Cluster names: plural: egresses diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 9041d3bd8fd..db088500210 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -224,13 +224,19 @@ func run(o *Options) error { } var egressController *egress.EgressController - var cluster *memberlist.Cluster if features.DefaultFeatureGate.Enabled(features.Egress) { - egressController = egress.NewEgressController(ofClient, egressInformer, antreaClientProvider, ifaceStore, routeClient, nodeConfig.Name) - cluster, err = memberlist.NewCluster(o.config.ClusterPort, nodeInformer, nodeConfig) - if err != nil { - return fmt.Errorf("initializing egress node memberlist cluster error: %v", err) + // turn on the egress auto failover switch, to-do + nodeAutoFailoverSupport := true + var cluster *memberlist.Cluster + if nodeAutoFailoverSupport { + cluster, err = memberlist.NewCluster(o.config.ClusterPort, nodeInformer, nodeConfig) + if err != nil { + return fmt.Errorf("initializing egress node memberlist cluster error: %v", err) + } } + egressController = egress.NewEgressController( + ofClient, egressInformer, antreaClientProvider, ifaceStore, routeClient, + nodeConfig.Name, cluster, crdClient, nodeAutoFailoverSupport) } isChaining := false if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { @@ -305,7 +311,6 @@ func run(o *Options) error { if features.DefaultFeatureGate.Enabled(features.Egress) { go egressController.Run(stopCh) - go cluster.Run(stopCh) } if features.DefaultFeatureGate.Enabled(features.NetworkPolicyStats) { diff --git a/pkg/agent/controller/egress/egress_controller.go b/pkg/agent/controller/egress/egress_controller.go index f7576151743..a5e2d4ccee0 100644 --- a/pkg/agent/controller/egress/egress_controller.go +++ b/pkg/agent/controller/egress/egress_controller.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" @@ -35,12 +36,15 @@ import ( "antrea.io/antrea/pkg/agent" "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/memberlist" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/route" cpv1b2 "antrea.io/antrea/pkg/apis/controlplane/v1beta2" crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" + clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/controller/metrics" "antrea.io/antrea/pkg/util/k8s" ) @@ -101,6 +105,7 @@ type egressBinding struct { type EgressController struct { ofClient openflow.Client routeClient route.Interface + crdClient clientsetversioned.Interface antreaClientProvider agent.AntreaClientProvider egressInformer cache.SharedIndexInformer @@ -127,6 +132,9 @@ type EgressController struct { egressIPStates map[string]*egressIPState egressIPStatesMutex sync.Mutex + + Cluster *memberlist.Cluster + autoFailoverSupport bool } func NewEgressController( @@ -136,6 +144,9 @@ func NewEgressController( ifaceStore interfacestore.InterfaceStore, routeClient route.Interface, nodeName string, + cluster *memberlist.Cluster, + crdClient clientsetversioned.Interface, + failoverSupport bool, ) *EgressController { localIPDetector := NewLocalIPDetector() c := &EgressController{ @@ -154,6 +165,9 @@ func NewEgressController( egressBindings: map[string]*egressBinding{}, localIPDetector: localIPDetector, idAllocator: newIDAllocator(minEgressMark, maxEgressMark), + Cluster: cluster, + crdClient: crdClient, + autoFailoverSupport: failoverSupport, } c.egressInformer.AddIndexers(cache.Indexers{egressIPIndex: func(obj interface{}) ([]string, error) { @@ -174,6 +188,9 @@ func NewEgressController( resyncPeriod, ) localIPDetector.AddEventHandler(c.onLocalIPUpdate) + if c.autoFailoverSupport { + c.Cluster.AddEventHandler(c.onClusterNodeUpdate) + } return c } @@ -209,6 +226,39 @@ func (c *EgressController) onLocalIPUpdate(ip string, added bool) { } } +func (c *EgressController) onClusterNodeUpdate(nodeName string, join bool) { + var egresses []*crdv1a2.Egress + if join { + // list all egress of local node, and move not hit egress + c.egressStatesMutex.RLock() + defer c.egressStatesMutex.RUnlock() + for egressName := range c.egressStates { + if c.Cluster.ShouldSelect(egressName) { + continue + } + egress, _ := c.egressLister.Get(egressName) + egresses = append(egresses, egress) + } + klog.V(0).Infof("Detected cluster node (%s) joined, and egress should unbind", nodeName) + } else { + // list egress owned by left node and handler the egress if hit by local node + egressesOfLeftNode, _ := c.egressLister.List(labels.Everything()) + for _, egress := range egressesOfLeftNode { + if egress.Status.NodeName != nodeName { + continue + } + if c.Cluster.ShouldSelect(egress.Name) { + egresses = append(egresses, egress) + } + } + klog.V(0).Infof("Detected cluster node (%s) left, and egress should bind", nodeName) + } + for _, egress := range egresses { + c.enqueueEgress(egress) + } + klog.V(0).Infof("%d egress should move and enqueue egress worker", len(egresses)) +} + // Run will create defaultWorkers workers (go routines) which will process the Egress events from the // workqueue. func (c *EgressController) Run(stopCh <-chan struct{}) { @@ -219,12 +269,28 @@ func (c *EgressController) Run(stopCh <-chan struct{}) { go c.localIPDetector.Run(stopCh) + if c.autoFailoverSupport { + go c.Cluster.Run(stopCh) + } + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.egressListerSynced, c.localIPDetector.HasSynced) { return } go wait.NonSlidingUntil(c.watchEgressGroup, 5*time.Second, stopCh) + // if node support egress failover + if c.autoFailoverSupport { + // when a new node join gossip cluster, it should handle the egress that hit itself(moved from other nodes) + egresses, _ := c.egressLister.List(labels.Everything()) + + for _, e := range egresses { + if e.Spec.EgressFailoverPolicyType == crdv1a2.EgressFailoverPolicyTypeAuto && c.Cluster.ShouldSelect(e.Name) { + c.enqueueEgress(e) + } + } + } + for i := 0; i < defaultWorkers; i++ { go wait.Until(c.worker, time.Second, stopCh) } @@ -443,6 +509,22 @@ func (c *EgressController) unbindPodEgress(pod, egress string) (string, bool) { return "", false } +func (c *EgressController) updateEgressStatus(egress *crdv1a2.Egress, nodeName string) error { + klog.V(0).Infof("Egress status : %#v, update to : %s", egress.Status, nodeName) + if egress.Status.NodeName == nodeName { + return nil + } + toUpdate := egress.DeepCopy() + toUpdate.Status.NodeName = nodeName + if _, err := c.crdClient.CrdV1alpha2().Egresses().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}); err != nil { + klog.Warningf("egress update error: %s", err) + return err + } + klog.V(0).Infof("update egress %s status success", egress.Name) + metrics.AntreaEgressStatusUpdates.Inc() + return nil +} + func (c *EgressController) syncEgress(egressName string) error { startTime := time.Now() defer func() { @@ -466,6 +548,29 @@ func (c *EgressController) syncEgress(egressName string) error { return err } + if c.autoFailoverSupport && egress.Spec.EgressFailoverPolicyType == crdv1a2.EgressFailoverPolicyTypeAuto { + if c.Cluster.ShouldSelect(egressName) { + if !c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.Cluster.NodeConfig) + if err := a.AssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + if err := c.updateEgressStatus(egress, c.nodeName); err != nil { + return fmt.Errorf("update egress status error:%s", err.Error()) + } + klog.V(0).Infof("Assigned owner node %s for egress %s and update status", + c.nodeName, egressName) + } else { + if c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.Cluster.NodeConfig) + if err := a.UnAssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + } + } + eState, exist := c.getEgressState(egressName) // If the EgressIP changes, uninstalls this Egress first. if exist && eState.egressIP != egress.Spec.EgressIP { diff --git a/pkg/agent/controller/egress/ip_assign_advertise.go b/pkg/agent/controller/egress/ip_assign_advertise.go new file mode 100644 index 00000000000..73396eef7d2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise.go @@ -0,0 +1,20 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +type INodeEgressAssigner interface { + AssignOwnerNodeEgressIP(egressIp string) error + UnAssignOwnerNodeEgressIP(egressIp string) error +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_linux.go b/pkg/agent/controller/egress/ip_assign_advertise_linux.go new file mode 100644 index 00000000000..275096d0eb7 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_linux.go @@ -0,0 +1,115 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "fmt" + "net" + + "github.com/vishvananda/netlink" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/util/arping" + "antrea.io/antrea/pkg/agent/util/ndp" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig) + if err != nil { + return err + } + if err := assignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig) + if err != nil { + return err + } + if err := unassignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func getNodeLinkAddr(egressIP string, nodeConfig *config.NodeConfig) (*netlink.Addr, netlink.Link, error) { + egressSpecIP := net.ParseIP(egressIP) + egressIPMask := nodeConfig.NodeIPAddr.Mask + addr := netlink.Addr{IPNet: &net.IPNet{IP: egressSpecIP, Mask: egressIPMask}} + gwLinkIndex := nodeConfig.GatewayConfig.LinkIndex + link, err := netlink.LinkByIndex(gwLinkIndex) + if err != nil { + return nil, nil, fmt.Errorf("get netlink by gw index(%d) error: %+v", gwLinkIndex, err) + } + klog.V(2).Infof("get node iface netlink and address: %+v, %+v", link, addr) + return &addr, link, nil +} + +func assignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // assign IP + ifaceName := link.Attrs().Name + ifaceH, _ := net.InterfaceByName(ifaceName) + klog.V(2).Infof("Adding address %+v to interface %s", addr, ifaceName) + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("failed to add address %v to interface %s: %v", addr, ifaceName, err) + } + egressSpecIP := addr.IP + isIPv4 := egressSpecIP.To4() + if isIPv4 != nil { + if err := arping.GratuitousARPOverIface(isIPv4, ifaceH); err != nil { + klog.Warningf("Failed to send gratuitous ARP: %v", err) + return err + } + klog.V(0).Infof("Send gratuitous ARP: %+v", isIPv4) + } else if isIPv6 := egressSpecIP.To16(); isIPv6 != nil { + if err := ndp.NDPNeighborAdvertiseOverIface(isIPv6, ifaceH); err != nil { + klog.Warningf("Failed to send Advertisement: %v", err) + return err + } + } + return nil +} + +func unassignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // check ip, if existed, uninstall ip + ifaceName := link.Attrs().Name + klog.V(2).Infof("Deleting address %v to interface %s", addr, ifaceName) + if err := netlink.AddrDel(link, addr); err != nil { + return fmt.Errorf("failed to delete address %v to interface %s: %v", addr, ifaceName, err) + } + return nil +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_windows.go b/pkg/agent/controller/egress/ip_assign_advertise_windows.go new file mode 100644 index 00000000000..88f657ddde2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_windows.go @@ -0,0 +1,45 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "antrea.io/antrea/pkg/agent/config" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index 4aba4745135..39b550a3739 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -101,7 +101,7 @@ func TestMemberlistServer_Run(t *testing.T) { cache.WaitForCacheSync(ctx.Done(), nodeInformer.Informer().HasSynced) s.AddEventHandler(func(ip string, added bool) { - t.Logf("notified node %s added (%t) node event handler", nodeName, added) + t.Logf("Notified node %s added (%t) node event handler", nodeName, added) }) go s.Run(stopCh) diff --git a/pkg/agent/util/ndp/ndp_test.go b/pkg/agent/util/ndp/ndp_test.go index f8d07b09a4c..1369799d62d 100644 --- a/pkg/agent/util/ndp/ndp_test.go +++ b/pkg/agent/util/ndp/ndp_test.go @@ -66,7 +66,7 @@ func TestAdvertiserMarshalMessage(t *testing.T) { b, err = marshalMessage(msg) return }(tt.ipv6Addr, tt.hardwareAddr); err != nil || !reflect.DeepEqual(got, tt.want) { - t.Errorf("NDPmarshalMessage() = %v, want %v", got, tt.want) + t.Errorf("NDPmarshalMessage() = %v, want %v, error:%v", got, tt.want, err) } }) } @@ -84,7 +84,7 @@ func MustIPv6(s string) (net.IP, error) { func TestNDPNeighborAdvertiseOverIface(t *testing.T) { ip, err := MustIPv6("fe80::250:56ff:fea7:e29d") if err != nil { - fmt.Println("invalid ipv6: ", err) + t.Logf("Invalid ipv6: %v", err) return } @@ -102,7 +102,7 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { conICMP, err := icmp.ListenPacket("ip6:ipv6-icmp", "fe80::1%lo0") if err != nil { - t.Logf("listen icmp ipv6 local error: %q", err.Error()) + t.Logf("Listen icmp ipv6 local error: %q", err.Error()) monkey.Patch(NewNDPAdvertiser, func(iface *net.Interface) (INDPAdvertiser, error) { s := &Advertiser{ fakeIntFace.HardwareAddr, @@ -115,7 +115,7 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { } return s, nil }) - t.Skipf("cannot test ICMPv6 NDP: %q", err.Error()) + t.Skipf("Cannot test ICMPv6 NDP: %q", err.Error()) } monkey.Patch(icmp.ListenPacket, func(network, address string) (*icmp.PacketConn, error) { return conICMP, nil @@ -125,11 +125,11 @@ func TestNDPNeighborAdvertiseOverIface(t *testing.T) { ifaceName := "ens192" iface, err := net.InterfaceByName(ifaceName) if err != nil { - fmt.Println("interface not found", ifaceName) + t.Errorf("Interface %s not found", ifaceName) return } if err := NDPNeighborAdvertiseOverIface(ip, iface); err != nil { - t.Errorf("new NDP gratuitous advertisement error:%q", err.Error()) + t.Errorf("New NDP gratuitous advertisement error:%q", err.Error()) } } diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 19c8afaef7f..711ece2b41f 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -190,7 +190,6 @@ type AppliedTo struct { // +genclient // +genclient:nonNamespaced -// +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // Egress defines which egress (SNAT) IP the traffic from the selected Pods to @@ -202,6 +201,15 @@ type Egress struct { // Specification of the desired behavior of Egress. Spec EgressSpec `json:"spec"` + + // Egress status show the owner node name of egress, if failoverPolicy set auto. + Status EgressStatus `json:"status"` +} + +// EgressStatus owner node name of Egress +type EgressStatus struct { + // The name of the Node that holds the Egress IP. + NodeName string `json:"nodeName"` } // EgressSpec defines the desired state for Egress. @@ -210,8 +218,24 @@ type EgressSpec struct { AppliedTo AppliedTo `json:"appliedTo"` // EgressIP specifies the SNAT IP address for the selected workloads. EgressIP string `json:"egressIP"` + // FailoverPolicy specifies if the Egress IP should failover to another Node + // when the Node that holds it becomes unavailable. + // The default value is "None". + EgressFailoverPolicyType `json:"failoverPolicy"` } +type EgressFailoverPolicyType string + +const ( + // EgressFailoverPolicyTypeNone disables failover for this Egress. The + // Egress IP must be assigned to an arbitrary interface of one Node, and be + // reassigned to another Node manually in case of Node failure. + EgressFailoverPolicyTypeNone EgressFailoverPolicyType = "None" + // EgressFailoverPolicyTypeAuto enables auto failover for this Egress. The + // Egress IP must not be assigned to any interface of any Node in advance. + EgressFailoverPolicyTypeAuto EgressFailoverPolicyType = "Auto" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type EgressList struct { diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index 1349bf9a243..62fc32200bc 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -122,6 +122,7 @@ func (in *Egress) DeepCopyInto(out *Egress) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status return } @@ -193,6 +194,22 @@ func (in *EgressSpec) DeepCopy() *EgressSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressStatus) DeepCopyInto(out *EgressStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressStatus. +func (in *EgressStatus) DeepCopy() *EgressStatus { + if in == nil { + return nil + } + out := new(EgressStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go index 784d5c1aec8..52326a2f0ce 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go @@ -38,6 +38,7 @@ type EgressesGetter interface { type EgressInterface interface { Create(ctx context.Context, egress *v1alpha2.Egress, opts v1.CreateOptions) (*v1alpha2.Egress, error) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) + UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha2.Egress, error) @@ -126,6 +127,21 @@ func (c *egresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1. return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *egresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (result *v1alpha2.Egress, err error) { + result = &v1alpha2.Egress{} + err = c.client.Put(). + Resource("egresses"). + Name(egress.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(egress). + Do(ctx). + Into(result) + return +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *egresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { return c.client.Delete(). diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go index 6e142d760b1..9c463ebf406 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go @@ -94,6 +94,17 @@ func (c *FakeEgresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts return obj.(*v1alpha2.Egress), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeEgresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(egressesResource, "status", egress), &v1alpha2.Egress{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.Egress), err +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *FakeEgresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. diff --git a/pkg/controller/metrics/prometheus.go b/pkg/controller/metrics/prometheus.go index 1d8735c0c9b..d346ab67efa 100644 --- a/pkg/controller/metrics/prometheus.go +++ b/pkg/controller/metrics/prometheus.go @@ -96,6 +96,13 @@ var ( Help: "The total number of actual status updates performed for Antrea NetworkPolicy Custom Resources", StabilityLevel: metrics.ALPHA, }) + AntreaEgressStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ + Namespace: metricNamespaceAntrea, + Subsystem: metricSubsystemController, + Name: "eg_status_updates", + Help: "The total number of actual status updates performed for Antrea Egress Custom Resources", + StabilityLevel: metrics.ALPHA, + }) AntreaClusterNetworkPolicyStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ Namespace: metricNamespaceAntrea, Subsystem: metricSubsystemController,