diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 5c8c59ef4c4..7e2ec1d8506 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3278,10 +3294,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 6b0a25e656c..00d5e1dbfcf 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3278,10 +3294,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 21e82a208eb..184a057f8cc 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3278,10 +3294,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index c8958d062fc..8dd196f7868 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3278,10 +3294,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index b75e88e6f1b..7860837318a 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1345,6 +1345,10 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string name: v1alpha2 schema: openAPIV3Schema: @@ -1412,14 +1416,26 @@ spec: type: string externalIPPool: type: string + failoverPolicy: + enum: + - Auto + - None + type: string required: - appliedTo type: object + status: + properties: + nodeName: + type: string + type: object required: - spec type: object served: true storage: true + subresources: + status: {} --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -3278,10 +3294,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/build/yamls/base/agent-rbac.yml b/build/yamls/base/agent-rbac.yml index 61db6ab0fa4..cce066df1bb 100644 --- a/build/yamls/base/agent-rbac.yml +++ b/build/yamls/base/agent-rbac.yml @@ -145,10 +145,13 @@ rules: - crd.antrea.io resources: - egresses + - egresses/status verbs: - get - watch - list + - update + - patch --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/build/yamls/base/crds.yml b/build/yamls/base/crds.yml index 38a1867b21c..382ab41ad0c 100644 --- a/build/yamls/base/crds.yml +++ b/build/yamls/base/crds.yml @@ -81,6 +81,16 @@ spec: - format: ipv6 externalIPPool: type: string + failoverPolicy: + type: string + enum: + - Auto + - None + status: + type: object + properties: + nodeName: + type: string additionalPrinterColumns: - description: Specifies the SNAT IP address for the selected workloads. jsonPath: .spec.egressIP @@ -89,6 +99,12 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - description: The Owner Node of egress IP + jsonPath: .status.nodeName + name: Status + type: string + subresources: + status: {} scope: Cluster names: plural: egresses diff --git a/go.mod b/go.mod index 44131667a54..746901f4b3f 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/go-openapi/spec v0.19.5 github.com/gogo/protobuf v1.3.2 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e - github.com/golang/mock v1.5.0 + github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.0 github.com/google/uuid v1.1.2 github.com/hashicorp/memberlist v0.2.4 @@ -47,10 +47,10 @@ require ( github.com/vmware/go-ipfix v0.5.2 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 - golang.org/x/mod v0.4.0 - golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 - golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 + golang.org/x/mod v0.4.2 + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba google.golang.org/grpc v1.27.1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 diff --git a/go.sum b/go.sum index 9d74c5f3f40..49e1f264b7f 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,9 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -624,6 +625,7 @@ github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6Ut github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= @@ -695,8 +697,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -735,8 +737,9 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -749,8 +752,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -801,8 +805,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -863,6 +869,7 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/agent/controller/egress/egress_controller.go b/pkg/agent/controller/egress/egress_controller.go index 0f8d16cf44c..2e9a888aa7c 100644 --- a/pkg/agent/controller/egress/egress_controller.go +++ b/pkg/agent/controller/egress/egress_controller.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" @@ -40,8 +41,10 @@ import ( "antrea.io/antrea/pkg/agent/route" cpv1b2 "antrea.io/antrea/pkg/apis/controlplane/v1beta2" crdv1a2 "antrea.io/antrea/pkg/apis/crd/v1alpha2" + clientsetversioned "antrea.io/antrea/pkg/client/clientset/versioned" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" crdlisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/controller/metrics" "antrea.io/antrea/pkg/util/k8s" ) @@ -102,6 +105,7 @@ type egressBinding struct { type EgressController struct { ofClient openflow.Client routeClient route.Interface + crdClient clientsetversioned.Interface antreaClientProvider agent.AntreaClientProvider egressInformer cache.SharedIndexInformer @@ -179,6 +183,7 @@ func NewEgressController( resyncPeriod, ) localIPDetector.AddEventHandler(c.onLocalIPUpdate) + c.cluster.AddClusterNodeEventHandler(c.onClusterNodeUpdate) return c } @@ -214,6 +219,39 @@ func (c *EgressController) onLocalIPUpdate(ip string, added bool) { } } +func (c *EgressController) onClusterNodeUpdate(nodeName string, join bool) { + var egresses []*crdv1a2.Egress + if join { + // list all egress of local node, and move not hit egress + c.egressStatesMutex.RLock() + defer c.egressStatesMutex.RUnlock() + for egressName := range c.egressStates { + if c.cluster.ShouldSelect(egressName) { + continue + } + egress, _ := c.egressLister.Get(egressName) + egresses = append(egresses, egress) + } + klog.V(0).InfoS("Detected cluster node joined, egress should unbind", "nodeName", nodeName) + } else { + // list egress owned by left node and handler the egress if hit by local node + egressesOfLeftNode, _ := c.egressLister.List(labels.Everything()) + for _, egress := range egressesOfLeftNode { + if egress.Status.NodeName != nodeName { + continue + } + if c.cluster.ShouldSelect(egress.Name) { + egresses = append(egresses, egress) + } + } + klog.V(0).InfoS("Detected cluster node left, and egress should bind", "nodeName", nodeName) + } + for _, egress := range egresses { + c.enqueueEgress(egress) + } + klog.V(0).InfoS("Egress should move and enqueue egress worker", "num", len(egresses)) +} + // Run will create defaultWorkers workers (go routines) which will process the Egress events from the // workqueue. func (c *EgressController) Run(stopCh <-chan struct{}) { @@ -232,6 +270,18 @@ func (c *EgressController) Run(stopCh <-chan struct{}) { go wait.NonSlidingUntil(c.watchEgressGroup, 5*time.Second, stopCh) + // if node support egress failover + if c.cluster.LocalNodeJoined() { + // when a new node join gossip cluster, it should handle the egress that hit itself(moved from other nodes) + egresses, _ := c.egressLister.List(labels.Everything()) + + for _, e := range egresses { + if e.Spec.ExternalIPPool != "" && c.cluster.ShouldSelect(e.Name) { + c.enqueueEgress(e) + } + } + } + for i := 0; i < defaultWorkers; i++ { go wait.Until(c.worker, time.Second, stopCh) } @@ -450,6 +500,22 @@ func (c *EgressController) unbindPodEgress(pod, egress string) (string, bool) { return "", false } +func (c *EgressController) updateEgressStatus(egress *crdv1a2.Egress, nodeName string) error { + klog.V(0).Infof("Egress status : %#v, update to : %s", egress.Status, nodeName) + if egress.Status.NodeName == nodeName { + return nil + } + toUpdate := egress.DeepCopy() + toUpdate.Status.NodeName = nodeName + if _, err := c.crdClient.CrdV1alpha2().Egresses().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}); err != nil { + klog.Warningf("egress update error: %s", err) + return err + } + klog.V(0).Infof("update egress %s status success", egress.Name) + metrics.AntreaEgressStatusUpdates.Inc() + return nil +} + func (c *EgressController) syncEgress(egressName string) error { startTime := time.Now() defer func() { @@ -473,6 +539,29 @@ func (c *EgressController) syncEgress(egressName string) error { return err } + if egress.Spec.ExternalIPPool != "" && c.cluster.LocalNodeJoined() { + if c.cluster.ShouldSelect(egressName) { + if !c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.cluster.NodeConfig) + if err := a.AssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + if err := c.updateEgressStatus(egress, c.nodeName); err != nil { + return fmt.Errorf("update egress status error:%v", err) + } + klog.V(0).InfoS("Assigned owner node for egress and update status", + "nodeName", c.nodeName, "egress", egressName) + } else { + if c.localIPDetector.IsLocalIP(egress.Spec.EgressIP) { + a := NewNodeEgressIPAssigner(c.cluster.NodeConfig) + if err := a.UnAssignOwnerNodeEgressIP(egress.Spec.EgressIP); err != nil { + return err + } + } + } + } + eState, exist := c.getEgressState(egressName) // If the EgressIP changes, uninstalls this Egress first. if exist && eState.egressIP != egress.Spec.EgressIP { diff --git a/pkg/agent/controller/egress/ip_assign_advertise.go b/pkg/agent/controller/egress/ip_assign_advertise.go new file mode 100644 index 00000000000..73396eef7d2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise.go @@ -0,0 +1,20 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +type INodeEgressAssigner interface { + AssignOwnerNodeEgressIP(egressIp string) error + UnAssignOwnerNodeEgressIP(egressIp string) error +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_linux.go b/pkg/agent/controller/egress/ip_assign_advertise_linux.go new file mode 100644 index 00000000000..6503f3e926f --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_linux.go @@ -0,0 +1,118 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "errors" + "fmt" + "net" + + "github.com/vishvananda/netlink" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/agent/util/arping" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig.NodeIPAddr.IP) + if err != nil { + return err + } + if err := assignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + addr, link, err := getNodeLinkAddr(egressIP, nodeConfig.NodeIPAddr.IP) + if err != nil { + return err + } + if err := unassignEgressIP(addr, link); err != nil { + return err + } + return nil +} + +func getNodeLinkAddr(egressIP string, nodeIPAddr net.IP) (*netlink.Addr, netlink.Link, error) { + egressSpecIP := net.ParseIP(egressIP) + localAddr, localIntf, err := util.GetIPNetDeviceFromIP(nodeIPAddr) + if err != nil { + return nil, nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err) + } + link, err := netlink.LinkByName(localIntf.Name) + if err != nil { + return nil, nil, fmt.Errorf("get netlink by name(%s) error: %+v", localIntf.Name, err) + } + addr := netlink.Addr{IPNet: &net.IPNet{IP: egressSpecIP, Mask: localAddr.Mask}} + + klog.V(2).Infof("get node iface netlink and address: %+v, %+v", link, addr) + return &addr, link, nil +} + +func assignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // assign IP + ifaceName := link.Attrs().Name + ifaceH, _ := net.InterfaceByName(ifaceName) + klog.V(2).Infof("Adding address %+v to interface %s", addr, ifaceName) + if err := netlink.AddrAdd(link, addr); err != nil { + return fmt.Errorf("failed to add address %v to interface %s: %v", addr, ifaceName, err) + } + egressSpecIP := addr.IP + isIPv4 := egressSpecIP.To4() + if isIPv4 != nil { + if err := arping.GratuitousARPOverIface(isIPv4, ifaceH); err != nil { + klog.Warningf("Failed to send gratuitous ARP: %v", err) + return err + } + klog.Infof("Send gratuitous ARP: %+v", isIPv4) + } else if isIPv6 := egressSpecIP.To16(); isIPv6 != nil { + err := errors.New("IPv6 not support") + klog.Warningf("Failed to send Advertisement: %v", err) + return err + } + return nil +} + +func unassignEgressIP(addr *netlink.Addr, link netlink.Link) error { + // check ip, if existed, uninstall ip + ifaceName := link.Attrs().Name + klog.V(2).Infof("Deleting address %v to interface %s", addr, ifaceName) + if err := netlink.AddrDel(link, addr); err != nil { + return fmt.Errorf("failed to delete address %v to interface %s: %v", addr, ifaceName, err) + } + return nil +} diff --git a/pkg/agent/controller/egress/ip_assign_advertise_windows.go b/pkg/agent/controller/egress/ip_assign_advertise_windows.go new file mode 100644 index 00000000000..88f657ddde2 --- /dev/null +++ b/pkg/agent/controller/egress/ip_assign_advertise_windows.go @@ -0,0 +1,45 @@ +// Copyright 2021 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package egress + +import ( + "antrea.io/antrea/pkg/agent/config" +) + +type nodeEgressAssigner struct { + nodeConfig *config.NodeConfig +} + +func NewNodeEgressIPAssigner(c *config.NodeConfig) *nodeEgressAssigner { + return &nodeEgressAssigner{ + c, + } +} + +func (a *nodeEgressAssigner) AssignOwnerNodeEgressIP(egressIp string) error { + return assignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func (a *nodeEgressAssigner) UnAssignOwnerNodeEgressIP(egressIp string) error { + return unAssignOwnerNodeEgressIP(egressIp, a.nodeConfig) +} + +func assignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} + +func unAssignOwnerNodeEgressIP(egressIP string, nodeConfig *config.NodeConfig) error { + return nil +} diff --git a/pkg/agent/memberlist/cluster.go b/pkg/agent/memberlist/cluster.go index d0faba9c0e5..f41e8ffe9a3 100644 --- a/pkg/agent/memberlist/cluster.go +++ b/pkg/agent/memberlist/cluster.go @@ -465,7 +465,7 @@ func (c *Cluster) updateLocalNodeStatus() { return } shouldJoin := c.shouldJoinCluster(c.nodeName) - joined := c.localNodeJoined() + joined := c.LocalNodeJoined() if joined && !shouldJoin { c.writeLocalNodeCh(true) } else if !joined && shouldJoin { @@ -492,13 +492,13 @@ func (c *Cluster) nodeList() []string { return nodes } -// localNodeJoined if merbers num in cluster is 1 means local node not joined in other clusters -func (c *Cluster) localNodeJoined() bool { +// LocalNodeJoined if merbers num in cluster is 1 means local node not joined in other clusters +func (c *Cluster) LocalNodeJoined() bool { // while node joined cluster, num of member will >= 1; if leave cluster, num of member will be 0 return c.mList.NumMembers() >= 1 } -func (c *Cluster) shouldSelect(name string) bool { +func (c *Cluster) ShouldSelect(name string) bool { c.conHashMapRWLock.RLock() defer c.conHashMapRWLock.RUnlock() myNode := c.nodeName @@ -517,6 +517,6 @@ func (c *Cluster) notify(nodeName string, isJoinNode bool) { } } -func (c *Cluster) addClusterNodeEventHandler(handler clusterNodeEventHandler) { +func (c *Cluster) AddClusterNodeEventHandler(handler clusterNodeEventHandler) { c.ClusterNodeEventHandlers = append(c.ClusterNodeEventHandlers, handler) } diff --git a/pkg/agent/memberlist/cluster_test.go b/pkg/agent/memberlist/cluster_test.go index 38a17528b3a..6f8b524a4bb 100644 --- a/pkg/agent/memberlist/cluster_test.go +++ b/pkg/agent/memberlist/cluster_test.go @@ -123,7 +123,7 @@ func TestNewCluster(t *testing.T) { cache.WaitForCacheSync(stopCh, nodeInformer.Informer().HasSynced) cache.WaitForCacheSync(stopCh, ipPoolInformer.Informer().HasSynced) - s.addClusterNodeEventHandler(func(nodeName string, added bool) { + s.AddClusterNodeEventHandler(func(nodeName string, added bool) { t.Logf("notified node %s added (%t) node event handler", nodeName, added) }) @@ -284,7 +284,7 @@ func TestCluster_Run(t *testing.T) { cache.WaitForCacheSync(stopCh, nodeInformer.Informer().HasSynced) cache.WaitForCacheSync(stopCh, ipPoolInformer.Informer().HasSynced) - s.addClusterNodeEventHandler(func(nodeName string, added bool) { + s.AddClusterNodeEventHandler(func(nodeName string, added bool) { t.Logf("notified node %s added (%t) node event handler", nodeName, added) }) @@ -752,7 +752,7 @@ func TestCluster_ShouldSelect(t *testing.T) { hitCount := 0 for i := 0; i < egressNum; i++ { egressName := fmt.Sprintf("%s-%d", genRandomStr(10), i) - if node1Cluster.shouldSelect(egressName) { + if node1Cluster.ShouldSelect(egressName) { hitCount++ } } diff --git a/pkg/apis/crd/v1alpha2/types.go b/pkg/apis/crd/v1alpha2/types.go index 6393886a43b..02c2a0e18f2 100644 --- a/pkg/apis/crd/v1alpha2/types.go +++ b/pkg/apis/crd/v1alpha2/types.go @@ -190,7 +190,6 @@ type AppliedTo struct { // +genclient // +genclient:nonNamespaced -// +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // Egress defines which egress (SNAT) IP the traffic from the selected Pods to @@ -202,6 +201,15 @@ type Egress struct { // Specification of the desired behavior of Egress. Spec EgressSpec `json:"spec"` + + // Egress status show the owner node name of egress, if failoverPolicy set auto. + Status EgressStatus `json:"status"` +} + +// EgressStatus owner node name of Egress +type EgressStatus struct { + // The name of the Node that holds the Egress IP. + NodeName string `json:"nodeName"` } // EgressSpec defines the desired state for Egress. diff --git a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go index cdf7e23580f..bb2587fb0c7 100644 --- a/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha2/zz_generated.deepcopy.go @@ -122,6 +122,7 @@ func (in *Egress) DeepCopyInto(out *Egress) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status return } @@ -193,6 +194,22 @@ func (in *EgressSpec) DeepCopy() *EgressSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressStatus) DeepCopyInto(out *EgressStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressStatus. +func (in *EgressStatus) DeepCopy() *EgressStatus { + if in == nil { + return nil + } + out := new(EgressStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Endpoint) DeepCopyInto(out *Endpoint) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go index 784d5c1aec8..52326a2f0ce 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/egress.go @@ -38,6 +38,7 @@ type EgressesGetter interface { type EgressInterface interface { Create(ctx context.Context, egress *v1alpha2.Egress, opts v1.CreateOptions) (*v1alpha2.Egress, error) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) + UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha2.Egress, error) @@ -126,6 +127,21 @@ func (c *egresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts v1. return } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *egresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (result *v1alpha2.Egress, err error) { + result = &v1alpha2.Egress{} + err = c.client.Put(). + Resource("egresses"). + Name(egress.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(egress). + Do(ctx). + Into(result) + return +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *egresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { return c.client.Delete(). diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go index 6e142d760b1..9c463ebf406 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha2/fake/fake_egress.go @@ -94,6 +94,17 @@ func (c *FakeEgresses) Update(ctx context.Context, egress *v1alpha2.Egress, opts return obj.(*v1alpha2.Egress), err } +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeEgresses) UpdateStatus(ctx context.Context, egress *v1alpha2.Egress, opts v1.UpdateOptions) (*v1alpha2.Egress, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(egressesResource, "status", egress), &v1alpha2.Egress{}) + if obj == nil { + return nil, err + } + return obj.(*v1alpha2.Egress), err +} + // Delete takes name of the egress and deletes it. Returns an error if one occurs. func (c *FakeEgresses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { _, err := c.Fake. diff --git a/pkg/controller/metrics/prometheus.go b/pkg/controller/metrics/prometheus.go index 1d8735c0c9b..d346ab67efa 100644 --- a/pkg/controller/metrics/prometheus.go +++ b/pkg/controller/metrics/prometheus.go @@ -96,6 +96,13 @@ var ( Help: "The total number of actual status updates performed for Antrea NetworkPolicy Custom Resources", StabilityLevel: metrics.ALPHA, }) + AntreaEgressStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ + Namespace: metricNamespaceAntrea, + Subsystem: metricSubsystemController, + Name: "eg_status_updates", + Help: "The total number of actual status updates performed for Antrea Egress Custom Resources", + StabilityLevel: metrics.ALPHA, + }) AntreaClusterNetworkPolicyStatusUpdates = metrics.NewCounter(&metrics.CounterOpts{ Namespace: metricNamespaceAntrea, Subsystem: metricSubsystemController, diff --git a/plugins/octant/go.sum b/plugins/octant/go.sum index dc61f2ce0a0..1a8d937a3fa 100644 --- a/plugins/octant/go.sum +++ b/plugins/octant/go.sum @@ -300,8 +300,9 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -662,6 +663,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= @@ -741,8 +743,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -788,8 +790,9 @@ golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201024042810-be3efd7ff127/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -805,8 +808,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -864,8 +868,10 @@ golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -939,8 +945,9 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=