From 0bac6f58be587ee25eae979ab65978ed74193117 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Wed, 18 May 2022 17:57:39 +0800 Subject: [PATCH 1/2] Add Multi-cluster feature in Agent * Add a new feature gate `Multicluster` and configs in antrea-agent.conf, and a few extra items in antrea-agent cluster role including access to `Gateway` and `ClusterInfoImport`. * Rename the `ServiceMarkTable` to `SNATMarkTable`. * Add a controller for Gateway Nodes to watch Gateway and ClusterInfoImport's events. It will set up a few openflow rules to forward cross-cluster traffic to remote Gateway Nodes. * Add a classification rule for cross-cluster traffic with global multicluster virtual MAC `aa:bb:cc:dd:ee:f0`. A sample is like below: ``` table=Classifier, priority=210,in_port="antrea-tun0",dl_dst=aa:bb:cc:dd:ee:f0 actions=load:0x1->NXM_NX_REG0[0..3],load:0x1->NXM_NX_REG0[9],resubmit(,SNATConntrackZone) ``` * Add a rule in `L3Forwarding` table for cross-cluster request packets that modifies the destination MAC to global multicluster virtual MAC. A sample is like below (the destination CIDR is remote Service ClusterIP CIDR, the tunnel IP in NXM_NX_TUN_IPV4_DST is remote Gateway IP): ``` table=L3Forwarding, priority=200,ip,nw_dst=10.96.0.0/12 actions=mod_dl_src:ee:73:a5:81:09:c6,mod_dl_dst:aa:bb:cc:dd:ee:f0,load:0xab01b39->NXM_NX_TUN_IPV4_DST[],load:0x1->NXM_NX_REG0[4..7],resubmit(,L3DecTTL) ``` * Add a rule in `L3Forwarding` table for cross-cluster reply packets. A sample is like below (the destination IP is remote Gateway IP): ``` table=L3Forwarding, priority=200,ct_state=+rpl+trk,ip,nw_dst=10.176.27.57 actions=mod_dl_src:ee:73:a5:81:09:c6,mod_dl_dst:aa:bb:cc:dd:ee:f0,load:0xab01b39->NXM_NX_TUN_IPV4_DST[],load:0x1->NXM_NX_REG0[4..7],resubmit(,L3DecTTL) ``` * Add a rule to `SNATMark` table to match the packets of multi-cluster Service connection and perform DNAT in DNAT zone. ``` table=SNATMark, priority=210,ip,nw_dst=10.96.0.0/12 actions=ct(commit,table=SNAT,zone=65520,exec(load:0x1->NXM_NX_CT_MARK[5])) ``` * Add a rule to `SNAT` table to perform SNAT for any remote cluster traffic. ``` table=SNAT, priority=200,ct_state=+new+trk,ip,nw_dst=10.96.0.0/12 actions=ct(commit,table=L2ForwardingCalc,zone=65521,nat(src=10.176.27.224)) ``` * Add a rule to `UnSNAT` table to perform de-SNAT if destination IP is local GatewayIP. ``` table=UnSNAT, priority=200,ip,nw_dst=10.176.27.224 actions=ct(table=ConntrackZone,zone=65521,nat) ``` * Add a rule in `L2ForwardingCalc` table to load the global virtual multi-cluster MAC's output to `antrea-tun0` ``` table=L2ForwardingCalc, priority=200,dl_dst=aa:bb:cc:dd:ee:f0 actions=load:0x1->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[8],resubmit(,22) ``` * Add a rule in `Output` table to match the multi-cluster traffic to forward the traffic from/to regular Node through the same port. ``` table=Output, priority=210,reg1=0x1,in_port=1 actions=IN_PORT ``` * Add a controller for regular Nodes to watch Gateway and ClusterInfoImport's events. It will set up a few openflow rules to forward cross-cluster traffic to local Gateway Node. * Add a rule in L3Forwarding table for cross-cluster request packets, and modify the destination MAC to global multicluster virtual MAC. A sample is like below (the destination CIDR is remote Service ClusterIP CIDR, the tunnel IP in NXM_NX_TUN_IPV4_DST is local Gateway's Internal IP.): ``` table=L3Forwarding, priority=200,ip,nw_dst=10.96.0.0/12 actions=mod_dl_src:f2:08:93:0c:82:bd,mod_dl_dst:aa:bb:cc:dd:ee:ff,load:0xab0193b->NXM_NX_TUN_IPV4_DST[],load:0x1->NXM_NX_REG0[4..7],resubmit(,L3DecTTL) ``` * Add a rule in L3Forwarding table for cross-cluster reply packets. A sample is like below (the destination IP is remote Gateway IP, the tunnel IP in NXM_NX_TUN_IPV4_DST is local Gateway's Internal IP): ``` table=L3Forwarding, priority=200,ct_state=+rpl+trk,ip,nw_dst=10.176.27.57 actions=mod_dl_src:f2:08:93:0c:82:bd,mod_dl_dst:aa:bb:cc:dd:ee:f0,load:0xab0193b->NXM_NX_TUN_IPV4_DST[],load:0x1->NXM_NX_REG0[4..7],resubmit(,L3DecTTL) ``` * Add a rule in L2ForwardingCalc table to load the global virtual multi-cluster MAC's output to `antrea-tun0` ``` table=L2ForwardingCalc, priority=200,dl_dst=aa:bb:cc:dd:ee:f0 actions=load:0x1->NXM_NX_REG1[],load:0x1->NXM_NX_REG0[8],resubmit(,22) ``` * Add unit test cases * Refine e2e test for data plane change Signed-off-by: Lan Luo Co-authored-by: Hongliang Liu --- build/charts/antrea/README.md | 2 + build/charts/antrea/conf/antrea-agent.conf | 14 + .../antrea/templates/agent/clusterrole.yaml | 16 + build/charts/antrea/values.yaml | 9 + build/yamls/antrea-aks.yml | 32 +- build/yamls/antrea-eks.yml | 32 +- build/yamls/antrea-gke.yml | 32 +- build/yamls/antrea-ipsec.yml | 32 +- build/yamls/antrea.yml | 32 +- ci/jenkins/test-mc.sh | 28 +- cmd/antrea-agent-simulator/simulator.go | 2 +- cmd/antrea-agent/agent.go | 35 +- cmd/antrea-agent/options.go | 5 + cmd/antrea-controller/controller.go | 2 +- .../multicluster/gateway_controller.go | 14 +- .../multicluster/gateway_controller_test.go | 14 +- multicluster/test/e2e/antreapolicy_test.go | 18 +- multicluster/test/e2e/fixtures.go | 10 +- multicluster/test/e2e/framework.go | 24 +- multicluster/test/e2e/main_test.go | 13 +- multicluster/test/e2e/service_test.go | 176 +++++-- pkg/agent/multicluster/mc_route_controller.go | 429 ++++++++++++++++++ .../multicluster/mc_route_controller_test.go | 287 ++++++++++++ pkg/agent/openflow/client.go | 93 +++- pkg/agent/openflow/client_test.go | 51 ++- pkg/agent/openflow/cookie/allocator.go | 3 + pkg/agent/openflow/framework.go | 14 +- pkg/agent/openflow/multicluster.go | 163 +++++++ pkg/agent/openflow/network_policy_test.go | 2 +- pkg/agent/openflow/pipeline.go | 27 +- pkg/agent/openflow/pipeline_test.go | 6 +- pkg/agent/openflow/testing/mock_openflow.go | 56 +++ .../handlers/featuregates/handler.go | 3 +- .../handlers/featuregates/handler_test.go | 1 + pkg/config/agent/config.go | 11 + pkg/features/antrea_features.go | 8 + pkg/ovs/openflow/ofctrl_action.go | 4 +- pkg/ovs/openflow/ofctrl_group.go | 2 +- pkg/util/k8s/client.go | 28 +- test/integration/agent/openflow_test.go | 44 +- 40 files changed, 1608 insertions(+), 166 deletions(-) create mode 100644 pkg/agent/multicluster/mc_route_controller.go create mode 100644 pkg/agent/multicluster/mc_route_controller_test.go create mode 100644 pkg/agent/openflow/multicluster.go diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index 0506a97cec2..c6e9b828cbc 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -82,6 +82,8 @@ Kubernetes: `>= 1.16.0-0` | logVerbosity | int | `0` | | | multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". | | multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. | +| multicluster.enable | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. | +| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is Antrea Agent Namespace if it's empty. | | noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. | | nodeIPAM.clusterCIDRs | list | `[]` | CIDR ranges to use when allocating Pod IP addresses. | | nodeIPAM.enable | bool | `false` | Enable Node IPAM in Antrea | diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index 0527661c0eb..e4d8fe64903 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -39,6 +39,10 @@ featureGates: # Enable multicast traffic. This feature is supported only with noEncap mode. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Multicast" "default" false) }} +# Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. +# This feature is supported only with encap mode. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "Multicluster" "default" false) }} + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -292,3 +296,13 @@ ipsec: # feature gate to be enabled. authenticationMode: {{ .authenticationMode | quote }} {{- end }} + +multicluster: +{{- with .Values.multicluster }} +# Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. +# This feature is supported only with encap mode. + enable: {{ .enable }} +# The Namespace where Antrea Multi-cluster Controller is running. +# The default is antrea-agent's Namespace. + namespace: {{ .namespace | quote }} +{{- end }} diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 4520872afdb..1759a860517 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -189,3 +189,19 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index b4a1b90bf67..759b8b2232c 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -281,6 +281,15 @@ logVerbosity: 0 whereabouts: enable: false +## -- Configure Multicluster, for use by the antrea-agent. +multicluster: + # -- Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # -- The Namespace where Antrea Multi-cluster Controller is running. + # The default is Antrea Agent Namespace if it's empty. + namespace: "" + testing: ## -- enable code coverage measurement (used when testing Antrea only). coverage: false diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 0957b19a79c..32a0a346e70 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -97,6 +97,10 @@ data: # Enable multicast traffic. This feature is supported only with noEncap mode. # Multicast: false + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + # Multicluster: false + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -323,6 +327,14 @@ data: # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` # feature gate to be enabled. authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -3033,6 +3045,22 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3569,7 +3597,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb + checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f labels: app: antrea component: antrea-agent @@ -3809,7 +3837,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb + checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index aa2c7efdd6a..e028e5ea3d7 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -97,6 +97,10 @@ data: # Enable multicast traffic. This feature is supported only with noEncap mode. # Multicast: false + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + # Multicluster: false + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -323,6 +327,14 @@ data: # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` # feature gate to be enabled. authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -3033,6 +3045,22 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3569,7 +3597,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb + checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f labels: app: antrea component: antrea-agent @@ -3811,7 +3839,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb + checksum/config: 1ede67e825b3122edca49b4f5bbb8932a921260b686a02c10c8889de24c8ae0f labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 47e83a0ec7c..00886191002 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -97,6 +97,10 @@ data: # Enable multicast traffic. This feature is supported only with noEncap mode. # Multicast: false + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + # Multicluster: false + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -323,6 +327,14 @@ data: # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` # feature gate to be enabled. authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -3033,6 +3045,22 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3569,7 +3597,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 + checksum/config: 2f5a57b910bfb442df5abb3268308a3f4ad8f69d506e4281dddad39dab334690 labels: app: antrea component: antrea-agent @@ -3809,7 +3837,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 + checksum/config: 2f5a57b910bfb442df5abb3268308a3f4ad8f69d506e4281dddad39dab334690 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 7ea65444221..933137e4824 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -110,6 +110,10 @@ data: # Enable multicast traffic. This feature is supported only with noEncap mode. # Multicast: false + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + # Multicluster: false + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -336,6 +340,14 @@ data: # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` # feature gate to be enabled. authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -3046,6 +3058,22 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3582,7 +3610,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b + checksum/config: 009306e63cddc96c9dd51d20543783c6a94e9859581dc9db4dffacc8a78976bc checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -3868,7 +3896,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b + checksum/config: 009306e63cddc96c9dd51d20543783c6a94e9859581dc9db4dffacc8a78976bc labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 3ec7336f8cc..034e446dd8e 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -97,6 +97,10 @@ data: # Enable multicast traffic. This feature is supported only with noEncap mode. # Multicast: false + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + # Multicluster: false + # Enable support for provisioning secondary network interfaces for Pods (using # Pod annotations). At the moment, Antrea can only create secondary network # interfaces using SR-IOV VFs on baremetal Nodes. @@ -323,6 +327,14 @@ data: # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` # feature gate to be enabled. authenticationMode: "psk" + + multicluster: + # Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. + # This feature is supported only with encap mode. + enable: false + # The Namespace where Antrea Multi-cluster Controller is running. + # The default is antrea-agent's Namespace. + namespace: "" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -3033,6 +3045,22 @@ rules: - watch - list - create + - apiGroups: + - multicluster.crd.antrea.io + resources: + - gateways + verbs: + - get + - list + - watch + - apiGroups: + - multicluster.crd.antrea.io + resources: + - clusterinfoimports + verbs: + - get + - list + - watch --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3569,7 +3597,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 + checksum/config: 9f60e42d8b53705c8d6fa0f789b0b362e78e81a93f4ae71dc590b95fb4a4933d labels: app: antrea component: antrea-agent @@ -3809,7 +3837,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 + checksum/config: 9f60e42d8b53705c8d6fa0f789b0b362e78e81a93f4ae71dc590b95fb4a4933d labels: app: antrea component: antrea-controller diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 470c26fd97f..3e209e5f5bd 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -32,8 +32,7 @@ MULTICLUSTER_KUBECONFIG_PATH=$WORKDIR/.kube LEADER_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/leader" EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east" WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west" - -NGINX_IMAGE=projects.registry.vmware.com/antrea/nginx:1.21.6-alpine +ENABLE_MC_GATEWAY=false CONTROL_PLANE_NODE_ROLE="control-plane,master" @@ -44,14 +43,15 @@ membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) CLEAN_STALE_IMAGES="docker system prune --force --all --filter until=48h" _usage="Usage: $0 [--kubeconfigs-path ] [--workdir ] - [--testcase ] + [--testcase ] [--mc-gateway] Run Antrea multi-cluster e2e tests on a remote (Jenkins) Linux Cluster Set. --kubeconfigs-path Path of cluster set kubeconfigs. --workdir Home path for Go, vSphere information and antrea_logs during cluster setup. Default is $WORKDIR. --testcase Antrea multi-cluster e2e test cases on a Linux cluster set. - --registry The docker registry to use instead of dockerhub." + --registry The docker registry to use instead of dockerhub. + --mc-gateway Enable Multicluster Gateway." function print_usage { echoerr "$_usage" @@ -79,6 +79,10 @@ case $key in DOCKER_REGISTRY="$2" shift 2 ;; + --mc-gateway) + ENABLE_MC_GATEWAY=true + shift + ;; -h|--help) print_usage exit 0 @@ -223,7 +227,7 @@ function deliver_multicluster_controller { export NO_PULL=1;make antrea-mc-controller - docker save projects.registry.vmware.com/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar + docker save "${DOCKER_REGISTRY}"/antrea/antrea-mc-controller:latest -o "${WORKDIR}"/antrea-mcs.tar ./multicluster/hack/generate-manifest.sh -l antrea-mcs-ns > ./multicluster/test/yamls/manifest.yml for kubeconfig in "${multicluster_kubeconfigs[@]}" @@ -257,14 +261,17 @@ function run_multicluster_e2e { export GOCACHE=${WORKDIR}/.cache/go-build export PATH=$GOROOT/bin:$PATH + if [[ ${ENABLE_MC_GATEWAY} ]]; then + sed -i.bak -E "s/#[[:space:]]*Multicluster[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/ Multicluster: true/" build/yamls/antrea.yml + fi wait_for_antrea_multicluster_pods_ready "${LEADER_CLUSTER_CONFIG}" wait_for_antrea_multicluster_pods_ready "${EAST_CLUSTER_CONFIG}" wait_for_antrea_multicluster_pods_ready "${WEST_CLUSTER_CONFIG}" wait_for_multicluster_controller_ready - docker pull $NGINX_IMAGE - docker save $NGINX_IMAGE -o "${WORKDIR}"/nginx.tar + docker pull "${DOCKER_REGISTRY}"/antrea/nginx:1.21.6-alpine + docker save "${DOCKER_REGISTRY}"/antrea/nginx:1.21.6-alpine -o "${WORKDIR}"/nginx.tar docker pull "${DOCKER_REGISTRY}/antrea/agnhost:2.26" docker tag "${DOCKER_REGISTRY}/antrea/agnhost:2.26" "agnhost:2.26" @@ -284,7 +291,12 @@ function run_multicluster_e2e { set +e mkdir -p `pwd`/antrea-multicluster-test-logs - go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs + if [[ ${ENABLE_MC_GATEWAY} ]];then + go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs --mc-gateway + else + go test -v antrea.io/antrea/multicluster/test/e2e --logs-export-dir `pwd`/antrea-multicluster-test-logs + fi + if [[ "$?" != "0" ]]; then TEST_FAILURE=true fi diff --git a/cmd/antrea-agent-simulator/simulator.go b/cmd/antrea-agent-simulator/simulator.go index bcc4835078d..ca17da3ff67 100644 --- a/cmd/antrea-agent-simulator/simulator.go +++ b/cmd/antrea-agent-simulator/simulator.go @@ -38,7 +38,7 @@ import ( func run() error { klog.Infof("Starting Antrea agent simulator (version %s)", version.GetFullVersion()) - k8sClient, _, _, _, err := k8s.CreateClients(componentbaseconfig.ClientConnectionConfiguration{}, "") + k8sClient, _, _, _, _, err := k8s.CreateClients(componentbaseconfig.ClientConnectionConfiguration{}, "") if err != nil { return fmt.Errorf("error creating K8s clients: %v", err) } diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 00a162044de..7da1d27548a 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -29,6 +29,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/agent" "antrea.io/antrea/pkg/agent/apiserver" "antrea.io/antrea/pkg/agent/cniserver" @@ -46,6 +47,7 @@ import ( "antrea.io/antrea/pkg/agent/memberlist" "antrea.io/antrea/pkg/agent/metrics" "antrea.io/antrea/pkg/agent/multicast" + mcroute "antrea.io/antrea/pkg/agent/multicluster" npl "antrea.io/antrea/pkg/agent/nodeportlocal" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/proxy" @@ -65,6 +67,7 @@ import ( "antrea.io/antrea/pkg/signals" "antrea.io/antrea/pkg/util/channel" "antrea.io/antrea/pkg/util/cipher" + "antrea.io/antrea/pkg/util/env" "antrea.io/antrea/pkg/util/k8s" "antrea.io/antrea/pkg/version" ) @@ -85,8 +88,8 @@ var excludeNodePortDevices = []string{"antrea-egress0", "antrea-ingress0", "kube func run(o *Options) error { klog.Infof("Starting Antrea agent (version %s)", version.GetFullVersion()) - // Create K8s Clientset, CRD Clientset and SharedInformerFactory for the given config. - k8sClient, _, crdClient, _, err := k8s.CreateClients(o.config.ClientConnection, o.config.KubeAPIServerOverride) + // Create K8s Clientset, CRD Clientset, Multicluster CRD Clientset and SharedInformerFactory for the given config. + k8sClient, _, crdClient, _, mcClient, err := k8s.CreateClients(o.config.ClientConnection, o.config.KubeAPIServerOverride) if err != nil { return fmt.Errorf("error creating K8s clients: %v", err) } @@ -135,6 +138,7 @@ func run(o *Options) error { connectUplinkToBridge, features.DefaultFeatureGate.Enabled(features.Multicast), features.DefaultFeatureGate.Enabled(features.TrafficControl), + features.DefaultFeatureGate.Enabled(features.Multicluster), ) _, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR) @@ -254,6 +258,28 @@ func run(o *Options) error { ipsecCertController, ) + var mcRouteController *mcroute.MCRouteController + var mcInformerFactory mcinformers.SharedInformerFactory + + if features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.Enable { + mcNamespace := env.GetPodNamespace() + if o.config.Multicluster.Namespace != "" { + mcNamespace = o.config.Multicluster.Namespace + } + mcInformerFactory = mcinformers.NewSharedInformerFactory(mcClient, informerDefaultResync) + gwInformer := mcInformerFactory.Multicluster().V1alpha1().Gateways() + ciImportInformer := mcInformerFactory.Multicluster().V1alpha1().ClusterInfoImports() + mcRouteController = mcroute.NewMCRouteController( + mcClient, + gwInformer, + ciImportInformer, + ofClient, + ovsBridgeClient, + ifaceStore, + nodeConfig, + mcNamespace, + ) + } var groupCounters []proxytypes.GroupCounter groupIDUpdates := make(chan string, 100) v4GroupIDAllocator := openflow.NewGroupAllocator(false) @@ -618,6 +644,11 @@ func run(o *Options) error { go mcastController.Run(stopCh) } + if features.DefaultFeatureGate.Enabled(features.Multicluster) && o.config.Multicluster.Enable { + mcInformerFactory.Start(stopCh) + go mcRouteController.Run(stopCh) + } + agentQuerier := querier.NewAgentQuerier( nodeConfig, networkConfig, diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index a04fae4166a..6dc0b30d394 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -181,6 +181,11 @@ func (o *Options) validate(args []string) error { } } } + if (features.DefaultFeatureGate.Enabled(features.Multicluster) || o.config.Multicluster.Enable) && + encapMode != config.TrafficEncapModeEncap { + // Only Encap mode is supported for Multi-cluster feature. + return fmt.Errorf("Multicluster is only applicable to the %s mode", config.TrafficEncapModeEncap) + } if features.DefaultFeatureGate.Enabled(features.NodePortLocal) { startPort, endPort, err := parsePortRange(o.config.NodePortLocal.PortRange) if err != nil { diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 47c0c553514..c1087c124ff 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -112,7 +112,7 @@ func run(o *Options) error { // Create K8s Clientset, Aggregator Clientset, CRD Clientset and SharedInformerFactory for the given config. // Aggregator Clientset is used to update the CABundle of the APIServices backed by antrea-controller so that // the aggregator can verify its serving certificate. - client, aggregatorClient, crdClient, apiExtensionClient, err := k8s.CreateClients(o.config.ClientConnection, "") + client, aggregatorClient, crdClient, apiExtensionClient, _, err := k8s.CreateClients(o.config.ClientConnection, "") if err != nil { return fmt.Errorf("error creating K8s clients: %v", err) } diff --git a/multicluster/controllers/multicluster/gateway_controller.go b/multicluster/controllers/multicluster/gateway_controller.go index d87f40ed717..247e03de917 100644 --- a/multicluster/controllers/multicluster/gateway_controller.go +++ b/multicluster/controllers/multicluster/gateway_controller.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "reflect" - "sort" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -156,13 +155,16 @@ func (r *GatewayReconciler) getLastCreatedGateway() (*mcsv1alpha1.GatewayInfo, e return nil, nil } - // Sort Gateways by CreationTimestamp, the last created Gateway will be the first element. - sort.Slice(gws.Items, func(i, j int) bool { - return !gws.Items[i].CreationTimestamp.Before(&gws.Items[j].CreationTimestamp) - }) + // Comparing Gateway's CreationTimestamp to get the last created Gateway. + lastCreatedGW := gws.Items[0] + for _, gw := range gws.Items { + if lastCreatedGW.CreationTimestamp.Before(&gw.CreationTimestamp) { + lastCreatedGW = gw + } + } // Make sure we only return the last created Gateway for now. - return &mcsv1alpha1.GatewayInfo{GatewayIP: gws.Items[0].GatewayIP}, nil + return &mcsv1alpha1.GatewayInfo{GatewayIP: lastCreatedGW.GatewayIP}, nil } func (r *GatewayReconciler) updateResourceExport(ctx context.Context, req ctrl.Request, diff --git a/multicluster/controllers/multicluster/gateway_controller_test.go b/multicluster/controllers/multicluster/gateway_controller_test.go index 5e506335329..0cd8a02e357 100644 --- a/multicluster/controllers/multicluster/gateway_controller_test.go +++ b/multicluster/controllers/multicluster/gateway_controller_test.go @@ -20,6 +20,7 @@ import ( "fmt" "reflect" "testing" + "time" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,18 +38,23 @@ var ( serviceCIDR = "10.96.0.0/12" clusterID = "cluster-a" + gw1CreationTime = metav1.NewTime(time.Now()) + gw2CreationTime = metav1.NewTime(time.Now().Add(10 * time.Minute)) + gwNode1 = mcsv1alpha1.Gateway{ ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - Namespace: "default", + Name: "node-1", + Namespace: "default", + CreationTimestamp: gw1CreationTime, }, GatewayIP: "10.10.10.10", InternalIP: "172.11.10.1", } gwNode2 = mcsv1alpha1.Gateway{ ObjectMeta: metav1.ObjectMeta{ - Name: "node-2", - Namespace: "default", + Name: "node-2", + Namespace: "default", + CreationTimestamp: gw2CreationTime, }, GatewayIP: "10.8.8.8", InternalIP: "172.11.10.1", diff --git a/multicluster/test/e2e/antreapolicy_test.go b/multicluster/test/e2e/antreapolicy_test.go index 59ad973a561..49a4a8b392b 100644 --- a/multicluster/test/e2e/antreapolicy_test.go +++ b/multicluster/test/e2e/antreapolicy_test.go @@ -19,8 +19,6 @@ import ( "testing" "time" - log "github.com/sirupsen/logrus" - antreae2e "antrea.io/antrea/test/e2e" "antrea.io/antrea/test/e2e/utils" ) @@ -42,7 +40,7 @@ var ( func failOnError(err error, t *testing.T) { if err != nil { - log.Errorf("%+v", err) + t.Errorf("%+v", err) for _, k8sUtils := range clusterK8sUtilsMap { k8sUtils.Cleanup(perClusterNamespaces) } @@ -93,7 +91,7 @@ func testMCAntreaPolicy(t *testing.T, data *MCTestData) { // for Namespace isolation, strict Namespace isolation is enforced in each of the member clusters. func (data *MCTestData) testAntreaPolicyCopySpanNSIsolation(t *testing.T) { setup := func() { - err := data.deployACNPResourceExport(acnpIsolationResourceExport) + err := data.deployACNPResourceExport(t, acnpIsolationResourceExport) failOnError(err, t) // Sleep 5s to wait resource export/import process to finish resource exchange. time.Sleep(5 * time.Second) @@ -123,9 +121,9 @@ func executeTestsOnAllMemberClusters(t *testing.T, testList []*antreae2e.TestCas setup() time.Sleep(networkPolicyDelay) for _, testCase := range testList { - log.Infof("Running test case %s", testCase.Name) + t.Logf("Running test case %s", testCase.Name) for _, step := range testCase.Steps { - log.Infof("Running step %s of test case %s", step.Name, testCase.Name) + t.Logf("Running step %s of test case %s", step.Name, testCase.Name) reachability := step.Reachability if reachability != nil { for clusterName, k8sUtils := range clusterK8sUtilsMap { @@ -151,11 +149,11 @@ func executeTestsOnAllMemberClusters(t *testing.T, testList []*antreae2e.TestCas teardown() } -func (data *MCTestData) deployACNPResourceExport(reFileName string) error { - log.Infof("Creating ResourceExport %s in the leader cluster", reFileName) +func (data *MCTestData) deployACNPResourceExport(t *testing.T, reFileName string) error { + t.Logf("Creating ResourceExport %s in the leader cluster", reFileName) rc, _, stderr, err := provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl apply -f %s", reFileName)) if err != nil || rc != 0 || stderr != "" { - return fmt.Errorf("error when deploying the ACNP ResourceExport in leader cluster: %v, stderr: %v", err, stderr) + return fmt.Errorf("error when deploying the ACNP ResourceExport in leader cluster: %v, stderr: %s", err, stderr) } return nil } @@ -163,7 +161,7 @@ func (data *MCTestData) deployACNPResourceExport(reFileName string) error { func (data *MCTestData) deleteACNPResourceExport(reFileName string) error { rc, _, stderr, err := provider.RunCommandOnNode(leaderCluster, fmt.Sprintf("kubectl delete -f %s", reFileName)) if err != nil || rc != 0 || stderr != "" { - return fmt.Errorf("error when deleting the ACNP ResourceExport in leader cluster: %v, stderr: %v", err, stderr) + return fmt.Errorf("error when deleting the ACNP ResourceExport in leader cluster: %v, stderr: %s", err, stderr) } return nil } diff --git a/multicluster/test/e2e/fixtures.go b/multicluster/test/e2e/fixtures.go index b3d1ef868af..d27bff026ce 100644 --- a/multicluster/test/e2e/fixtures.go +++ b/multicluster/test/e2e/fixtures.go @@ -65,10 +65,10 @@ func teardownTest(tb testing.TB, data *MCTestData) { } } -func createPodWrapper(tb testing.TB, data *MCTestData, cluster string, namespace string, name string, image string, ctr string, command []string, +func createPodWrapper(tb testing.TB, data *MCTestData, cluster string, namespace string, name string, nodeName string, image string, ctr string, command []string, args []string, env []corev1.EnvVar, ports []corev1.ContainerPort, hostNetwork bool, mutateFunc func(pod *corev1.Pod)) error { - tb.Logf("Creating Pod '%s'", name) - if err := data.createPod(cluster, name, namespace, ctr, image, command, args, env, ports, hostNetwork, mutateFunc); err != nil { + tb.Logf("Creating Pod '%s' in Namespace %s of cluster %s", name, namespace, cluster) + if err := data.createPod(cluster, name, nodeName, namespace, ctr, image, command, args, env, ports, hostNetwork, mutateFunc); err != nil { return err } @@ -80,14 +80,14 @@ func createPodWrapper(tb testing.TB, data *MCTestData, cluster string, namespace } func deletePodWrapper(tb testing.TB, data *MCTestData, clusterName string, namespace string, name string) { - tb.Logf("Deleting Pod '%s'", name) + tb.Logf("Deleting Pod '%s' in Namespace %s of cluster %s", name, namespace, clusterName) if err := data.deletePod(clusterName, namespace, name); err != nil { tb.Logf("Error when deleting Pod: %v", err) } } func deleteServiceWrapper(tb testing.TB, data *MCTestData, clusterName string, namespace string, name string) { - tb.Logf("Deleting Service '%s'", name) + tb.Logf("Deleting Service '%s' in Namespace %s of cluster %s", name, namespace, clusterName) if err := data.deleteService(clusterName, namespace, name); err != nil { tb.Logf("Error when deleting Service: %v", err) } diff --git a/multicluster/test/e2e/framework.go b/multicluster/test/e2e/framework.go index 1fd0a862f9b..bdad7435912 100644 --- a/multicluster/test/e2e/framework.go +++ b/multicluster/test/e2e/framework.go @@ -35,7 +35,7 @@ var ( const ( defaultTimeout = 90 * time.Second - importServiceDelay = 10 * time.Second + importServiceDelay = 2 * time.Second multiClusterTestNamespace string = "antrea-multicluster-test" eastClusterTestService string = "east-nginx" @@ -57,15 +57,18 @@ type TestOptions struct { leaderClusterKubeConfigPath string westClusterKubeConfigPath string eastClusterKubeConfigPath string + enableGateway bool logsExportDir string } var testOptions TestOptions type MCTestData struct { - clusters []string - clusterTestDataMap map[string]antreae2e.TestData - logsDirForTestCase string + clusters []string + clusterTestDataMap map[string]antreae2e.TestData + logsDirForTestCase string + clusterGateways map[string]string + clusterRegularNodes map[string]string } var testData *MCTestData @@ -149,19 +152,10 @@ func (data *MCTestData) deleteTestNamespaces(timeout time.Duration) error { return nil } -func (data *MCTestData) deleteNamespace(clusterName, namespace string, timeout time.Duration) error { - if d, ok := data.clusterTestDataMap[clusterName]; ok { - if err := d.DeleteNamespace(namespace, timeout); err != nil { - return err - } - } - return nil -} - -func (data *MCTestData) createPod(clusterName, name, namespace, ctrName, image string, command []string, +func (data *MCTestData) createPod(clusterName, name, nodeName, namespace, ctrName, image string, command []string, args []string, env []corev1.EnvVar, ports []corev1.ContainerPort, hostNetwork bool, mutateFunc func(pod *corev1.Pod)) error { if d, ok := data.clusterTestDataMap[clusterName]; ok { - return d.CreatePodOnNodeInNamespace(name, namespace, "", ctrName, image, command, args, env, ports, hostNetwork, mutateFunc) + return d.CreatePodOnNodeInNamespace(name, namespace, nodeName, ctrName, image, command, args, env, ports, hostNetwork, mutateFunc) } return fmt.Errorf("clusterName %s not found", clusterName) } diff --git a/multicluster/test/e2e/main_test.go b/multicluster/test/e2e/main_test.go index ad3a13cf0a0..735bcff429c 100644 --- a/multicluster/test/e2e/main_test.go +++ b/multicluster/test/e2e/main_test.go @@ -65,6 +65,7 @@ func testMain(m *testing.M) int { flag.StringVar(&testOptions.leaderClusterKubeConfigPath, "leader-cluster-kubeconfig-path", path.Join(homedir, ".kube", "leader"), "Kubeconfig Path of the leader cluster") flag.StringVar(&testOptions.eastClusterKubeConfigPath, "east-cluster-kubeconfig-path", path.Join(homedir, ".kube", "east"), "Kubeconfig Path of the east cluster") flag.StringVar(&testOptions.westClusterKubeConfigPath, "west-cluster-kubeconfig-path", path.Join(homedir, ".kube", "west"), "Kubeconfig Path of the west cluster") + flag.BoolVar(&testOptions.enableGateway, "mc-gateway", false, "Run tests with Multicluster Gateway") flag.Parse() cleanupLogging := testOptions.setupLogging() @@ -96,13 +97,23 @@ func TestConnectivity(t *testing.T) { } defer teardownTest(t, data) + if testOptions.enableGateway { + initializeGateway(t, data) + defer teardownGateway(t, data) + + // Sleep 5s to wait resource export/import process to finish resource + // exchange, and data path realization. + time.Sleep(5 * time.Second) + } + t.Run("testServiceExport", func(t *testing.T) { testServiceExport(t, data) }) + t.Run("testAntreaPolicy", func(t *testing.T) { + defer tearDownForPolicyTest() initializeForPolicyTest(t, data) testMCAntreaPolicy(t, data) - tearDownForPolicyTest() }) // Wait 5 seconds to let both member and leader controllers clean up all resources, // otherwise, Namespace deletion may stuck into termininating status. diff --git a/multicluster/test/e2e/service_test.go b/multicluster/test/e2e/service_test.go index af2a8a99de7..1a90fd49d95 100644 --- a/multicluster/test/e2e/service_test.go +++ b/multicluster/test/e2e/service_test.go @@ -16,7 +16,9 @@ package e2e import ( "fmt" + "math/rand" "strconv" + "strings" "testing" "time" @@ -39,63 +41,84 @@ func (data *MCTestData) testServiceExport(t *testing.T) { podName := randName("test-nginx-") clientPodName := "test-service-client" - if err := createPodWrapper(t, data, westCluster, multiClusterTestNamespace, podName, nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil { - t.Fatalf("Error when creating nginx Pod in west cluster: %v", err) - } + createPodAndService := func(clusterName, clusterServiceName string) { + if err := createPodWrapper(t, data, clusterName, multiClusterTestNamespace, podName, "", nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil { + t.Fatalf("Error when creating nginx Pod in cluster %s: %v", clusterName, err) + } + if _, err := data.createService(clusterName, clusterServiceName, multiClusterTestNamespace, 80, 80, corev1.ProtocolTCP, map[string]string{"app": "nginx"}, false, + false, corev1.ServiceTypeClusterIP, nil, nil); err != nil { + t.Fatalf("Error when creating Service %s in cluster %s: %v", clusterServiceName, clusterName, err) + } + } + // Create Pod and Service in west cluster + createPodAndService(westCluster, westClusterTestService) defer deletePodWrapper(t, data, westCluster, multiClusterTestNamespace, podName) - - if err := createPodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName, nginxImage, "nginx", nil, nil, nil, nil, false, nil); err != nil { - t.Fatalf("Error when creating nginx Pod in east cluster: %v", err) - } - defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName) - - if _, err := data.createService(westCluster, westClusterTestService, multiClusterTestNamespace, 80, 80, corev1.ProtocolTCP, map[string]string{"app": "nginx"}, false, - false, corev1.ServiceTypeClusterIP, nil, nil); err != nil { - t.Fatalf("Error when creating Servie %s in west cluster: %v", westClusterTestService, err) - } defer deleteServiceWrapper(t, testData, westCluster, multiClusterTestNamespace, westClusterTestService) - if _, err := data.createService(eastCluster, eastClusterTestService, multiClusterTestNamespace, 80, 80, corev1.ProtocolTCP, map[string]string{"app": "nginx"}, false, - false, corev1.ServiceTypeClusterIP, nil, nil); err != nil { - t.Fatalf("Error when creating Servie %s in east cluster: %v", eastClusterTestService, err) - } + // Create Pod and Service in east cluster + createPodAndService(eastCluster, eastClusterTestService) + defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName) defer deleteServiceWrapper(t, testData, eastCluster, multiClusterTestNamespace, eastClusterTestService) - if err := data.deployServiceExport(westCluster); err != nil { - t.Fatalf("Error when deploy ServiceExport in west cluster: %v", err) + deployServiceExport := func(clusterName string) { + if err := data.deployServiceExport(clusterName); err != nil { + t.Fatalf("Error when deploy ServiceExport in cluster %s: %v", clusterName, err) + } } + + // Deploy ServiceExport in west cluster + deployServiceExport(westCluster) defer data.deleteServiceExport(westCluster) - if err := data.deployServiceExport(eastCluster); err != nil { - t.Fatalf("Error when deploy ServiceExport in east cluster: %v", err) - } + + // Deploy ServiceExport in east cluster + deployServiceExport(eastCluster) defer data.deleteServiceExport(eastCluster) time.Sleep(importServiceDelay) - // Create a Pod in east cluster and verify the MC Service connectivity from it. - if err := data.createPod(eastCluster, clientPodName, multiClusterTestNamespace, "client", agnhostImage, - []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { - t.Fatalf("Error when creating client Pod in east cluster: %v", err) - } - defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, clientPodName) - _, err := data.podWaitFor(defaultTimeout, eastCluster, clientPodName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) { - return pod.Status.Phase == corev1.PodRunning, nil - }) - if err != nil { - t.Fatalf("Error when waiting for Pod '%s' in east cluster: %v", clientPodName, err) - } - svc, err := data.getService(eastCluster, multiClusterTestNamespace, fmt.Sprintf("antrea-mc-%s", westClusterTestService)) if err != nil { t.Fatalf("Error when getting the imported service %s: %v", fmt.Sprintf("antrea-mc-%s", westClusterTestService), err) } eastIP := svc.Spec.ClusterIP - if err := data.probeServiceFromPodInCluster(eastCluster, clientPodName, "client", multiClusterTestNamespace, eastIP); err != nil { - t.Fatalf("Error when probing service from %s", eastCluster) + gwClientName := clientPodName + "-gateway" + regularClientName := clientPodName + "-regularnode" + + createEastPod := func(nodeName string, podName string) { + if err := data.createPod(eastCluster, podName, nodeName, multiClusterTestNamespace, "client", agnhostImage, + []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { + t.Fatalf("Error when creating client Pod in east cluster: %v", err) + } + t.Logf("Checking Pod status %s in Namespace %s of cluster %s", podName, multiClusterTestNamespace, eastCluster) + _, err := data.podWaitFor(defaultTimeout, eastCluster, podName, multiClusterTestNamespace, func(pod *corev1.Pod) (bool, error) { + return pod.Status.Phase == corev1.PodRunning, nil + }) + if err != nil { + deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, podName) + t.Fatalf("Error when waiting for Pod '%s' in east cluster: %v", podName, err) + } + } + + // Create a Pod in east cluster's Gateway and verify the MC Service connectivity from it. + createEastPod(data.clusterGateways[eastCluster], gwClientName) + defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, gwClientName) + + t.Logf("Probing Service from client Pod %s in cluster %s", gwClientName, eastCluster) + if err := data.probeServiceFromPodInCluster(eastCluster, gwClientName, "client", multiClusterTestNamespace, eastIP); err != nil { + t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", gwClientName, eastCluster, err) + } + + // Create a Pod in east cluster's regular Node and verify the MC Service connectivity from it. + createEastPod(data.clusterRegularNodes[eastCluster], regularClientName) + defer deletePodWrapper(t, data, eastCluster, multiClusterTestNamespace, regularClientName) + + t.Logf("Probing Service from client Pod %s in cluster %s", regularClientName, eastCluster) + if err := data.probeServiceFromPodInCluster(eastCluster, regularClientName, "client", multiClusterTestNamespace, eastIP); err != nil { + t.Fatalf("Error when probing Service from client Pod %s in cluster %s, err: %v", regularClientName, eastCluster, err) } // Create a Pod in west cluster and verify the MC Service connectivity from it. - if err := data.createPod(westCluster, clientPodName, multiClusterTestNamespace, "client", agnhostImage, + if err := data.createPod(westCluster, clientPodName, "", multiClusterTestNamespace, "client", agnhostImage, []string{"sleep", strconv.Itoa(3600)}, nil, nil, nil, false, nil); err != nil { t.Fatalf("Error when creating client Pod in west cluster: %v", err) } @@ -113,11 +136,11 @@ func (data *MCTestData) testServiceExport(t *testing.T) { } westIP := svc.Spec.ClusterIP if err := data.probeServiceFromPodInCluster(westCluster, clientPodName, "client", multiClusterTestNamespace, westIP); err != nil { - t.Fatalf("Error when probing service from %s", westCluster) + t.Fatalf("Error when probing service from %s, err: %v", westCluster, err) } // Verify that ACNP works fine with new Multicluster Service. - data.verifyMCServiceACNP(t, clientPodName, eastIP) + data.verifyMCServiceACNP(t, gwClientName, eastIP) } func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP string) { @@ -146,7 +169,7 @@ func (data *MCTestData) verifyMCServiceACNP(t *testing.T, clientPodName, eastIP func (data *MCTestData) deployServiceExport(clusterName string) error { rc, _, stderr, err := provider.RunCommandOnNode(clusterName, fmt.Sprintf("kubectl apply -f %s", serviceExportYML)) if err != nil || rc != 0 || stderr != "" { - return fmt.Errorf("error when deploying the ServiceExport: %v, stderr: %v", err, stderr) + return fmt.Errorf("error when deploying the ServiceExport: %v, stderr: %s", err, stderr) } return nil @@ -155,8 +178,77 @@ func (data *MCTestData) deployServiceExport(clusterName string) error { func (data *MCTestData) deleteServiceExport(clusterName string) error { rc, _, stderr, err := provider.RunCommandOnNode(clusterName, fmt.Sprintf("kubectl delete -f %s", serviceExportYML)) if err != nil || rc != 0 || stderr != "" { - return fmt.Errorf("error when deleting the ServiceExport: %v, stderr: %v", err, stderr) + return fmt.Errorf("error when deleting the ServiceExport: %v, stderr: %s", err, stderr) + } + + return nil +} + +// getNodeNamesFromCluster will pick up a Node randomly as the Gateway +// and also a regular Node from the specified cluster. +func getNodeNamesFromCluster(nodeName string) (string, string, error) { + rc, output, stderr, err := provider.RunCommandOnNode(nodeName, "kubectl get node -o custom-columns=:metadata.name --no-headers") + if err != nil || rc != 0 || stderr != "" { + return "", "", fmt.Errorf("error when getting Node list: %v, stderr: %s", err, stderr) + } + nodes := strings.Split(output, "\n") + gwIdx := rand.Intn(len(nodes)) // #nosec G404: for test only + var regularNode string + for i, node := range nodes { + if i != gwIdx { + regularNode = node + break + } + } + return nodes[gwIdx], regularNode, nil +} + +// setGatewayNode adds an annotation to assign it as Gateway Node. +func (data *MCTestData) setGatewayNode(t *testing.T, clusterName string, nodeName string) error { + rc, _, stderr, err := provider.RunCommandOnNode(clusterName, fmt.Sprintf("kubectl annotate node %s multicluster.antrea.io/gateway=true", nodeName)) + if err != nil || rc != 0 || stderr != "" { + return fmt.Errorf("error when annotate the Node %s: %s, stderr: %s", nodeName, err, stderr) } + t.Logf("The Node %s is annotated as Gateway in cluster %s", nodeName, clusterName) + return nil +} +func (data *MCTestData) unsetGatewayNode(clusterName string, nodeName string) error { + rc, _, stderr, err := provider.RunCommandOnNode(clusterName, fmt.Sprintf("kubectl annotate node %s multicluster.antrea.io/gateway-", nodeName)) + if err != nil || rc != 0 || stderr != "" { + return fmt.Errorf("error when cleaning up annotation of the Node: %v, stderr: %s", err, stderr) + } return nil } + +func initializeGateway(t *testing.T, data *MCTestData) { + data.clusterGateways = make(map[string]string) + data.clusterRegularNodes = make(map[string]string) + // Annotates a Node as Gateway, then member controller will create a Gateway correspondingly. + for clusterName := range data.clusterTestDataMap { + if clusterName == leaderCluster { + // Skip Gateway initialization for the leader cluster + continue + } + gwName, regularNode, err := getNodeNamesFromCluster(clusterName) + failOnError(err, t) + err = data.setGatewayNode(t, clusterName, gwName) + failOnError(err, t) + data.clusterGateways[clusterName] = gwName + data.clusterRegularNodes[clusterName] = regularNode + } +} + +func teardownGateway(t *testing.T, data *MCTestData) { + for clusterName := range data.clusterTestDataMap { + if clusterName == leaderCluster { + continue + } + if _, ok := data.clusterGateways[clusterName]; ok { + t.Logf("Removing the Gateway annotation on Node %s in cluster %s", data.clusterGateways[clusterName], clusterName) + if err := data.unsetGatewayNode(clusterName, data.clusterGateways[clusterName]); err != nil { + t.Errorf("Error: %v", err) + } + } + } +} diff --git a/pkg/agent/multicluster/mc_route_controller.go b/pkg/agent/multicluster/mc_route_controller.go new file mode 100644 index 00000000000..ff872d432ea --- /dev/null +++ b/pkg/agent/multicluster/mc_route_controller.go @@ -0,0 +1,429 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package noderoute + +import ( + "errors" + "fmt" + "net" + "strings" + "time" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + mcv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + mcclientset "antrea.io/antrea/multicluster/pkg/client/clientset/versioned" + mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions/multicluster/v1alpha1" + mclisters "antrea.io/antrea/multicluster/pkg/client/listers/multicluster/v1alpha1" + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/interfacestore" + "antrea.io/antrea/pkg/agent/openflow" + "antrea.io/antrea/pkg/ovs/ovsconfig" +) + +const ( + controllerName = "AntreaAgentMCRouteController" + + // Set resyncPeriod to 0 to disable resyncing + resyncPeriod = 0 * time.Second + // How long to wait before retrying the processing of a resource change + minRetryDelay = 2 * time.Second + maxRetryDelay = 120 * time.Second + + // Default number of workers processing a resource change + defaultWorkers = 1 + workerItemKey = "key" +) + +// MCRouteController watches Gateway and ClusterInfoImport events. +// It is responsible for setting up necessary Openflow entries for multi-cluster +// traffic on a Gateway or a regular Node. +type MCRouteController struct { + mcClient mcclientset.Interface + ovsBridgeClient ovsconfig.OVSBridgeClient + ofClient openflow.Client + interfaceStore interfacestore.InterfaceStore + nodeConfig *config.NodeConfig + gwInformer mcinformers.GatewayInformer + gwLister mclisters.GatewayLister + gwListerSynced cache.InformerSynced + ciImportInformer mcinformers.ClusterInfoImportInformer + ciImportLister mclisters.ClusterInfoImportLister + ciImportListerSynced cache.InformerSynced + queue workqueue.RateLimitingInterface + // installedCIImports is for saving ClusterInfos which have been processed + // in MCRouteController. Need to use mutex to protect 'installedCIImports' if + // we change the number of 'defaultWorkers'. + installedCIImports map[string]*mcv1alpha1.ClusterInfoImport + // Need to use mutex to protect 'installedActiveGWName' if we change + // the number of 'defaultWorkers' to run multiple go routines to handle + // events. + installedActiveGWName string + // The Namespace where Antrea Multi-cluster Controller is running. + namespace string +} + +func NewMCRouteController( + mcClient mcclientset.Interface, + gwInformer mcinformers.GatewayInformer, + ciImportInformer mcinformers.ClusterInfoImportInformer, + client openflow.Client, + ovsBridgeClient ovsconfig.OVSBridgeClient, + interfaceStore interfacestore.InterfaceStore, + nodeConfig *config.NodeConfig, + namespace string, +) *MCRouteController { + controller := &MCRouteController{ + mcClient: mcClient, + ovsBridgeClient: ovsBridgeClient, + ofClient: client, + interfaceStore: interfaceStore, + nodeConfig: nodeConfig, + gwInformer: gwInformer, + gwLister: gwInformer.Lister(), + gwListerSynced: gwInformer.Informer().HasSynced, + ciImportInformer: ciImportInformer, + ciImportLister: ciImportInformer.Lister(), + ciImportListerSynced: ciImportInformer.Informer().HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "gatewayroute"), + installedCIImports: make(map[string]*mcv1alpha1.ClusterInfoImport), + namespace: namespace, + } + controller.gwInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(cur interface{}) { + controller.enqueueGateway(cur, false) + }, + UpdateFunc: func(old, cur interface{}) { + controller.enqueueGateway(cur, false) + }, + DeleteFunc: func(old interface{}) { + controller.enqueueGateway(old, true) + }, + }, + resyncPeriod, + ) + controller.ciImportInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(cur interface{}) { + controller.enqueueClusterInfoImport(cur, false) + }, + UpdateFunc: func(old, cur interface{}) { + controller.enqueueClusterInfoImport(cur, false) + }, + DeleteFunc: func(old interface{}) { + controller.enqueueClusterInfoImport(old, true) + }, + }, + resyncPeriod, + ) + return controller +} + +func (c *MCRouteController) enqueueGateway(obj interface{}, isDelete bool) { + gw, isGW := obj.(*mcv1alpha1.Gateway) + if !isGW { + deletedState, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.ErrorS(nil, "Received unexpected object", "object", obj) + return + } + gw, ok = deletedState.Obj.(*mcv1alpha1.Gateway) + if !ok { + klog.ErrorS(nil, "DeletedFinalStateUnknown contains non-Gateway object", "object", deletedState.Obj) + return + } + } + if !isDelete { + if net.ParseIP(gw.InternalIP) == nil || net.ParseIP(gw.GatewayIP) == nil { + klog.ErrorS(nil, "No valid Internal IP or Gateway IP is found in Gateway", "gateway", gw.Namespace+"/"+gw.Name) + return + } + } + c.queue.Add(workerItemKey) +} + +func (c *MCRouteController) enqueueClusterInfoImport(obj interface{}, isDelete bool) { + ciImp, isciImp := obj.(*mcv1alpha1.ClusterInfoImport) + if !isciImp { + deletedState, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.ErrorS(nil, "Received unexpected object", "object", obj) + return + } + ciImp, ok = deletedState.Obj.(*mcv1alpha1.ClusterInfoImport) + if !ok { + klog.ErrorS(nil, "DeletedFinalStateUnknown contains non-ClusterInfoImport object", "object", deletedState.Obj) + return + } + } + + if !isDelete { + if len(ciImp.Spec.GatewayInfos) == 0 { + klog.ErrorS(nil, "Received invalid ClusterInfoImport", "object", obj) + return + } + if net.ParseIP(ciImp.Spec.GatewayInfos[0].GatewayIP) == nil { + klog.ErrorS(nil, "Received ClusterInfoImport with invalid Gateway IP", "object", obj) + return + } + } + + c.queue.Add(workerItemKey) +} + +// Run will create defaultWorkers workers (go routines) which will process +// the Gateway events from the workqueue. +func (c *MCRouteController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + cacheSyncs := []cache.InformerSynced{c.gwListerSynced, c.ciImportListerSynced} + klog.InfoS("Starting controller", "controller", controllerName) + defer klog.InfoS("Shutting down controller", "controller", controllerName) + if !cache.WaitForNamedCacheSync(controllerName, stopCh, cacheSyncs...) { + return + } + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +// worker is a long-running function that will continually call the processNextWorkItem +// function in order to read and process a message on the workqueue. +func (c *MCRouteController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *MCRouteController) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + + if k, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.InfoS("Expected string in work queue but got", "object", obj) + return true + } else if err := c.syncMCFlows(); err == nil { + c.queue.Forget(k) + } else { + // Put the item back on the workqueue to handle any transient errors. + c.queue.AddRateLimited(k) + klog.ErrorS(err, "Error syncing key, requeuing", "key", k) + } + return true +} + +func (c *MCRouteController) syncMCFlows() error { + startTime := time.Now() + defer func() { + klog.V(4).InfoS("Finished syncing flows for Multi-cluster", "time", time.Since(startTime)) + }() + activeGW, err := c.getActiveGateway() + if err != nil { + return err + } + if activeGW == nil && c.installedActiveGWName == "" { + klog.V(2).InfoS("No active Gateway is found") + return nil + } + + klog.V(2).InfoS("Installed Gateway", "gateway", c.installedActiveGWName) + if activeGW != nil && activeGW.Name == c.installedActiveGWName { + // Active Gateway name doesn't change but do a full flows resync + // for any Gateway Spec or ClusterInfoImport changes. + if err := c.syncMCFlowsForAllCIImps(activeGW); err != nil { + return err + } + return nil + } + + if c.installedActiveGWName != "" { + if err := c.deleteMCFlowsForAllCIImps(); err != nil { + return err + } + klog.V(2).InfoS("Deleted flows for installed Gateway", "gateway", c.installedActiveGWName) + c.installedActiveGWName = "" + } + + if activeGW != nil { + if err := c.ofClient.InstallMulticlusterClassifierFlows(config.DefaultTunOFPort, activeGW.Name == c.nodeConfig.Name); err != nil { + return err + } + c.installedActiveGWName = activeGW.Name + return c.addMCFlowsForAllCIImps(activeGW) + } + return nil +} + +func (c *MCRouteController) syncMCFlowsForAllCIImps(activeGW *mcv1alpha1.Gateway) error { + desiredCIImports, err := c.ciImportLister.ClusterInfoImports(c.namespace).List(labels.Everything()) + if err != nil { + return err + } + + installedCIImportNames := sets.StringKeySet(c.installedCIImports) + for idx := range desiredCIImports { + if err = c.addMCFlowsForSingleCIImp(activeGW, desiredCIImports[idx], c.installedCIImports[desiredCIImports[idx].Name]); err != nil { + if strings.Contains(err.Error(), "invalid Gateway IP") { + continue + } + return err + } + installedCIImportNames.Delete(desiredCIImports[idx].Name) + } + + for name := range installedCIImportNames { + if err := c.deleteMCFlowsForSingleCIImp(name); err != nil { + return err + } + } + return nil +} + +func (c *MCRouteController) addMCFlowsForAllCIImps(activeGW *mcv1alpha1.Gateway) error { + allCIImports, err := c.ciImportLister.ClusterInfoImports(c.namespace).List(labels.Everything()) + if err != nil { + return err + } + if len(allCIImports) == 0 { + klog.V(2).InfoS("No remote ClusterInfo imported, do nothing") + return nil + } + + for _, ciImport := range allCIImports { + if err := c.addMCFlowsForSingleCIImp(activeGW, ciImport, nil); err != nil { + if strings.Contains(err.Error(), "invalid Gateway IP") { + continue + } + return err + } + } + + return nil +} + +func (c *MCRouteController) addMCFlowsForSingleCIImp(activeGW *mcv1alpha1.Gateway, ciImport *mcv1alpha1.ClusterInfoImport, installedCIImp *mcv1alpha1.ClusterInfoImport) error { + tunnelPeerIPToRemoteGW := getPeerGatewayIP(ciImport.Spec) + if tunnelPeerIPToRemoteGW == nil { + return errors.New("invalid Gateway IP") + } + + if installedCIImp != nil { + oldTunnelPeerIPToRemoteGW := getPeerGatewayIP(installedCIImp.Spec) + if oldTunnelPeerIPToRemoteGW.Equal(tunnelPeerIPToRemoteGW) && installedCIImp.Spec.ServiceCIDR == ciImport.Spec.ServiceCIDR { + klog.V(2).InfoS("No difference between new and installed ClusterInfoImports, skip updating", "clusterinfoimport", ciImport.Name) + return nil + } + } + + klog.InfoS("Adding/updating remote Gateway Node flows for Multi-cluster", "gateway", klog.KObj(activeGW), + "node", c.nodeConfig.Name, "peer", tunnelPeerIPToRemoteGW) + allCIDRs := []string{ciImport.Spec.ServiceCIDR} + peerConfigs, err := generatePeerConfigs(allCIDRs, tunnelPeerIPToRemoteGW) + if err != nil { + klog.ErrorS(err, "Parse error for serviceCIDR from remote cluster", "clusterinfoimport", ciImport.Name, "gateway", activeGW.Name) + return err + } + if activeGW.Name == c.nodeConfig.Name { + klog.V(2).InfoS("Adding/updating flows to remote Gateway Node for Multi-cluster traffic", "clusterinfoimport", ciImport.Name, "cidrs", allCIDRs) + localGatewayIP := net.ParseIP(activeGW.GatewayIP) + if err := c.ofClient.InstallMulticlusterGatewayFlows( + ciImport.Name, + peerConfigs, + tunnelPeerIPToRemoteGW, + localGatewayIP); err != nil { + return fmt.Errorf("failed to install flows to remote Gateway in ClusterInfoImport %s: %v", ciImport.Name, err) + } + } else { + klog.V(2).InfoS("Adding/updating flows to the local active Gateway for Multi-cluster traffic", "clusterinfoimport", ciImport.Name, "cidrs", allCIDRs) + tunnelPeerIPToLocalGW := net.ParseIP(activeGW.InternalIP) + if err := c.ofClient.InstallMulticlusterNodeFlows( + ciImport.Name, + peerConfigs, + tunnelPeerIPToLocalGW); err != nil { + return fmt.Errorf("failed to install flows to Gateway %s: %v", activeGW.Name, err) + } + } + + c.installedCIImports[ciImport.Name] = ciImport + return nil +} + +func (c *MCRouteController) deleteMCFlowsForSingleCIImp(ciImpName string) error { + if err := c.ofClient.UninstallMulticlusterFlows(ciImpName); err != nil { + return fmt.Errorf("failed to uninstall multi-cluster flows to remote Gateway Node %s: %v", ciImpName, err) + } + delete(c.installedCIImports, ciImpName) + return nil +} + +func (c *MCRouteController) deleteMCFlowsForAllCIImps() error { + for _, ciImp := range c.installedCIImports { + c.deleteMCFlowsForSingleCIImp(ciImp.Name) + } + return nil +} + +// getActiveGateway compares Gateway's CreationTimestamp to get the active Gateway, +// The last created Gateway will be the active Gateway. +func (c *MCRouteController) getActiveGateway() (*mcv1alpha1.Gateway, error) { + gws, err := c.gwLister.Gateways(c.namespace).List(labels.Everything()) + if err != nil { + return nil, err + } + if len(gws) == 0 { + return nil, nil + } + // Comparing Gateway's CreationTimestamp to get the last created Gateway. + lastCreatedGW := gws[0] + for _, gw := range gws { + if lastCreatedGW.CreationTimestamp.Before(&gw.CreationTimestamp) { + lastCreatedGW = gw + } + } + if net.ParseIP(lastCreatedGW.GatewayIP) == nil || net.ParseIP(lastCreatedGW.InternalIP) == nil { + return nil, fmt.Errorf("the last created Gateway %s has no valid GatewayIP or InternalIP", lastCreatedGW.Name) + } + return lastCreatedGW, nil +} + +func generatePeerConfigs(subnets []string, gatewayIP net.IP) (map[*net.IPNet]net.IP, error) { + peerConfigs := make(map[*net.IPNet]net.IP, len(subnets)) + for _, subnet := range subnets { + _, peerCIDR, err := net.ParseCIDR(subnet) + if err != nil { + return nil, err + } + peerConfigs[peerCIDR] = gatewayIP + } + return peerConfigs, nil +} + +// getPeerGatewayIP will always return the first Gateway IP. +func getPeerGatewayIP(spec mcv1alpha1.ClusterInfo) net.IP { + if len(spec.GatewayInfos) == 0 { + return nil + } + return net.ParseIP(spec.GatewayInfos[0].GatewayIP) +} diff --git a/pkg/agent/multicluster/mc_route_controller_test.go b/pkg/agent/multicluster/mc_route_controller_test.go new file mode 100644 index 00000000000..ba0d645faa0 --- /dev/null +++ b/pkg/agent/multicluster/mc_route_controller_test.go @@ -0,0 +1,287 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package noderoute + +import ( + "context" + "net" + "testing" + "time" + + "github.com/golang/mock/gomock" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + mcv1alpha1 "antrea.io/antrea/multicluster/apis/multicluster/v1alpha1" + mcfake "antrea.io/antrea/multicluster/pkg/client/clientset/versioned/fake" + mcinformers "antrea.io/antrea/multicluster/pkg/client/informers/externalversions" + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/interfacestore" + oftest "antrea.io/antrea/pkg/agent/openflow/testing" + ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" +) + +type fakeRouteController struct { + *MCRouteController + mcClient *mcfake.Clientset + informerFactory mcinformers.SharedInformerFactory + ofClient *oftest.MockClient + ovsClient *ovsconfigtest.MockOVSBridgeClient + interfaceStore interfacestore.InterfaceStore +} + +func newMCRouteController(t *testing.T, nodeConfig *config.NodeConfig) (*fakeRouteController, func()) { + mcClient := mcfake.NewSimpleClientset() + mcInformerFactory := mcinformers.NewSharedInformerFactory(mcClient, 60*time.Second) + gwInformer := mcInformerFactory.Multicluster().V1alpha1().Gateways() + ciImpInformer := mcInformerFactory.Multicluster().V1alpha1().ClusterInfoImports() + + ctrl := gomock.NewController(t) + ofClient := oftest.NewMockClient(ctrl) + ovsClient := ovsconfigtest.NewMockOVSBridgeClient(ctrl) + interfaceStore := interfacestore.NewInterfaceStore() + c := NewMCRouteController( + mcClient, + gwInformer, + ciImpInformer, + ofClient, + ovsClient, + interfaceStore, + nodeConfig, + "default", + ) + return &fakeRouteController{ + MCRouteController: c, + mcClient: mcClient, + informerFactory: mcInformerFactory, + ofClient: ofClient, + ovsClient: ovsClient, + interfaceStore: interfaceStore, + }, ctrl.Finish +} + +var ( + gw1CreationTime = metav1.NewTime(time.Now()) + gw2CreationTime = metav1.NewTime(time.Now().Add(10 * time.Minute)) + gateway1 = mcv1alpha1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Namespace: "default", + CreationTimestamp: gw1CreationTime, + }, + GatewayIP: "172.17.0.11", + InternalIP: "192.17.0.11", + } + gateway2 = mcv1alpha1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Namespace: "default", + CreationTimestamp: gw2CreationTime, + }, + GatewayIP: "172.17.0.12", + InternalIP: "192.17.0.12", + } + gw1GatewayIP = net.ParseIP(gateway1.GatewayIP) + gw2InternalIP = net.ParseIP(gateway2.InternalIP) + + clusterInfoImport1 = mcv1alpha1.ClusterInfoImport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-b-default-clusterinfo", + Namespace: "default", + }, + Spec: mcv1alpha1.ClusterInfo{ + ClusterID: "cluster-b", + ServiceCIDR: "10.12.2.0/12", + GatewayInfos: []mcv1alpha1.GatewayInfo{ + { + GatewayIP: "172.18.0.10", + }, + }, + }, + } + + clusterInfoImport2 = mcv1alpha1.ClusterInfoImport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-c-default-clusterinfo", + Namespace: "default", + }, + Spec: mcv1alpha1.ClusterInfo{ + ClusterID: "cluster-c", + ServiceCIDR: "13.13.2.0/12", + GatewayInfos: []mcv1alpha1.GatewayInfo{ + { + GatewayIP: "12.11.0.10", + }, + }, + }, + } +) + +func TestMCRouteControllerAsGateway(t *testing.T) { + c, closeFn := newMCRouteController(t, &config.NodeConfig{Name: "node-1"}) + defer closeFn() + defer c.queue.ShutDown() + + stopCh := make(chan struct{}) + defer close(stopCh) + c.informerFactory.Start(stopCh) + c.informerFactory.WaitForCacheSync(stopCh) + + finishCh := make(chan struct{}) + go func() { + defer close(finishCh) + + // Create Gateway1 + c.mcClient.MulticlusterV1alpha1().Gateways(gateway1.GetNamespace()).Create(context.TODO(), + &gateway1, metav1.CreateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), true).Times(1) + c.processNextWorkItem() + + // Create two ClusterInfoImports + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport1.GetNamespace()). + Create(context.TODO(), &clusterInfoImport1, metav1.CreateOptions{}) + peerNodeIP1 := getPeerGatewayIP(clusterInfoImport1.Spec) + c.ofClient.EXPECT().InstallMulticlusterGatewayFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1, gw1GatewayIP).Times(1) + c.processNextWorkItem() + + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport2.GetNamespace()). + Create(context.TODO(), &clusterInfoImport2, metav1.CreateOptions{}) + peerNodeIP2 := getPeerGatewayIP(clusterInfoImport2.Spec) + c.ofClient.EXPECT().InstallMulticlusterGatewayFlows(clusterInfoImport2.Name, + gomock.Any(), peerNodeIP2, gw1GatewayIP).Times(1) + c.processNextWorkItem() + + // Update a ClusterInfoImport + clusterInfoImport1.Spec.ServiceCIDR = "192.10.1.0/24" + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport1.GetNamespace()). + Update(context.TODO(), &clusterInfoImport1, metav1.UpdateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterGatewayFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1, gw1GatewayIP).Times(1) + c.processNextWorkItem() + + // Delete a ClusterInfoImport + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport2.GetNamespace()).Delete(context.TODO(), + clusterInfoImport2.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport2.Name).Times(1) + c.processNextWorkItem() + + // Create Gateway2 as active Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway2.GetNamespace()).Create(context.TODO(), + &gateway2, metav1.CreateOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), false).Times(1) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport1.Name, gomock.Any(), gw2InternalIP).Times(1) + c.processNextWorkItem() + + // Delete Gateway2, then Gateway1 become active Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway2.GetNamespace()).Delete(context.TODO(), + gateway2.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), true).Times(1) + c.ofClient.EXPECT().InstallMulticlusterGatewayFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1, gw1GatewayIP).Times(1) + c.processNextWorkItem() + + // Delete last Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway1.GetNamespace()).Delete(context.TODO(), + gateway1.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.processNextWorkItem() + }() + select { + case <-time.After(5 * time.Second): + t.Errorf("Test didn't finish in time") + case <-finishCh: + } +} + +func TestMCRouteControllerAsRegularNode(t *testing.T) { + c, closeFn := newMCRouteController(t, &config.NodeConfig{Name: "node-3"}) + defer closeFn() + defer c.queue.ShutDown() + + stopCh := make(chan struct{}) + defer close(stopCh) + c.informerFactory.Start(stopCh) + c.informerFactory.WaitForCacheSync(stopCh) + + finishCh := make(chan struct{}) + go func() { + defer close(finishCh) + peerNodeIP1 := net.ParseIP(gateway1.InternalIP) + peerNodeIP2 := net.ParseIP(gateway2.InternalIP) + + // Create Gateway1 + c.mcClient.MulticlusterV1alpha1().Gateways(gateway1.GetNamespace()).Create(context.TODO(), + &gateway1, metav1.CreateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), false).Times(1) + c.processNextWorkItem() + + // Create two ClusterInfoImports + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport1.GetNamespace()). + Create(context.TODO(), &clusterInfoImport1, metav1.CreateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1).Times(1) + c.processNextWorkItem() + + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport2.GetNamespace()). + Create(context.TODO(), &clusterInfoImport2, metav1.CreateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport2.Name, + gomock.Any(), peerNodeIP1).Times(1) + c.processNextWorkItem() + + // Update a ClusterInfoImport + clusterInfoImport1.Spec.ServiceCIDR = "192.12.1.0/24" + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport1.GetNamespace()). + Update(context.TODO(), &clusterInfoImport1, metav1.UpdateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1).Times(1) + c.processNextWorkItem() + + // Delete a ClusterInfoImport + c.mcClient.MulticlusterV1alpha1().ClusterInfoImports(clusterInfoImport2.GetNamespace()).Delete(context.TODO(), + clusterInfoImport2.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport2.Name).Times(1) + c.processNextWorkItem() + + // Create Gateway2 as the active Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway2.GetNamespace()).Create(context.TODO(), + &gateway2, metav1.CreateOptions{}) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), false).Times(1) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport1.Name, gomock.Any(), peerNodeIP2).Times(1) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.processNextWorkItem() + + // Delete Gateway2, then Gateway1 become active Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway2.GetNamespace()).Delete(context.TODO(), + gateway2.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.ofClient.EXPECT().InstallMulticlusterClassifierFlows(uint32(1), false).Times(1) + c.ofClient.EXPECT().InstallMulticlusterNodeFlows(clusterInfoImport1.Name, + gomock.Any(), peerNodeIP1).Times(1) + c.processNextWorkItem() + + // Delete last Gateway + c.mcClient.MulticlusterV1alpha1().Gateways(gateway1.GetNamespace()).Delete(context.TODO(), + gateway1.Name, metav1.DeleteOptions{}) + c.ofClient.EXPECT().UninstallMulticlusterFlows(clusterInfoImport1.Name).Times(1) + c.processNextWorkItem() + }() + select { + case <-time.After(5 * time.Second): + t.Errorf("Test didn't finish in time") + case <-finishCh: + } +} diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index 869d4d28c57..2ec85a85fae 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -294,6 +294,27 @@ type Client interface { UninstallTrafficControlReturnPortFlow(returnOFPort uint32) error InstallMulticastGroup(ofGroupID binding.GroupIDType, localReceivers []uint32) error + + // InstallMulticlusterNodeFlows installs flows to handle cross-cluster packets between a regular + // Node and a local Gateway. + InstallMulticlusterNodeFlows( + clusterID string, + peerConfigs map[*net.IPNet]net.IP, + tunnelPeerIP net.IP) error + + // InstallMulticlusterGatewayFlows installs flows to handle cross-cluster packets between Gateways. + InstallMulticlusterGatewayFlows( + clusterID string, + peerConfigs map[*net.IPNet]net.IP, + tunnelPeerIP net.IP, + localGatewayIP net.IP) error + + // InstallMulticlusterClassifierFlows installs flows to classify cross-cluster packets. + InstallMulticlusterClassifierFlows(tunnelOFPort uint32, isGateway bool) error + + // UninstallMulticlusterFlows removes cross-cluster flows matching the given clusterID on + // a regular Node or a Gateway. + UninstallMulticlusterFlows(clusterID string) error } // GetFlowTableStatus returns an array of flow table status. @@ -749,6 +770,12 @@ func (c *client) generatePipelines() { c.featureMulticast = newFeatureMulticast(c.cookieAllocator, []binding.Protocol{binding.ProtocolIP}, c.bridge) c.activatedFeatures = append(c.activatedFeatures, c.featureMulticast) } + + if c.enableMulticluster { + c.featureMulticluster = newFeatureMulticluster(c.cookieAllocator, []binding.Protocol{binding.ProtocolIP}) + c.activatedFeatures = append(c.activatedFeatures, c.featureMulticluster) + } + c.featureTraceflow = newFeatureTraceflow() c.activatedFeatures = append(c.activatedFeatures, c.featureTraceflow) @@ -1108,7 +1135,7 @@ func (c *client) SendUDPPacketOut( func (c *client) InstallMulticastInitialFlows(pktInReason uint8) error { flows := c.featureMulticast.igmpPktInFlows(pktInReason) flows = append(flows, c.featureMulticast.externalMulticastReceiverFlow()) - cacheKey := fmt.Sprintf("multicast") + cacheKey := "multicast" c.replayMutex.RLock() defer c.replayMutex.RUnlock() return c.addFlows(c.featureMulticast.cachedFlows, cacheKey, flows) @@ -1189,3 +1216,67 @@ func (c *client) InstallMulticastGroup(groupID binding.GroupIDType, localReceive } return nil } + +// InstallMulticlusterNodeFlows installs flows to handle cross-cluster packets between a regular +// Node and a local Gateway. +func (c *client) InstallMulticlusterNodeFlows(clusterID string, + peerConfigs map[*net.IPNet]net.IP, + tunnelPeerIP net.IP) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + cacheKey := fmt.Sprintf("cluster_%s", clusterID) + var flows []binding.Flow + localGatewayMAC := c.nodeConfig.GatewayConfig.MAC + for peerCIDR, remoteGatewayIP := range peerConfigs { + flows = append(flows, c.featureMulticluster.l3FwdFlowToRemoteViaTun(localGatewayMAC, *peerCIDR, tunnelPeerIP, remoteGatewayIP)...) + } + return c.modifyFlows(c.featureMulticluster.cachedFlows, cacheKey, flows) +} + +// InstallMulticlusterGatewayFlows installs flows to handle cross-cluster packets between Gateways. +func (c *client) InstallMulticlusterGatewayFlows(clusterID string, + peerConfigs map[*net.IPNet]net.IP, + tunnelPeerIP net.IP, + localGatewayIP net.IP, +) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + cacheKey := fmt.Sprintf("cluster_%s", clusterID) + var flows []binding.Flow + localGatewayMAC := c.nodeConfig.GatewayConfig.MAC + for peerCIDR, remoteGatewayIP := range peerConfigs { + flows = append(flows, c.featureMulticluster.l3FwdFlowToRemoteViaTun(localGatewayMAC, *peerCIDR, tunnelPeerIP, remoteGatewayIP)...) + // Add SNAT flows to change cross-cluster packets' source IP to local Gateway IP. + flows = append(flows, c.featureMulticluster.snatConntrackFlows(*peerCIDR, localGatewayIP)...) + } + return c.modifyFlows(c.featureMulticluster.cachedFlows, cacheKey, flows) +} + +// InstallMulticlusterClassifierFlows adds the following flows: +// * One flow in L2ForwardingCalcTable for the global virtual multicluster MAC 'aa:bb:cc:dd:ee:f0' +// to set its target output port as 'antrea-tun0'. This flow will be on both Gateway and regular Node. +// * One flow to match MC virtual MAC 'aa:bb:cc:dd:ee:f0' in ClassifierTable for Gateway only. +// * One flow in L2ForwardingOutTable to allow multicluster hairpin traffic for Gateway only. +func (c *client) InstallMulticlusterClassifierFlows(tunnelOFPort uint32, isGateway bool) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + + flows := []binding.Flow{ + c.featurePodConnectivity.l2ForwardCalcFlow(GlobalVirtualMACForMulticluster, tunnelOFPort), + } + + if isGateway { + flows = append(flows, + c.featureMulticluster.tunnelClassifierFlow(tunnelOFPort), + c.featureMulticluster.outputHairpinTunnelFlow(tunnelOFPort), + ) + } + return c.modifyFlows(c.featureMulticluster.cachedFlows, "multicluster-classifier", flows) +} + +func (c *client) UninstallMulticlusterFlows(clusterID string) error { + c.replayMutex.RLock() + defer c.replayMutex.RUnlock() + cacheKey := fmt.Sprintf("cluster_%s", clusterID) + return c.deleteFlows(c.featureMulticluster.cachedFlows, cacheKey) +} diff --git a/pkg/agent/openflow/client_test.go b/pkg/agent/openflow/client_test.go index 57d9eddb061..d40b91e5fb1 100644 --- a/pkg/agent/openflow/client_test.go +++ b/pkg/agent/openflow/client_test.go @@ -107,7 +107,7 @@ func TestIdempotentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -140,7 +140,7 @@ func TestIdempotentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -186,7 +186,7 @@ func TestFlowInstallationFailed(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -225,7 +225,7 @@ func TestConcurrentFlowInstallation(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() m := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) client := ofClient.(*client) client.cookieAllocator = cookie.NewAllocator(0) client.ofEntryOperations = m @@ -420,7 +420,7 @@ func Test_client_SendTraceflowPacket(t *testing.T) { } func prepareTraceflowFlow(ctrl *gomock.Controller) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false, false) c := ofClient.(*client) c.cookieAllocator = cookie.NewAllocator(0) c.nodeConfig = nodeConfig @@ -446,7 +446,7 @@ func prepareTraceflowFlow(ctrl *gomock.Controller) *client { } func prepareSendTraceflowPacket(ctrl *gomock.Controller, success bool) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false, false) c := ofClient.(*client) c.nodeConfig = nodeConfig m := ovsoftest.NewMockBridge(ctrl) @@ -534,7 +534,7 @@ func Test_client_setBasePacketOutBuilder(t *testing.T) { } func prepareSetBasePacketOutBuilder(ctrl *gomock.Controller, success bool) *client { - ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, true, false, false, false, false, false, false, false) c := ofClient.(*client) m := ovsoftest.NewMockBridge(ctrl) c.bridge = m @@ -545,3 +545,40 @@ func prepareSetBasePacketOutBuilder(ctrl *gomock.Controller, success bool) *clie } return c } + +// TestMulticlusterFlowsInstallation checks that InstallMulticlusterNodeFlows +// and UninstallMulticlusterFlows works as expected. +func TestMulticlusterFlowsInstallation(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + m := oftest.NewMockOFEntryOperations(ctrl) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, true, false, false, false, false, false, false, false, true) + client := ofClient.(*client) + client.cookieAllocator = cookie.NewAllocator(0) + client.ofEntryOperations = m + client.nodeConfig = nodeConfig + client.networkConfig = networkConfig + client.egressConfig = egressConfig + client.serviceConfig = serviceConfig + client.ipProtocols = []binding.Protocol{binding.ProtocolIP} + client.generatePipelines() + + m.EXPECT().AddAll(gomock.Any()).Return(nil).Times(1) + clusterID := "cluster-a" + tunnelPeerIP := net.ParseIP("172.17.0.11") + peerConfigs := make(map[*net.IPNet]net.IP, 1) + _, peerCIDR, _ := net.ParseCIDR("10.16.0.1/18") + peerConfigs[peerCIDR] = net.ParseIP("10.17.0.11") + err := ofClient.InstallMulticlusterNodeFlows(clusterID, peerConfigs, tunnelPeerIP) + require.NoError(t, err) + cacheKey := fmt.Sprintf("cluster_%s", clusterID) + fCacheI, ok := client.featureMulticluster.cachedFlows.Load(cacheKey) + require.True(t, ok) + require.Len(t, fCacheI.(flowCache), 2) + + m.EXPECT().DeleteAll(gomock.Any()).Return(nil).Times(1) + err = ofClient.UninstallMulticlusterFlows(clusterID) + require.NoError(t, err) + _, ok = client.featureMulticluster.cachedFlows.Load(cacheKey) + require.False(t, ok) +} diff --git a/pkg/agent/openflow/cookie/allocator.go b/pkg/agent/openflow/cookie/allocator.go index 72df44b35d8..70d363854a7 100644 --- a/pkg/agent/openflow/cookie/allocator.go +++ b/pkg/agent/openflow/cookie/allocator.go @@ -36,6 +36,7 @@ const ( Service Egress Multicast + Multicluster Traceflow ) @@ -53,6 +54,8 @@ func (c Category) String() string { return "Egress" case Multicast: return "Multicast" + case Multicluster: + return "Multicluster" case Traceflow: return "Traceflow" default: diff --git a/pkg/agent/openflow/framework.go b/pkg/agent/openflow/framework.go index 08c92b4be56..d0c284f446d 100644 --- a/pkg/agent/openflow/framework.go +++ b/pkg/agent/openflow/framework.go @@ -225,7 +225,7 @@ func (f *featureService) getRequiredTables() []*Table { ServiceLBTable, EndpointDNATTable, L3ForwardingTable, - ServiceMarkTable, + SNATMarkTable, SNATTable, ConntrackCommitTable, L2ForwardingOutTable, @@ -250,6 +250,18 @@ func (f *featureMulticast) getRequiredTables() []*Table { } } +func (f *featureMulticluster) getRequiredTables() []*Table { + return []*Table{ + ClassifierTable, + ConntrackTable, + L3ForwardingTable, + SNATTable, + UnSNATTable, + SNATMarkTable, + L2ForwardingOutTable, + } +} + func (f *featureTraceflow) getRequiredTables() []*Table { return nil } diff --git a/pkg/agent/openflow/multicluster.go b/pkg/agent/openflow/multicluster.go new file mode 100644 index 00000000000..71a50ee56c8 --- /dev/null +++ b/pkg/agent/openflow/multicluster.go @@ -0,0 +1,163 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openflow + +import ( + "net" + + "antrea.io/antrea/pkg/agent/openflow/cookie" + binding "antrea.io/antrea/pkg/ovs/openflow" +) + +// GlobalVirtualMACForMulticluster is a vritual MAC which will be used only +// for cross-cluster traffic to distinguish from in-cluster traffic. +var GlobalVirtualMACForMulticluster, _ = net.ParseMAC("aa:bb:cc:dd:ee:f0") + +type featureMulticluster struct { + cookieAllocator cookie.Allocator + cachedFlows *flowCategoryCache + category cookie.Category + ipProtocols []binding.Protocol + dnatCtZones map[binding.Protocol]int + snatCtZones map[binding.Protocol]int +} + +func (f *featureMulticluster) getFeatureName() string { + return "Multicluster" +} + +func newFeatureMulticluster(cookieAllocator cookie.Allocator, ipProtocols []binding.Protocol) *featureMulticluster { + snatCtZones := make(map[binding.Protocol]int) + dnatCtZones := make(map[binding.Protocol]int) + snatCtZones[ipProtocols[0]] = SNATCtZone + dnatCtZones[ipProtocols[0]] = CtZone + return &featureMulticluster{ + cookieAllocator: cookieAllocator, + cachedFlows: newFlowCategoryCache(), + category: cookie.Multicluster, + ipProtocols: ipProtocols, + snatCtZones: snatCtZones, + dnatCtZones: dnatCtZones, + } +} + +func (f *featureMulticluster) initFlows() []binding.Flow { + return []binding.Flow{} +} + +func (f *featureMulticluster) replayFlows() []binding.Flow { + return getCachedFlows(f.cachedFlows) +} + +func (f *featureMulticluster) l3FwdFlowToRemoteViaTun( + localGatewayMAC net.HardwareAddr, + peerServiceCIDR net.IPNet, + tunnelPeer net.IP, + remoteGatewayIP net.IP) []binding.Flow { + ipProtocol := getIPProtocol(peerServiceCIDR.IP) + cookieID := f.cookieAllocator.Request(f.category).Raw() + var flows []binding.Flow + flows = append(flows, + // This generates the flow to forward cross-cluster request packets based + // on Service ClusterIP range. + L3ForwardingTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchDstIPNet(peerServiceCIDR). + Action().SetSrcMAC(localGatewayMAC). // Rewrite src MAC to local gateway MAC. + Action().SetDstMAC(GlobalVirtualMACForMulticluster). // Rewrite dst MAC to virtual MC MAC. + Action().SetTunnelDst(tunnelPeer). // Flow based tunnel. Set tunnel destination. + Action().LoadRegMark(ToTunnelRegMark). + Action().GotoTable(L3DecTTLTable.GetID()). + Done(), + // This generates the flow to forward cross-cluster reply traffic based + // on Gateway IP. + L3ForwardingTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateRpl(true). + MatchCTStateTrk(true). + MatchDstIP(remoteGatewayIP). + Action().SetSrcMAC(localGatewayMAC). + Action().SetDstMAC(GlobalVirtualMACForMulticluster). + Action().SetTunnelDst(tunnelPeer). // Flow based tunnel. Set tunnel destination. + Action().LoadRegMark(ToTunnelRegMark). + Action().GotoTable(L3DecTTLTable.GetID()). + Done(), + ) + return flows +} + +func (f *featureMulticluster) tunnelClassifierFlow(tunnelOFPort uint32) binding.Flow { + return ClassifierTable.ofTable.BuildFlow(priorityHigh). + Cookie(f.cookieAllocator.Request(f.category).Raw()). + MatchInPort(tunnelOFPort). + MatchDstMAC(GlobalVirtualMACForMulticluster). + Action().LoadRegMark(FromTunnelRegMark). + Action().LoadRegMark(RewriteMACRegMark). + Action().GotoStage(stageConntrackState). + Done() +} + +func (f *featureMulticluster) outputHairpinTunnelFlow(tunnelOFPort uint32) binding.Flow { + return L2ForwardingOutTable.ofTable.BuildFlow(priorityHigh). + Cookie(f.cookieAllocator.Request(f.category).Raw()). + MatchRegFieldWithValue(TargetOFPortField, tunnelOFPort). + MatchInPort(tunnelOFPort). + Action().OutputInPort(). + Done() +} + +// snatConntrackFlows generates flows on a multi-cluster Gateway Node to perform SNAT for cross-cluster connections. +func (f *featureMulticluster) snatConntrackFlows(serviceCIDR net.IPNet, localGatewayIP net.IP) []binding.Flow { + var flows []binding.Flow + ipProtocol := getIPProtocol(localGatewayIP) + cookieID := f.cookieAllocator.Request(f.category).Raw() + flows = append(flows, + // This generates the flow to match the first packet of multicluster Service connection, and commit them into + // DNAT zone to make sure DNAT is performed before SNAT for any remote cluster traffic. + SNATMarkTable.ofTable.BuildFlow(priorityHigh). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchDstIPNet(serviceCIDR). + MatchCTStateNew(true). + MatchCTStateTrk(true). + Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], nil). + LoadToCtMark(ConnSNATCTMark). + CTDone(). + Done(), + // This generates the flow to perform SNAT for the cross-cluster Service connections. + SNATTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchCTStateNew(true). + MatchCTStateTrk(true). + MatchDstIPNet(serviceCIDR). + Action().CT(true, SNATTable.GetNext(), f.snatCtZones[ipProtocol], nil). + SNAT(&binding.IPRange{StartIP: localGatewayIP, EndIP: localGatewayIP}, nil). + CTDone(). + Done(), + // This generates the flow to unSNAT reply packets of connections committed in SNAT CT zone by the above flows. + UnSNATTable.ofTable.BuildFlow(priorityNormal). + Cookie(cookieID). + MatchProtocol(ipProtocol). + MatchDstIP(localGatewayIP). + Action().CT(false, UnSNATTable.GetNext(), f.snatCtZones[ipProtocol], nil). + NAT(). + CTDone(). + Done(), + ) + return flows +} diff --git a/pkg/agent/openflow/network_policy_test.go b/pkg/agent/openflow/network_policy_test.go index d520689a518..2f4486cae0b 100644 --- a/pkg/agent/openflow/network_policy_test.go +++ b/pkg/agent/openflow/network_policy_test.go @@ -524,7 +524,7 @@ func TestBatchInstallPolicyRuleFlows(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() mockOperations := oftest.NewMockOFEntryOperations(ctrl) - ofClient := NewClient(bridgeName, bridgeMgmtAddr, false, true, false, false, false, false, false, false) + ofClient := NewClient(bridgeName, bridgeMgmtAddr, false, true, false, false, false, false, false, false, false) c = ofClient.(*client) c.cookieAllocator = cookie.NewAllocator(0) c.ofEntryOperations = mockOperations diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index da5031f01d4..1efcca30339 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -155,8 +155,8 @@ var ( L3DecTTLTable = newTable("L3DecTTL", stageRouting, pipelineIP) // Tables in stagePostRouting: - ServiceMarkTable = newTable("ServiceMark", stagePostRouting, pipelineIP) - SNATTable = newTable("SNAT", stagePostRouting, pipelineIP) + SNATMarkTable = newTable("SNATMark", stagePostRouting, pipelineIP) + SNATTable = newTable("SNAT", stagePostRouting, pipelineIP) // Tables in stageSwitching: L2ForwardingCalcTable = newTable("L2ForwardingCalc", stageSwitching, pipelineIP) @@ -385,6 +385,7 @@ type client struct { enableEgress bool enableMulticast bool enableTrafficControl bool + enableMulticluster bool connectUplinkToBridge bool roundInfo types.RoundInfo cookieAllocator cookie.Allocator @@ -395,6 +396,7 @@ type client struct { featureEgress *featureEgress featureNetworkPolicy *featureNetworkPolicy featureMulticast *featureMulticast + featureMulticluster *featureMulticluster activatedFeatures []feature featureTraceflow *featureTraceflow @@ -679,6 +681,7 @@ func (f *featurePodConnectivity) conntrackFlows() []binding.Flow { MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). + MatchCTStateSNAT(false). MatchCTMark(NotServiceCTMark). Action().CT(true, ConntrackCommitTable.GetNext(), f.ctZones[ipProtocol], f.ctZoneSrcField). MoveToCtMarkField(PktSourceField, ConnSourceCTMarkField). @@ -722,10 +725,10 @@ func (f *featureService) snatConntrackFlows() []binding.Flow { cookieID := f.cookieAllocator.Request(f.category).Raw() var flows []binding.Flow for _, ipProtocol := range f.ipProtocols { - gatewayIP, _ := f.gatewayIPs[ipProtocol] + gatewayIP := f.gatewayIPs[ipProtocol] // virtualIP is used as SNAT IP when a request's source IP is gateway IP and we need to forward it back to // gateway interface to avoid asymmetry path. - virtualIP, _ := f.virtualIPs[ipProtocol] + virtualIP := f.virtualIPs[ipProtocol] flows = append(flows, // SNAT should be performed for the following connections: // - Hairpin Service connection initiated through a local Pod, and SNAT should be performed with the Antrea @@ -2650,7 +2653,8 @@ func NewClient(bridgeName string, proxyAll bool, connectUplinkToBridge bool, enableMulticast bool, - enableTrafficControl bool) Client { + enableTrafficControl bool, + enableMulticluster bool) Client { bridge := binding.NewOFBridge(bridgeName, mgmtAddr) c := &client{ bridge: bridge, @@ -2661,6 +2665,7 @@ func NewClient(bridgeName string, enableEgress: enableEgress, enableMulticast: enableMulticast, enableTrafficControl: enableTrafficControl, + enableMulticluster: enableMulticluster, connectUplinkToBridge: connectUplinkToBridge, pipelines: make(map[binding.PipelineID]binding.Pipeline), packetInHandlers: map[uint8]map[string]PacketInHandler{}, @@ -2866,14 +2871,14 @@ func (f *featureService) l3FwdFlowsToExternalEndpoint() []binding.Flow { // ConnSNATCTMark and HairpinCTMark will be loaded in DNAT CT zone. func (f *featureService) podHairpinSNATFlow(endpoint net.IP) binding.Flow { ipProtocol := getIPProtocol(endpoint) - return ServiceMarkTable.ofTable.BuildFlow(priorityLow). + return SNATMarkTable.ofTable.BuildFlow(priorityLow). Cookie(f.cookieAllocator.Request(f.category).Raw()). MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). MatchSrcIP(endpoint). MatchDstIP(endpoint). - Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). + Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). LoadToCtMark(ConnSNATCTMark, HairpinCTMark). CTDone(). Done() @@ -2887,13 +2892,13 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow { for _, ipProtocol := range f.ipProtocols { // This generates the flow to match the first packet of hairpin connection initiated through the Antrea gateway. // ConnSNATCTMark and HairpinCTMark will be loaded in DNAT CT zone. - flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityNormal). + flows = append(flows, SNATMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). MatchRegMark(FromGatewayRegMark, ToGatewayRegMark). - Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). + Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). LoadToCtMark(ConnSNATCTMark, HairpinCTMark). CTDone(). Done()) @@ -2909,13 +2914,13 @@ func (f *featureService) gatewaySNATFlows() []binding.Flow { // This generates the flow to match the first packet of NodePort / LoadBalancer connection initiated through the // Antrea gateway and externalTrafficPolicy of the Service is Cluster, and the selected Endpoint is on a remote // Node, then ConnSNATCTMark will be loaded in DNAT CT zone, indicating that SNAT is required for the connection. - flows = append(flows, ServiceMarkTable.ofTable.BuildFlow(priorityNormal). + flows = append(flows, SNATMarkTable.ofTable.BuildFlow(priorityNormal). Cookie(cookieID). MatchProtocol(ipProtocol). MatchCTStateNew(true). MatchCTStateTrk(true). MatchRegMark(FromGatewayRegMark, pktDstRegMark, ToClusterServiceRegMark). - Action().CT(true, ServiceMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). + Action().CT(true, SNATMarkTable.GetNext(), f.dnatCtZones[ipProtocol], f.ctZoneSrcField). LoadToCtMark(ConnSNATCTMark). CTDone(). Done()) diff --git a/pkg/agent/openflow/pipeline_test.go b/pkg/agent/openflow/pipeline_test.go index 5ce9894d517..9eeb08415e1 100644 --- a/pkg/agent/openflow/pipeline_test.go +++ b/pkg/agent/openflow/pipeline_test.go @@ -73,7 +73,7 @@ func TestBuildPipeline(t *testing.T) { L3ForwardingTable, EgressMarkTable, L3DecTTLTable, - ServiceMarkTable, + SNATMarkTable, SNATTable, L2ForwardingCalcTable, AntreaPolicyIngressRuleTable, @@ -120,7 +120,7 @@ func TestBuildPipeline(t *testing.T) { L3ForwardingTable, EgressMarkTable, L3DecTTLTable, - ServiceMarkTable, + SNATMarkTable, SNATTable, L2ForwardingCalcTable, AntreaPolicyIngressRuleTable, @@ -199,7 +199,7 @@ func TestBuildPipeline(t *testing.T) { L3ForwardingTable, EgressMarkTable, L3DecTTLTable, - ServiceMarkTable, + SNATMarkTable, SNATTable, L2ForwardingCalcTable, AntreaPolicyIngressRuleTable, diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 38feb97b970..604753b0e6d 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -324,6 +324,48 @@ func (mr *MockClientMockRecorder) InstallMulticastInitialFlows(arg0 interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallMulticastInitialFlows", reflect.TypeOf((*MockClient)(nil).InstallMulticastInitialFlows), arg0) } +// InstallMulticlusterClassifierFlows mocks base method +func (m *MockClient) InstallMulticlusterClassifierFlows(arg0 uint32, arg1 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallMulticlusterClassifierFlows", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallMulticlusterClassifierFlows indicates an expected call of InstallMulticlusterClassifierFlows +func (mr *MockClientMockRecorder) InstallMulticlusterClassifierFlows(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallMulticlusterClassifierFlows", reflect.TypeOf((*MockClient)(nil).InstallMulticlusterClassifierFlows), arg0, arg1) +} + +// InstallMulticlusterGatewayFlows mocks base method +func (m *MockClient) InstallMulticlusterGatewayFlows(arg0 string, arg1 map[*net.IPNet]net.IP, arg2, arg3 net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallMulticlusterGatewayFlows", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallMulticlusterGatewayFlows indicates an expected call of InstallMulticlusterGatewayFlows +func (mr *MockClientMockRecorder) InstallMulticlusterGatewayFlows(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallMulticlusterGatewayFlows", reflect.TypeOf((*MockClient)(nil).InstallMulticlusterGatewayFlows), arg0, arg1, arg2, arg3) +} + +// InstallMulticlusterNodeFlows mocks base method +func (m *MockClient) InstallMulticlusterNodeFlows(arg0 string, arg1 map[*net.IPNet]net.IP, arg2 net.IP) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InstallMulticlusterNodeFlows", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// InstallMulticlusterNodeFlows indicates an expected call of InstallMulticlusterNodeFlows +func (mr *MockClientMockRecorder) InstallMulticlusterNodeFlows(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstallMulticlusterNodeFlows", reflect.TypeOf((*MockClient)(nil).InstallMulticlusterNodeFlows), arg0, arg1, arg2) +} + // InstallNodeFlows mocks base method func (m *MockClient) InstallNodeFlows(arg0 string, arg1 map[*net.IPNet]net.IP, arg2 *ip.DualStackIPs, arg3 uint32, arg4 net.HardwareAddr) error { m.ctrl.T.Helper() @@ -682,6 +724,20 @@ func (mr *MockClientMockRecorder) UninstallMulticastFlows(arg0 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallMulticastFlows", reflect.TypeOf((*MockClient)(nil).UninstallMulticastFlows), arg0) } +// UninstallMulticlusterFlows mocks base method +func (m *MockClient) UninstallMulticlusterFlows(arg0 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UninstallMulticlusterFlows", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// UninstallMulticlusterFlows indicates an expected call of UninstallMulticlusterFlows +func (mr *MockClientMockRecorder) UninstallMulticlusterFlows(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UninstallMulticlusterFlows", reflect.TypeOf((*MockClient)(nil).UninstallMulticlusterFlows), arg0) +} + // UninstallNodeFlows mocks base method func (m *MockClient) UninstallNodeFlows(arg0 string) error { m.ctrl.T.Helper() diff --git a/pkg/apiserver/handlers/featuregates/handler.go b/pkg/apiserver/handlers/featuregates/handler.go index 4fb0bfd2391..b091b4238f6 100644 --- a/pkg/apiserver/handlers/featuregates/handler.go +++ b/pkg/apiserver/handlers/featuregates/handler.go @@ -30,7 +30,8 @@ import ( ) var controllerGates = sets.NewString("Traceflow", "AntreaPolicy", "Egress", "NetworkPolicyStats", "NodeIPAM", "ServiceExternalIP") -var agentGates = sets.NewString("AntreaPolicy", "AntreaProxy", "Egress", "EndpointSlice", "Traceflow", "FlowExporter", "NetworkPolicyStats", "NodePortLocal", "AntreaIPAM", "Multicast", "ServiceExternalIP") +var agentGates = sets.NewString("AntreaPolicy", "AntreaProxy", "Egress", "EndpointSlice", "Traceflow", "FlowExporter", "NetworkPolicyStats", + "NodePortLocal", "AntreaIPAM", "Multicast", "ServiceExternalIP", "Multicluster") type ( Config struct { diff --git a/pkg/apiserver/handlers/featuregates/handler_test.go b/pkg/apiserver/handlers/featuregates/handler_test.go index 05bb8f1d28f..7670579e09b 100644 --- a/pkg/apiserver/handlers/featuregates/handler_test.go +++ b/pkg/apiserver/handlers/featuregates/handler_test.go @@ -62,6 +62,7 @@ func Test_getGatesResponse(t *testing.T) { {Component: "agent", Name: "NodePortLocal", Status: nplStatus, Version: "BETA"}, {Component: "agent", Name: "Multicast", Status: "Disabled", Version: "ALPHA"}, {Component: "agent", Name: "ServiceExternalIP", Status: "Disabled", Version: "ALPHA"}, + {Component: "agent", Name: "Multicluster", Status: "Disabled", Version: "ALPHA"}, }, }, } diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index 4604434c930..18c8e0f50bb 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -194,6 +194,8 @@ type AgentConfig struct { Egress EgressConfig `yaml:"egress"` // IPsec related configurations. IPsec IPsecConfig `yaml:"ipsec"` + // Multicluster configuration options. + Multicluster MulticlusterConfig `yaml:"multicluster,omitempty"` } type AntreaProxyConfig struct { @@ -255,3 +257,12 @@ type IPsecConfig struct { // - cert: Use CA-signed certificates for IKE authentication. AuthenticationMode string `yaml:"authenticationMode,omitempty"` } + +type MulticlusterConfig struct { + // Enable Multicluster which allow cross-cluster traffic between member clusters + // in a ClusterSet. + Enable bool `yaml:"enable,omitempty"` + // The Namespace where the Antrea Multi-cluster controller is running. + // The default is antrea-agent's Namespace. + Namespace string `yaml:"namespace,omitempty"` +} diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 9cfa5264234..0355d10276e 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -84,6 +84,10 @@ const ( // Enable Multicast. Multicast featuregate.Feature = "Multicast" + // alpha: v1.7 + // Enable Multicluster. + Multicluster featuregate.Feature = "Multicluster" + // alpha: v1.5 // Enable Secondary interface feature for Antrea. SecondaryNetwork featuregate.Feature = "SecondaryNetwork" @@ -124,6 +128,7 @@ var ( NodePortLocal: {Default: true, PreRelease: featuregate.Beta}, NodeIPAM: {Default: false, PreRelease: featuregate.Alpha}, Multicast: {Default: false, PreRelease: featuregate.Alpha}, + Multicluster: {Default: false, PreRelease: featuregate.Alpha}, SecondaryNetwork: {Default: false, PreRelease: featuregate.Alpha}, ServiceExternalIP: {Default: false, PreRelease: featuregate.Alpha}, TrafficControl: {Default: false, PreRelease: featuregate.Alpha}, @@ -147,6 +152,9 @@ var ( SecondaryNetwork: {}, ServiceExternalIP: {}, IPsecCertAuth: {}, + // Multicluster feature is not validated on Windows yet. This can removed + // in the future if it's fully tested on Windows. + Multicluster: {}, } ) diff --git a/pkg/ovs/openflow/ofctrl_action.go b/pkg/ovs/openflow/ofctrl_action.go index 889796fae21..e4ec8cc7769 100644 --- a/pkg/ovs/openflow/ofctrl_action.go +++ b/pkg/ovs/openflow/ofctrl_action.go @@ -270,14 +270,14 @@ func (a *ofFlowAction) SetVLAN(vlanID uint16) FlowBuilder { return a.builder } -// LoadARPOperation is an action to Load data to NXM_OF_ARP_OP field. +// LoadARPOperation is an action to load data to NXM_OF_ARP_OP field. func (a *ofFlowAction) LoadARPOperation(value uint16) FlowBuilder { loadAct, _ := ofctrl.NewNXLoadAction(NxmFieldARPOp, uint64(value), openflow13.NewNXRange(0, 15)) a.builder.ApplyAction(loadAct) return a.builder } -// LoadRange is an action to Load data to the target field at specified range. +// LoadRange is an action to load data to the target field at specified range. func (a *ofFlowAction) LoadRange(name string, value uint64, rng *Range) FlowBuilder { loadAct, _ := ofctrl.NewNXLoadAction(name, value, rng.ToNXRange()) if a.builder.ofFlow.Table != nil && a.builder.ofFlow.Table.Switch != nil { diff --git a/pkg/ovs/openflow/ofctrl_group.go b/pkg/ovs/openflow/ofctrl_group.go index 5b78a639d81..ee37a35adc5 100644 --- a/pkg/ovs/openflow/ofctrl_group.go +++ b/pkg/ovs/openflow/ofctrl_group.go @@ -103,7 +103,7 @@ func (b *bucketBuilder) LoadXXReg(regID int, data []byte) BucketBuilder { return b } -// LoadRegRange is an action to Load data to the target register at specified range. +// LoadRegRange is an action to load data to the target register at specified range. func (b *bucketBuilder) LoadRegRange(regID int, data uint32, rng *Range) BucketBuilder { reg := fmt.Sprintf("%s%d", NxmFieldReg, regID) regField, _ := openflow13.FindFieldHeaderByName(reg, true) diff --git a/pkg/util/k8s/client.go b/pkg/util/k8s/client.go index 3c2e53674c3..b9af5bfe0d8 100644 --- a/pkg/util/k8s/client.go +++ b/pkg/util/k8s/client.go @@ -15,6 +15,7 @@ package k8s import ( + netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" apiextensionclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -23,39 +24,44 @@ import ( "k8s.io/klog/v2" aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + mcclientset "antrea.io/antrea/multicluster/pkg/client/clientset/versioned" crdclientset "antrea.io/antrea/pkg/client/clientset/versioned" - - netdefclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" ) // CreateClients creates kube clients from the given config. func CreateClients(config componentbaseconfig.ClientConnectionConfiguration, kubeAPIServerOverride string) ( - clientset.Interface, aggregatorclientset.Interface, crdclientset.Interface, apiextensionclientset.Interface, error) { + clientset.Interface, aggregatorclientset.Interface, crdclientset.Interface, apiextensionclientset.Interface, mcclientset.Interface, error) { kubeConfig, err := createRestConfig(config, kubeAPIServerOverride) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } client, err := clientset.NewForConfig(kubeConfig) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } aggregatorClient, err := aggregatorclientset.NewForConfig(kubeConfig) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } - // Create client for crd operations + // Create client for CRD operations. crdClient, err := crdclientset.NewForConfig(kubeConfig) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err } - // Create client for crd manipulations + // Create client for CRD manipulations. apiExtensionClient, err := apiextensionclientset.NewForConfig(kubeConfig) if err != nil { - return nil, nil, nil, nil, err + return nil, nil, nil, nil, nil, err + } + + // Create client for multicluster CRD operations. + mcClient, err := mcclientset.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, nil, nil, nil, err } - return client, aggregatorClient, crdClient, apiExtensionClient, nil + return client, aggregatorClient, crdClient, apiExtensionClient, mcClient, nil } // CreateNetworkAttachDefClient creates net-attach-def client handle from the given config. diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 4961768cc65..ec18222681d 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -117,7 +117,7 @@ func TestConnectivityFlows(t *testing.T) { antrearuntime.WindowsOS = runtime.GOOS } - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) defer func() { @@ -165,7 +165,7 @@ func TestAntreaFlexibleIPAMConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, true, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, true, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) defer func() { @@ -224,7 +224,7 @@ func TestReplayFlowsConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -266,7 +266,7 @@ func TestReplayFlowsNetworkPolicyFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -443,7 +443,7 @@ func TestNetworkPolicyFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -557,7 +557,7 @@ func TestIPv6ConnectivityFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, true, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge: %v", err)) @@ -605,7 +605,7 @@ func TestProxyServiceFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, true, false, false, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -739,7 +739,7 @@ func expectedProxyServiceGroupAndFlows(gid uint32, svc svcConfig, endpointList [ }, }} epDNATFlows := expectTableFlows{tableName: "EndpointDNAT", flows: []*ofTestUtils.ExpectFlow{}} - hairpinFlows := expectTableFlows{tableName: "ServiceMark", flows: []*ofTestUtils.ExpectFlow{}} + hairpinFlows := expectTableFlows{tableName: "SNATMark", flows: []*ofTestUtils.ExpectFlow{}} groupBuckets = make([]string, 0) for _, ep := range endpointList { epIP := ipToHexString(net.ParseIP(ep.IP())) @@ -1361,7 +1361,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { } tableL3DecTTLFlows := expectTableFlows{ tableName: "L3DecTTL", - flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:ServiceMark"}}, + flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:SNATMark"}}, } tableUnSNATFlows := expectTableFlows{ tableName: "UnSNAT", @@ -1371,8 +1371,8 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { tableName: "ConntrackZone", flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:ConntrackState"}}, } - tableServiceMarkFlows := expectTableFlows{ - tableName: "ServiceMark", + tableSNATMarkFlows := expectTableFlows{ + tableName: "SNATMark", flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:SNAT"}}, } if config.enableIPv4 { @@ -1391,7 +1391,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ct_state=+inv+trk,ip", ActStr: "drop"}, ) tableConntrackCommitFlows.flows = append(tableConntrackCommitFlows.flows, - &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ct_mark=0/0x10,ip", ActStr: fmt.Sprintf("ct(commit,table=%s,zone=%s,exec(move:NXM_NX_REG0[0..3]->NXM_NX_CT_MARK[0..3]))", outputStageTable, ctZone)}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk-snat,ct_mark=0/0x10,ip", ActStr: fmt.Sprintf("ct(commit,table=%s,zone=%s,exec(move:NXM_NX_REG0[0..3]->NXM_NX_CT_MARK[0..3]))", outputStageTable, ctZone)}, ) tableSNATFlows.flows = append(tableSNATFlows.flows, &ofTestUtils.ExpectFlow{ @@ -1415,13 +1415,13 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { tableL3ForwardingFlows.flows = append(tableL3ForwardingFlows.flows, &ofTestUtils.ExpectFlow{MatchStr: fmt.Sprintf("priority=190,ip,reg0=0/0x200%s,nw_dst=%s", matchVLANString, podCIDR), ActStr: "goto_table:L2ForwardingCalc"}, ) - tableServiceMarkFlows.flows = append(tableServiceMarkFlows.flows, + tableSNATMarkFlows.flows = append(tableSNATMarkFlows.flows, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x22/0xff", ActStr: fmt.Sprintf("ct(commit,table=SNAT,zone=%s,exec(load:0x1->NXM_NX_CT_MARK[5],load:0x1->NXM_NX_CT_MARK[6]))", ctZone)}, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x12/0xff,reg4=0x200000/0x200000", ActStr: fmt.Sprintf("ct(commit,table=SNAT,zone=%s,exec(load:0x1->NXM_NX_CT_MARK[5]))", ctZone)}, ) tableL3DecTTLFlows.flows = append(tableL3DecTTLFlows.flows, - &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ip,reg0=0x2/0xf", ActStr: "goto_table:ServiceMark"}, - &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ip", ActStr: "dec_ttl,goto_table:ServiceMark"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ip,reg0=0x2/0xf", ActStr: "goto_table:SNATMark"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ip", ActStr: "dec_ttl,goto_table:SNATMark"}, ) } if config.enableIPv6 { @@ -1436,7 +1436,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ct_state=+inv+trk,ipv6", ActStr: "drop"}, ) tableConntrackCommitFlows.flows = append(tableConntrackCommitFlows.flows, - &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ct_mark=0/0x10,ipv6", ActStr: fmt.Sprintf("ct(commit,table=Output,zone=%s,exec(move:NXM_NX_REG0[0..3]->NXM_NX_CT_MARK[0..3]))", ctZoneV6)}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk-snat,ct_mark=0/0x10,ipv6", ActStr: fmt.Sprintf("ct(commit,table=Output,zone=%s,exec(move:NXM_NX_REG0[0..3]->NXM_NX_CT_MARK[0..3]))", ctZoneV6)}, ) tableSNATFlows.flows = append(tableSNATFlows.flows, &ofTestUtils.ExpectFlow{ @@ -1460,13 +1460,13 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { tableL3ForwardingFlows.flows = append(tableL3ForwardingFlows.flows, &ofTestUtils.ExpectFlow{MatchStr: fmt.Sprintf("priority=190,ipv6,reg0=0/0x200,ipv6_dst=%s", podCIDR), ActStr: "goto_table:L2ForwardingCalc"}, ) - tableServiceMarkFlows.flows = append(tableServiceMarkFlows.flows, + tableSNATMarkFlows.flows = append(tableSNATMarkFlows.flows, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ipv6,reg0=0x22/0xff", ActStr: "ct(commit,table=SNAT,zone=65510,exec(load:0x1->NXM_NX_CT_MARK[5],load:0x1->NXM_NX_CT_MARK[6]))"}, &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ipv6,reg0=0x12/0xff,reg4=0x200000/0x200000", ActStr: "ct(commit,table=SNAT,zone=65510,exec(load:0x1->NXM_NX_CT_MARK[5]))"}, ) tableL3DecTTLFlows.flows = append(tableL3DecTTLFlows.flows, - &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ipv6,reg0=0x2/0xf", ActStr: "goto_table:ServiceMark"}, - &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ipv6", ActStr: "dec_ttl,goto_table:ServiceMark"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ipv6,reg0=0x2/0xf", ActStr: "goto_table:SNATMark"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ipv6", ActStr: "dec_ttl,goto_table:SNATMark"}, ) } if config.enableIPv4 && config.connectUplinkToBridge { @@ -1486,7 +1486,7 @@ func prepareDefaultFlows(config *testConfig) []expectTableFlows { tableL3ForwardingFlows, tableL3DecTTLFlows, tableUnSNATFlows, - tableServiceMarkFlows, + tableSNATMarkFlows, tableVLANFlows, { "Classifier", @@ -1683,7 +1683,7 @@ func TestEgressMarkFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, false, false, true, false, false, false, false, false) + c = ofClient.NewClient(br, bridgeMgmtAddr, false, false, true, false, false, false, false, false, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) @@ -1740,7 +1740,7 @@ func TestTrafficControlFlows(t *testing.T) { legacyregistry.Reset() metrics.InitializeOVSMetrics() - c = ofClient.NewClient(br, bridgeMgmtAddr, false, false, false, false, false, false, false, true) + c = ofClient.NewClient(br, bridgeMgmtAddr, false, false, false, false, false, false, false, true, false) err := ofTestUtils.PrepareOVSBridge(br) require.Nil(t, err, fmt.Sprintf("Failed to prepare OVS bridge %s", br)) From f77ebbd87844705357d2a1956f0e545383177301 Mon Sep 17 00:00:00 2001 From: Lan Luo Date: Thu, 9 Jun 2022 10:53:44 +0800 Subject: [PATCH 2/2] Use Service ClusterIPs as MC Service's Endpoints 1. Use Service ClusterIPs instead of Pod IPs as MC Service's Endpoints. The ServiceExport controller will only watch ServiceExport and Service events, and wrap Service's ClusterIPs into a new Endpoint kind of ResourceExport. 2. Includes local Serivce ClusterIP as multi-cluster Service's Endpoints as well. Signed-off-by: Lan Luo --- build/charts/antrea/README.md | 2 +- build/charts/antrea/values.yaml | 2 +- ci/jenkins/test-mc.sh | 52 ++++++-- .../controllers/multicluster/common/helper.go | 26 ++++ .../commonarea/resourceimport_controller.go | 41 +----- .../resourceimport_controller_test.go | 26 ++-- .../multicluster/serviceexport_controller.go | 125 +++--------------- .../serviceexport_controller_test.go | 41 +----- multicluster/hack/mc-integration-test.sh | 6 +- multicluster/test/e2e/service_test.go | 6 +- multicluster/test/integration/README.md | 2 +- .../resourceexport_controller_test.go | 2 +- .../serviceexport_controller_test.go | 85 +++--------- multicluster/test/integration/test_data.go | 4 - 14 files changed, 136 insertions(+), 284 deletions(-) diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index c6e9b828cbc..2124c8b075f 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -83,7 +83,7 @@ Kubernetes: `>= 1.16.0-0` | multicast.igmpQueryInterval | string | `"125s"` | The interval at which the antrea-agent sends IGMP queries to Pods. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". | | multicast.multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. | | multicluster.enable | bool | `false` | Enable Antrea Multi-cluster Gateway to support cross-cluster traffic. This feature is supported only with encap mode. | -| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is Antrea Agent Namespace if it's empty. | +| multicluster.namespace | string | `""` | The Namespace where Antrea Multi-cluster Controller is running. The default is antrea-agent's Namespace. | | noSNAT | bool | `false` | Whether or not to SNAT (using the Node IP) the egress traffic from a Pod to the external network. | | nodeIPAM.clusterCIDRs | list | `[]` | CIDR ranges to use when allocating Pod IP addresses. | | nodeIPAM.enable | bool | `false` | Enable Node IPAM in Antrea | diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 759b8b2232c..32bd2739fae 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -287,7 +287,7 @@ multicluster: # This feature is supported only with encap mode. enable: false # -- The Namespace where Antrea Multi-cluster Controller is running. - # The default is Antrea Agent Namespace if it's empty. + # The default is antrea-agent's Namespace. namespace: "" testing: diff --git a/ci/jenkins/test-mc.sh b/ci/jenkins/test-mc.sh index 3e209e5f5bd..21e34824b02 100755 --- a/ci/jenkins/test-mc.sh +++ b/ci/jenkins/test-mc.sh @@ -33,8 +33,7 @@ LEADER_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/leader" EAST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/east" WEST_CLUSTER_CONFIG="--kubeconfig=$MULTICLUSTER_KUBECONFIG_PATH/west" ENABLE_MC_GATEWAY=false - -CONTROL_PLANE_NODE_ROLE="control-plane,master" +IS_CONTAINERD=false multicluster_kubeconfigs=($EAST_CLUSTER_CONFIG $LEADER_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) membercluster_kubeconfigs=($EAST_CLUSTER_CONFIG $WEST_CLUSTER_CONFIG) @@ -160,6 +159,7 @@ function wait_for_antrea_multicluster_pods_ready { } function wait_for_multicluster_controller_ready { + echo "====== Deploying Antrea Multicluster Leader Cluster with ${LEADER_CLUSTER_CONFIG} ======" kubectl create ns antrea-mcs-ns "${LEADER_CLUSTER_CONFIG}" || true kubectl apply -f ./multicluster/test/yamls/manifest.yml "${LEADER_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-leader-global.yml "${LEADER_CLUSTER_CONFIG}" @@ -175,15 +175,17 @@ function wait_for_multicluster_controller_ready { sed -i '/creationTimestamp/d' ./multicluster/test/yamls/leader-access-token.yml sed -i 's/antrea-multicluster-member-access-sa/antrea-multicluster-controller/g' ./multicluster/test/yamls/leader-access-token.yml sed -i 's/antrea-mcs-ns/kube-system/g' ./multicluster/test/yamls/leader-access-token.yml - echo "type: Opaque" >>./multicluster/test/yamls/leader-access-token.yml + echo "type: Opaque" >> ./multicluster/test/yamls/leader-access-token.yml for config in "${membercluster_kubeconfigs[@]}"; do + echo "====== Deploying Antrea Multicluster Member Cluster with ${config} ======" kubectl apply -f ./multicluster/build/yamls/antrea-multicluster-member.yml ${config} kubectl rollout status deployment/antrea-mc-controller -n kube-system ${config} kubectl apply -f ./multicluster/test/yamls/leader-access-token.yml ${config} done + echo "====== ClusterSet Initialization in Leader and Member Clusters ======" kubectl apply -f ./multicluster/test/yamls/east-member-cluster.yml "${EAST_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/test/yamls/west-member-cluster.yml "${WEST_CLUSTER_CONFIG}" kubectl apply -f ./multicluster/test/yamls/clusterset.yml "${LEADER_CLUSTER_CONFIG}" @@ -213,7 +215,11 @@ function deliver_antrea_multicluster { do kubectl get nodes -o wide --no-headers=true ${kubeconfig}| awk '{print $6}' | while read IP; do rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-ubuntu.tar jenkins@[${IP}]:${WORKDIR}/antrea-ubuntu.tar - ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true + if [[ ${IS_CONTAINERD} ]];then + ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; sudo ctr -n=k8s.io images import ${WORKDIR}/antrea-ubuntu.tar" || true + else + ssh -o StrictHostKeyChecking=no -n jenkins@${IP} "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-ubuntu.tar" || true + fi done done } @@ -232,13 +238,17 @@ function deliver_multicluster_controller { for kubeconfig in "${multicluster_kubeconfigs[@]}" do - kubectl get nodes -o wide --no-headers=true "${kubeconfig}"| awk '{print $6}' | while read IP; do + kubectl get nodes -o wide --no-headers=true "${kubeconfig}" | awk '{print $6}' | while read IP; do rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" "${WORKDIR}"/antrea-mcs.tar jenkins@[${IP}]:${WORKDIR}/antrea-mcs.tar - ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true + if [[ ${IS_CONTAINERD} ]];then + ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; sudo ctr -n=k8s.io images import ${WORKDIR}/antrea-mcs.tar" || true + else + ssh -o StrictHostKeyChecking=no -n jenkins@"${IP}" "${CLEAN_STALE_IMAGES}; docker load -i ${WORKDIR}/antrea-mcs.tar" || true + fi done done - leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') + leader_ip=$(kubectl get nodes -o wide --no-headers=true ${LEADER_CLUSTER_CONFIG} | awk -v role1="master" -v role2="control-plane" '($3 ~ role1 || $3 ~ role2) {print $6}') sed -i "s||${leader_ip}|" ./multicluster/test/yamls/east-member-cluster.yml sed -i "s||${leader_ip}|" ./multicluster/test/yamls/west-member-cluster.yml rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-acnp-copy-span-ns-isolation.yml jenkins@["${leader_ip}"]:"${WORKDIR}"/test-acnp-copy-span-ns-isolation.yml @@ -248,7 +258,7 @@ function deliver_multicluster_controller { # Remove the longest matched substring '*/' from a string like '--kubeconfig=/var/lib/jenkins/.kube/east' # to get the last element which is the cluster name. cluster=${kubeconfig##*/} - ip=$(kubectl get nodes -o wide --no-headers=true ${kubeconfig} | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 == role {print $6}') + ip=$(kubectl get nodes -o wide --no-headers=true ${kubeconfig} | awk -v role1="master" -v role2="control-plane" '($3 ~ role1 || $3 ~ role2) {print $6}') rsync -avr --progress --inplace -e "ssh -o StrictHostKeyChecking=no" ./multicluster/test/yamls/test-${cluster}-serviceexport.yml jenkins@["${ip}"]:"${WORKDIR}"/serviceexport.yml done } @@ -262,7 +272,14 @@ function run_multicluster_e2e { export PATH=$GOROOT/bin:$PATH if [[ ${ENABLE_MC_GATEWAY} ]]; then - sed -i.bak -E "s/#[[:space:]]*Multicluster[[:space:]]*:[[:space:]]*[a-z]+[[:space:]]*$/ Multicluster: true/" build/yamls/antrea.yml + cat > build/yamls/chart-values/antrea.yml < /tmp/mc-integration-kubeconfig -kind create cluster --name=antrea-integration-kind --kubeconfig=/tmp/mc-integration-kubeconfig +kind create cluster --name=antrea-integration --kubeconfig=/tmp/mc-integration-kubeconfig sleep 5 export KUBECONFIG=/tmp/mc-integration-kubeconfig kubectl create namespace leader-ns @@ -38,7 +38,7 @@ kubectl apply -f test/integration/cluster-admin.yml if [[ $NO_LOCAL == "true" ]];then # Run go test in a Docker container - container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' antrea-integration-kind-control-plane) + container_ip=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' antrea-integration-control-plane) sed -i "s|server: https://.*|server: https://${container_ip}:6443|" /tmp/mc-integration-kubeconfig docker run --network kind --privileged --rm \ -w /usr/src/antrea.io/antrea \ diff --git a/multicluster/test/e2e/service_test.go b/multicluster/test/e2e/service_test.go index 1a90fd49d95..578075ef894 100644 --- a/multicluster/test/e2e/service_test.go +++ b/multicluster/test/e2e/service_test.go @@ -186,12 +186,12 @@ func (data *MCTestData) deleteServiceExport(clusterName string) error { // getNodeNamesFromCluster will pick up a Node randomly as the Gateway // and also a regular Node from the specified cluster. -func getNodeNamesFromCluster(nodeName string) (string, string, error) { - rc, output, stderr, err := provider.RunCommandOnNode(nodeName, "kubectl get node -o custom-columns=:metadata.name --no-headers") +func getNodeNamesFromCluster(clusterName string) (string, string, error) { + rc, output, stderr, err := provider.RunCommandOnNode(clusterName, "kubectl get node -o jsonpath='{range .items[*]}{.metadata.name}{\" \"}{end}'") if err != nil || rc != 0 || stderr != "" { return "", "", fmt.Errorf("error when getting Node list: %v, stderr: %s", err, stderr) } - nodes := strings.Split(output, "\n") + nodes := strings.Split(strings.TrimRight(output, " "), " ") gwIdx := rand.Intn(len(nodes)) // #nosec G404: for test only var regularNode string for i, node := range nodes { diff --git a/multicluster/test/integration/README.md b/multicluster/test/integration/README.md index 42b1b77e267..6fb9da365b5 100644 --- a/multicluster/test/integration/README.md +++ b/multicluster/test/integration/README.md @@ -12,7 +12,7 @@ like Service, Endpoints controllers etc. The tests must be run on an real Kubernetes cluster. At the moment, you can simply run `make test-integration` in `antrea/multicluster` folder. It will -create a Kind cluster named `antrea-integration-kind` and execute the integration +create a Kind cluster named `antrea-integration` and execute the integration codes. if you'd like to run the integration test in an existing Kubernetes cluster, you diff --git a/multicluster/test/integration/resourceexport_controller_test.go b/multicluster/test/integration/resourceexport_controller_test.go index 6cb8bcab0dd..8c0f6748743 100644 --- a/multicluster/test/integration/resourceexport_controller_test.go +++ b/multicluster/test/integration/resourceexport_controller_test.go @@ -163,8 +163,8 @@ var _ = Describe("ResourceExport controller", func() { Subsets: []corev1.EndpointSubset{ { Addresses: []corev1.EndpointAddress{ + addr2, addr3, - addr4, }, Ports: epPorts, }, diff --git a/multicluster/test/integration/serviceexport_controller_test.go b/multicluster/test/integration/serviceexport_controller_test.go index 439a2c9e6c2..d01dbc7ced2 100644 --- a/multicluster/test/integration/serviceexport_controller_test.go +++ b/multicluster/test/integration/serviceexport_controller_test.go @@ -53,25 +53,7 @@ var _ = Describe("ServiceExport controller", func() { Namespace: svc.Namespace, Name: svc.Name, } - epNamespacedName := svcNamespacedName - ep := &corev1.Endpoints{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nginx-svc", - Namespace: testNamespace, - }, - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - addr1, - }, - NotReadyAddresses: []corev1.EndpointAddress{ - addr2, - }, - Ports: epPorts, - }, - }, - } svcExport := &k8smcsv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: "nginx-svc", @@ -85,7 +67,7 @@ var _ = Describe("ServiceExport controller", func() { }, } svcResExportName := LocalClusterID + "-" + svc.Namespace + "-" + svc.Name + "-service" - epResExportName := LocalClusterID + "-" + ep.Namespace + "-" + ep.Name + "-endpoints" + epResExportName := LocalClusterID + "-" + svc.Namespace + "-" + svc.Name + "-endpoints" expectedEpResExport := &mcsv1alpha1.ResourceExport{ ObjectMeta: metav1.ObjectMeta{ @@ -94,8 +76,8 @@ var _ = Describe("ServiceExport controller", func() { }, Spec: mcsv1alpha1.ResourceExportSpec{ ClusterID: LocalClusterID, - Name: ep.Name, - Namespace: ep.Namespace, + Name: svc.Name, + Namespace: svc.Namespace, Kind: common.EndpointsKind, }, } @@ -103,20 +85,7 @@ var _ = Describe("ServiceExport controller", func() { ctx := context.Background() It("Should create ResourceExports when new ServiceExport for ClusterIP Service is created", func() { By("By exposing a ClusterIP type of Service") - expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - { - IP: "192.168.17.11", - }, - }, - Ports: epPorts, - }, - }, - } Expect(k8sClient.Create(ctx, svc)).Should(Succeed()) - Expect(k8sClient.Create(ctx, ep)).Should(Succeed()) Expect(k8sClient.Create(ctx, svcExport)).Should(Succeed()) var err error latestSvc := &corev1.Service{} @@ -131,6 +100,18 @@ var _ = Describe("ServiceExport controller", func() { Expect(svcResExport.ObjectMeta.Labels["sourceKind"]).Should(Equal("Service")) Expect(svcResExport.Spec.Service.ServiceSpec.ClusterIP).Should(Equal(latestSvc.Spec.ClusterIP)) Expect(len(svcResExport.Spec.Service.ServiceSpec.Ports)).Should(Equal(len(svcPorts))) + expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: latestSvc.Spec.ClusterIP, + }, + }, + Ports: epPorts, + }, + }, + } Eventually(func() bool { err = k8sClient.Get(ctx, types.NamespacedName{Namespace: LeaderNamespace, Name: epResExportName}, epResExport) return err == nil @@ -180,42 +161,6 @@ var _ = Describe("ServiceExport controller", func() { Expect(*conditions[0].Message).Should(Equal("the Service does not exist")) }) - It("Should update existing ResourceExport when corresponding Endpoints has new Endpoint", func() { - By("By update an Endpoint with a new address") - latestEp := &corev1.Endpoints{} - Expect(k8sClient.Get(ctx, epNamespacedName, latestEp)).Should(Succeed()) - addresses := latestEp.Subsets[0].Addresses - addresses = append(addresses, addr3) - latestEp.Subsets[0].Addresses = addresses - Expect(k8sClient.Update(ctx, latestEp)).Should(Succeed()) - time.Sleep(2 * time.Second) - epResExport := &mcsv1alpha1.ResourceExport{} - expectedEpResExport.Spec.Endpoints = &mcsv1alpha1.EndpointsExport{ - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{ - { - IP: "192.168.17.11", - }, - { - IP: "192.168.17.13", - }, - }, - Ports: epPorts, - }, - }, - } - - var err error - Eventually(func() bool { - err = k8sClient.Get(ctx, types.NamespacedName{Namespace: LeaderNamespace, Name: epResExportName}, epResExport) - return err == nil - }, timeout, interval).Should(BeTrue()) - Expect(epResExport.ObjectMeta.Labels["sourceKind"]).Should(Equal("Endpoints")) - Expect(epResExport.Spec).Should(Equal(expectedEpResExport.Spec)) - - }) - It("Should delete existing ResourceExport when existing ServiceExport is deleted", func() { By("By remove a ServiceExport resource") err := k8sClient.Delete(ctx, svcExport) diff --git a/multicluster/test/integration/test_data.go b/multicluster/test/integration/test_data.go index c2b6c59cd8f..e5abc57ddd1 100644 --- a/multicluster/test/integration/test_data.go +++ b/multicluster/test/integration/test_data.go @@ -29,10 +29,6 @@ var ( IP: "192.168.17.13", Hostname: "pod3", } - addr4 = corev1.EndpointAddress{ - IP: "192.168.17.14", - Hostname: "pod4", - } epPorts = []corev1.EndpointPort{ { Name: "http",