From 393740ce2b7ba72f5d6f9f50a6fbbca5983d47d9 Mon Sep 17 00:00:00 2001 From: Jianjun Shen Date: Thu, 31 Mar 2022 19:58:56 -0400 Subject: [PATCH] Add documentation for secondary network IPAM Add a document for secondary network IPAM, and add the Antrea IPAM option to the Multus cookbook. Signed-off-by: Jianjun Shen --- build/yamls/antrea-aks.yml | 22 +-- build/yamls/antrea-eks.yml | 22 +-- build/yamls/antrea-gke.yml | 22 +-- build/yamls/antrea-ipsec.yml | 22 +-- build/yamls/antrea.yml | 22 +-- build/yamls/base/conf/antrea-agent.conf | 9 +- build/yamls/base/conf/antrea-controller.conf | 5 +- docs/antrea-ipam.md | 166 +++++++++++++++++- docs/cookbooks/multus/README.md | 101 ++++++++--- .../multus/resources/netAttachDef.yml | 14 -- .../multus/resources/netattachdef-ippool.yml | 27 +++ docs/feature-gates.md | 36 ++-- pkg/config/agent/config.go | 6 +- pkg/features/antrea_features.go | 2 +- 14 files changed, 360 insertions(+), 116 deletions(-) delete mode 100644 docs/cookbooks/multus/resources/netAttachDef.yml create mode 100644 docs/cookbooks/multus/resources/netattachdef-ippool.yml diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index eee762e7e45..14e4f1ab777 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -2771,7 +2771,8 @@ data: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the - # bridging mode and allocates IPs to Pods in bridging mode. + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -2840,9 +2841,9 @@ data: #trafficEncryptionMode: none # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - # to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - # allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - # forwarded/routed by the underlay network. + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. @@ -3016,8 +3017,9 @@ data: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false - # Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, - # Deployments and StatefulSets via IP Pool annotation. + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. @@ -3074,7 +3076,7 @@ kind: ConfigMap metadata: labels: app: antrea - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 namespace: kube-system --- apiVersion: v1 @@ -3145,7 +3147,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-82h2mk24gg + value: antrea-config-mkbgmf6ct6 image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -3196,7 +3198,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 name: antrea-config - name: antrea-controller-tls secret: @@ -3435,7 +3437,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 53a6ef8c0a3..1a89b336234 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -2771,7 +2771,8 @@ data: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the - # bridging mode and allocates IPs to Pods in bridging mode. + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -2840,9 +2841,9 @@ data: #trafficEncryptionMode: none # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - # to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - # allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - # forwarded/routed by the underlay network. + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. @@ -3016,8 +3017,9 @@ data: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false - # Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, - # Deployments and StatefulSets via IP Pool annotation. + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. @@ -3074,7 +3076,7 @@ kind: ConfigMap metadata: labels: app: antrea - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 namespace: kube-system --- apiVersion: v1 @@ -3145,7 +3147,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-82h2mk24gg + value: antrea-config-mkbgmf6ct6 image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -3196,7 +3198,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 name: antrea-config - name: antrea-controller-tls secret: @@ -3437,7 +3439,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-82h2mk24gg + name: antrea-config-mkbgmf6ct6 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 78552174cbd..e4dda439b4d 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -2771,7 +2771,8 @@ data: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the - # bridging mode and allocates IPs to Pods in bridging mode. + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -2840,9 +2841,9 @@ data: #trafficEncryptionMode: none # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - # to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - # allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - # forwarded/routed by the underlay network. + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. @@ -3016,8 +3017,9 @@ data: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false - # Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, - # Deployments and StatefulSets via IP Pool annotation. + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. @@ -3074,7 +3076,7 @@ kind: ConfigMap metadata: labels: app: antrea - name: antrea-config-c9ck44454h + name: antrea-config-2c8t9465tc namespace: kube-system --- apiVersion: v1 @@ -3145,7 +3147,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-c9ck44454h + value: antrea-config-2c8t9465tc image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -3196,7 +3198,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-c9ck44454h + name: antrea-config-2c8t9465tc name: antrea-config - name: antrea-controller-tls secret: @@ -3438,7 +3440,7 @@ spec: path: /home/kubernetes/bin name: host-cni-bin - configMap: - name: antrea-config-c9ck44454h + name: antrea-config-2c8t9465tc name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 9241703a4f8..37c138caec0 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -2771,7 +2771,8 @@ data: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the - # bridging mode and allocates IPs to Pods in bridging mode. + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -2840,9 +2841,9 @@ data: trafficEncryptionMode: ipsec # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - # to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - # allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - # forwarded/routed by the underlay network. + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. @@ -3021,8 +3022,9 @@ data: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false - # Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, - # Deployments and StatefulSets via IP Pool annotation. + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. @@ -3079,7 +3081,7 @@ kind: ConfigMap metadata: labels: app: antrea - name: antrea-config-tmhkc66d6c + name: antrea-config-29g6gtcctg namespace: kube-system --- apiVersion: v1 @@ -3159,7 +3161,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-tmhkc66d6c + value: antrea-config-29g6gtcctg image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -3210,7 +3212,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-tmhkc66d6c + name: antrea-config-29g6gtcctg name: antrea-config - name: antrea-controller-tls secret: @@ -3484,7 +3486,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-tmhkc66d6c + name: antrea-config-29g6gtcctg name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index d5a7d4db29f..7bd2c8cabeb 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -2771,7 +2771,8 @@ data: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the - # bridging mode and allocates IPs to Pods in bridging mode. + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -2840,9 +2841,9 @@ data: #trafficEncryptionMode: none # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - # to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - # allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - # forwarded/routed by the underlay network. + # to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + # allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + # underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. @@ -3021,8 +3022,9 @@ data: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false - # Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, - # Deployments and StatefulSets via IP Pool annotation. + # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the + # bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for + # IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. @@ -3079,7 +3081,7 @@ kind: ConfigMap metadata: labels: app: antrea - name: antrea-config-hkhbh5gf99 + name: antrea-config-bb75mkktfg namespace: kube-system --- apiVersion: v1 @@ -3150,7 +3152,7 @@ spec: fieldRef: fieldPath: spec.serviceAccountName - name: ANTREA_CONFIG_MAP_NAME - value: antrea-config-hkhbh5gf99 + value: antrea-config-bb75mkktfg image: projects.registry.vmware.com/antrea/antrea-ubuntu:latest imagePullPolicy: IfNotPresent livenessProbe: @@ -3201,7 +3203,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-hkhbh5gf99 + name: antrea-config-bb75mkktfg name: antrea-config - name: antrea-controller-tls secret: @@ -3440,7 +3442,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-hkhbh5gf99 + name: antrea-config-bb75mkktfg name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index 912cd971bf9..01745c081e1 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -32,7 +32,8 @@ featureGates: # Egress: true # Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the -# bridging mode and allocates IPs to Pods in bridging mode. +# bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for +# IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable multicast traffic. This feature is supported only with noEncap mode. @@ -101,9 +102,9 @@ featureGates: #trafficEncryptionMode: none # Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected -# to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are -# allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and -# forwarded/routed by the underlay network. +# to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are +# allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the +# underlay network. # This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only # IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, # `trafficEncapMode` is `noEncap`, and `noSNAT` is true. diff --git a/build/yamls/base/conf/antrea-controller.conf b/build/yamls/base/conf/antrea-controller.conf index 7db2983b300..204153cb40c 100644 --- a/build/yamls/base/conf/antrea-controller.conf +++ b/build/yamls/base/conf/antrea-controller.conf @@ -17,8 +17,9 @@ featureGates: # Run Kubernetes NodeIPAMController with Antrea. # NodeIPAM: false -# Enable flexible IPAM mode for Antrea. This mode allows to assign IP Ranges to Namespaces, -# Deployments and StatefulSets via IP Pool annotation. +# Enable AntreaIPAM, which can allocate IP addresses from IPPools. AntreaIPAM is required by the +# bridging mode and allocates IPs to Pods in bridging mode. It is also required to use Antrea for +# IPAM when configuring secondary network interfaces with Multus. # AntreaIPAM: false # Enable managing external IPs of Services of LoadBalancer type. diff --git a/docs/antrea-ipam.md b/docs/antrea-ipam.md index 6cf76d8b6cc..a63a1110d1d 100644 --- a/docs/antrea-ipam.md +++ b/docs/antrea-ipam.md @@ -93,7 +93,7 @@ Antrea deployment YAML are: #### Create IPPool CR -The following example YAML manifests create an IPPool CR. +The following example YAML manifest creates an IPPool CR. ```yaml apiVersion: "crd.antrea.io/v1alpha2" @@ -112,7 +112,7 @@ spec: #### IPPool Annotations on Namespace -The following example YAML manifests create a Namespace to allocate Pod IPs from the IP pool. +The following example YAML manifest creates a Namespace to allocate Pod IPs from the IP pool. ```yaml kind: Namespace @@ -214,3 +214,165 @@ IP, because inter-Node traffic of AntreaIPAM Pods is forwarded by the Node netwo router should provide the network connectivity for these VLANs. Only a single IP pool can be included in the Namespace annotation. In the future, annotation of up to two pools for IPv4 and IPv6 respectively will be supported. + +## IPAM for Secondary Network + +With the AntreaIPAM feature, Antrea can allocate IPs for Pod secondary networks. At the +moment, AntreaIPAM supports secondary networks managed by [Multus](https://github.com/k8snetworkplumbingwg/multus-cni), +we will add support for [secondary networks managed by Antrea](feature-gates.md#secondarynetwork) +in the future. + +### Prerequisites + +The IPAM capability for secondary network was added in Antrea version 1.7. It +requires the `AntreaIPAM` feature gate to be enabled on both `antrea-controller` +and `antrea-agent`, as `AntreaIPAM` is still an alpha feature at this moment and +is not enabled by default. + +### CNI IPAM configuration + +To configure Antrea IPAM, `antrea` should be specified as the IPAM plugin in the +the CNI IPAM configuration, and at least one Antrea IPPool should be specified +in the `ippools` field. IPs will be allocated from the specified IPPool(s) for +the secondary network. + +```json +{ + "cniVersion": "0.3.0", + "name": "ipv4-net-1", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "antrea", + "ippools": [ "ipv4-pool-1" ] + } +} +``` + +Multiple IPPools can be specified to allocate multiple IPs from each IPPool for +the secondary network. For example, you can specify one IPPool to allocate an +IPv4 address and another IPPool to allocate an IPv6 address in the dual-stack +case. + +```json +{ + "cniVersion": "0.3.0", + "name": "dual-stack-net-1", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "antrea", + "ippools": [ "ipv4-pool-1", "ipv6-pool-1" ] + } +} +``` + +Additionally, Antrea IPAM also supports the same configuration of static IP +addresses, static routes, and DNS settings, as what is supported by the +[static IPAM plugin](https://www.cni.dev/plugins/current/ipam/static). The +following example requests an IP from an IPPool and also specifies two +additional static IP addresses. It also includes static routes and DNS settings. + +```json +{ + "cniVersion": "0.3.0", + "name": "pool-and-static-net-1", + "type": "bridge", + "bridge": "br0" + "ipam": { + "type": "antrea", + "ippools": [ "ipv4-pool-1" ], + "addresses": [ + { + "address": "10.10.0.1/24", + "gateway": "10.10.0.254" + }, + { + "address": "3ffe:ffff:0:01ff::1/64", + "gateway": "3ffe:ffff:0::1" + } + ], + "routes": [ + { "dst": "0.0.0.0/0" }, + { "dst": "192.168.0.0/16", "gw": "10.10.5.1" }, + { "dst": "3ffe:ffff:0:01ff::1/64" } + ], + "dns": { + "nameservers" : ["8.8.8.8"], + "domain": "example.com", + "search": [ "example.com" ] + } + } +} +``` + +The CNI IPAM configuration can include only static addresses without IPPools, if +only static IP addresses are needed. + +### Configuration with `NetworkAttachmentDefinition` CRD + +CNI and IPAM configuration of a secondary network is typically defined with the +`NetworkAttachmentDefinition` CRD. For example: + +```yaml +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: ipv4-net-1 +spec: + { + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "antrea", + "ippools": [ "ipv4-pool-1" ] + } + } +``` + +## `IPPool` CRD + +Antrea IP pools are defined with the `IPPool` CRD. The following two examples +define an IPv4 and an IPv6 IP pool respectively. + +```yaml +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: ipv4-pool-1 +spec: + ipVersion: 4 + ipRanges: + - cidr: "10.10.1.0/26" + gateway: "10.10.1.1" + prefixLength: 24 +``` + +```yaml +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: ipv6-pool-1 +spec: + ipVersion: 6 + ipRanges: + - start: "3ffe:ffff:1:01ff::0100" + end: "3ffe:ffff:1:01ff::0200" + gateway: "3ffe:ffff:1:01ff::1" + prefixLength: 64 +``` + +VLAN ID in the IP range subnet definition of `IPPool` CRD is not supported for +secondary network IPAM. + +### Secondary Network creation with Multus + +To leverage Antrea for secondary network IPAM, Antrea must be used as the CNI +for the Pods' primary network, while the secondary networks are implemented by +other CNIs which are managed by Multus. The [Antrea + Multus guide](cookbooks/multus) +talks about how to use Antrea with Multus, including the option of using Antrea +IPAM for secondary networks. diff --git a/docs/cookbooks/multus/README.md b/docs/cookbooks/multus/README.md index d159621be17..0f30174ef18 100644 --- a/docs/cookbooks/multus/README.md +++ b/docs/cookbooks/multus/README.md @@ -12,7 +12,7 @@ e.g. [ipvlan](https://github.com/containernetworking/plugins/tree/master/plugins ## Prerequisites -The only prerequisites are: +The general prerequisites are: * a K8s cluster (Linux Nodes) running a K8s version supported by Antrea. At the time of writing, we recommend version 1.16 or later. Typically the cluster @@ -20,6 +20,11 @@ The only prerequisites are: using macvlan networking will not work on public clouds like AWS. * [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +The Antrea IPAM capability for secondary network was added in Antrea version +1.7. To leverage Antrea IPAM for IP assignment of secondary networks, an Antrea +version >= 1.7.0 should be used. There is no Antrea version requirement for +other IPAM options. + All the required software will be deployed using YAML manifests, and the corresponding container images will be downloaded from public registries. @@ -79,26 +84,57 @@ use as an example in this guide. ### Step 1: Deploying Antrea For detailed information on the Antrea requirements and instructions on how to -deploy Antrea, please refer to -[getting-started.md](../../getting-started.md). To deploy the latest version of -Antrea, use: +deploy Antrea, please refer to [getting-started.md](../../getting-started.md). +You can deploy the latest version of Antrea with +[the manifest](https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml). +You may also choose a [released Antrea version](https://github.com/antrea-io/antrea/releases). + +To leverage Antrea IPAM to assign IP addresses for the secondary network, you +need to edit the Antrea deployment manifest and enable the `AntreaIPAM` feature +gate for both `antrea-controller` and `antrea-agent`, and then deploy Antrea +with: ```bash kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml ``` -You may also choose a [released Antrea -version](https://github.com/antrea-io/antrea/releases). +If you choose other IPAM options like DHCP or Whereabouts, you can just deploy +Antrea with the Antrea deployment manifest without modification: + +```bash +kubectl apply -f https://raw.githubusercontent.com/antrea-io/antrea/main/build/yamls/antrea.yml +``` ### Step 2: Deploy Multus as a DaemonSet ```bash git clone https://github.com/k8snetworkplumbingwg/multus-cni && cd multus-cni -cat ./images/multus-daemonset.yml | kubectl apply -f - +cat ./deployments/multus-daemonset-thick-plugin.yml | kubectl apply -f - ``` -### Step 3: Create a NetworkAttachmentDefinition +### Step 3: Create an `IPPool` and a `NetworkAttachmentDefinition` + +With Antrea IPAM, the subnet and IP ranges for the secondary network are defined +with an Antrea `IPPool` CR. To learn more information about Antrea IPAM for +secondary network, please refer to the [Antrea IPAM documentation](../../antrea-ipam.md#ipam-for-secondary-network). + +```bash +cat < -## Using [whereabouts] for IPAM +## Using [Whereabouts] for IPAM If you do not already have a DHCP server for the underlying parent network and you find that deploying one in-cluster is impractical, you may want to consider -using [whereabouts] to assign IP addresses to the secondary interfaces. When -using [whereabouts], follow steps 1 and 2 above, along with step 4 if you want +using [Whereabouts] to assign IP addresses to the secondary interfaces. When +using [Whereabouts], follow steps 1 and 2 above, along with step 4 if you want the Nodes to be able to communicate with the Pods using the secondary network. -The next step is to install the [whereabouts] plugin as follows: +The next step is to install the [Whereabouts] plugin as follows: ```bash git clone https://github.com/dougbtv/whereabouts && cd whereabouts @@ -331,11 +377,12 @@ EOF You can then validate that the configuration works by running the same [test](#testing) as above. -[whereabouts]: https://github.com/dougbtv/whereabouts +[Whereabouts]: https://github.com/dougbtv/whereabouts [test cluster]: #suggested-test-cluster +[DHCP]: #using-dhcp-for-ipam [step-1]: #step-1-deploying-antrea [step-2]: #step-2-deploy-multus-as-a-daemonset -[step-3]: #step-3-create-a-networkattachmentdefinition +[step-3]: #step-3-create-an-ippool-and-a-networkattachmentdefinition [step-4]: #step-4-optional-create-a-macvlan-subinterface-on-each-node [step-5]: #step-5-run-a-dhcp-server [step-6]: #step-6-run-the-dhcp-daemons diff --git a/docs/cookbooks/multus/resources/netAttachDef.yml b/docs/cookbooks/multus/resources/netAttachDef.yml deleted file mode 100644 index 650058a7530..00000000000 --- a/docs/cookbooks/multus/resources/netAttachDef.yml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: "k8s.cni.cncf.io/v1" -kind: NetworkAttachmentDefinition -metadata: - name: macvlan-conf -spec: - config: '{ - "cniVersion": "0.3.0", - "type": "macvlan", - "master": "enp0s9", - "mode": "bridge", - "ipam": { - "type": "dhcp" - } - }' diff --git a/docs/cookbooks/multus/resources/netattachdef-ippool.yml b/docs/cookbooks/multus/resources/netattachdef-ippool.yml new file mode 100644 index 00000000000..55f8da11a7b --- /dev/null +++ b/docs/cookbooks/multus/resources/netattachdef-ippool.yml @@ -0,0 +1,27 @@ +apiVersion: "crd.antrea.io/v1alpha2" +kind: IPPool +metadata: + name: macvlan-ippool +spec: + ipVersion: 4 + ipRanges: + - start: "192.168.78.200" + end: "192.168.78.250" + gateway: "192.168.78.1" + prefixLength: 24 +--- +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-conf +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "enp0s9", + "mode": "bridge", + "ipam": { + "type": "antrea", + "ippools" [ "macvlan-ippool" ] + } + }' diff --git a/docs/feature-gates.md b/docs/feature-gates.md index fdb82952bcf..6de850ca268 100644 --- a/docs/feature-gates.md +++ b/docs/feature-gates.md @@ -248,23 +248,33 @@ there is a risk of conflicts in CIDR allocation between the two. ### AntreaIPAM -`AntreaIPAM` feature allows flexible control over Pod IP addressing. This can be -achieved by configuring `IPPool` CRD with a desired set of IP ranges and VLANs. The -`IPPool` can be annotated to Namespace, Pod and PodTemplate of StatefulSet/Deployment. -Antrea will manage IP address assignment for corresponding Pods according to `IPPool` -spec. Refer to this [document](antrea-ipam.md) for more information. +`AntreaIPAM` feature allocates IP addresses from IPPools. It is required by +bridging mode Pods. The bridging mode allows flexible control over Pod IP +addressing. The desired set of IP ranges, optionally with VLANs, are defined +with `IPPool` CRD. An IPPool can be annotated to Namespace, Pod and PodTemplate +of StatefulSet/Deployment. Then, Antrea will manage IP address assignment for +corresponding Pods according to `IPPool` spec. On a Node, cross-Node/VLAN +traffic of AntreaIPAM Pods is sent to the underlay network, and forwarded/routed +by the underlay network. For more information, please refer to the +[Antrea IPAM document](antrea-ipam.md#antrea-flexible-ipam). + +This feature gate also needs to be enabled to use Antrea for IPAM when +configuring secondary network interfaces with Multus, in which case Antrea works +as an IPAM plugin and allocates IP addresses for Pods' secondary networks, +again from the configured IPPools of a secondary network. Refer to the +[secondary network IPAM document](antrea-ipam.md#ipam-for-secondary-network) to +learn more information. #### Requirements for this Feature -As of now, this feature is supported on Linux Nodes, with IPv4, `system` OVS datapath -type, and `noEncap`, `noSNAT` traffic mode. +Both bridging mode and secondary network IPAM are supported only on Linux Nodes. -The IPs in the `IPPools` without VLAN must be in the same underlay subnet as the Node -IP, because inter-Node traffic of AntreaIPAM Pods is forwarded by the Node network. -`IPPools` with VLAN must not overlap with other network subnets, and the underlay network -router should provide the network connectivity for these VLANs. Only a single IP pool can -be included in the Namespace annotation. In the future, annotation of up to two pools for -IPv4 and IPv6 respectively will be supported. +The bridging mode works only with `system` OVS datapath type; and `noEncap`, +`noSNAT` traffic mode. At the moment, it supports only IPv4. The IPs in an IP +range without a VLAN must be in the same underlay subnet as the Node IPs, + because inter-Node traffic of AntreaIPAM Pods is forwarded by the Node network. +IP ranges with a VLAN must not overlap with other network subnets, and the +underlay network router should provide the network connectivity for these VLANs. ### Multicast diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index 2d5dbe2a272..690e8491a4f 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -104,9 +104,9 @@ type AgentConfig struct { // WireGuard related configurations. WireGuard WireGuardConfig `yaml:"wireGuard"` // Enable bridging mode of Pod network on Nodes, in which the Node's transport interface is connected - // to the OVS bridge, and cross-Node/VLAN traffic from AntreaIPAM Pods (Pods whose IP addresses are - // allocated by AntreaIPAM from IPPools) is sent to the underlay network via the uplink, and - // forwarded/routed by the underlay network. + // to the OVS bridge, and cross-Node/VLAN traffic of AntreaIPAM Pods (Pods whose IP addresses are + // allocated by AntreaIPAM from IPPools) is sent to the underlay network, and forwarded/routed by the + // underlay network. // This option requires the `AntreaIPAM` feature gate to be enabled. At this moment, it supports only // IPv4 and Linux Nodes, and can be enabled only when `ovsDatapathType` is `system`, // `trafficEncapMode` is `noEncap`, and `noSNAT` is true. diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 6eb6f0adb64..9b638f9132c 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -77,7 +77,7 @@ const ( NodeIPAM featuregate.Feature = "NodeIPAM" // alpha: v1.4 - // Enable flexible IPAM for Pods. + // Enable AntreaIPAM, which is required by bridging mode Pods and secondary network IPAM. AntreaIPAM featuregate.Feature = "AntreaIPAM" // alpha: v1.5