From 82e6835bc0578024f896e0dc24f376dfa5bcabe0 Mon Sep 17 00:00:00 2001 From: Xu Liu Date: Fri, 13 May 2022 02:45:35 +0000 Subject: [PATCH] Support IPsec Certificate-based Authentication Introduce a new authentication mode for IPsec tunnel encryption. A new config `ipsec.authenticationMode` is added to the Agent. Now Antrea supports both "psk" and "cert" modes for IPsec authentication If "cert" is enabled, Antrea Agent will request IPsec certificates by Kubernetes CertificateSigningRequests API with signer name `antrea.io/antrea-agent-ipsec-tunnel`. Antrea Controller will validate the requests and issue certificates if the request is permitted. The signer `antrea.io/antrea-agent-ipsec-tunnel` in Antrea Controller has the following properties: - Trust distribution - CA certificate will be saved as a ConfigMap `antrea-ipsec-ca` in `kube-system` namespace. - Permitted subjects - organizations are exactly ["antrea.io"], common name must be one of the Node names. - Permitted x509 extension - honors key usage and DNSName extensions, forbids other subjectAltName extensions. DNS name must be the same as the common name. - Permitted key usages - exactly ["ipsec tunnel"] - Expiration/certificate lifetime - defaults to 1 year. - CA bit allowed/disallowed - not allowed. The CSRs can be automatically approved by default. Antrea Controller will validate Pod identity to provide maximum security if the Kubernetes `BoundServiceAccountTokenVolume` feature gate is enabled. Antrea Agents will renew certificates automatically when the certificate reaches approximately 80% of the lifetime. This feature requires feature gate `IPsecCertAuth` to be enabled. Closes: #3765 Signed-off-by: Xu Liu --- build/charts/antrea/README.md | 4 + build/charts/antrea/conf/antrea-agent.conf | 13 + .../charts/antrea/conf/antrea-controller.conf | 19 + .../antrea/templates/agent/clusterrole.yaml | 9 + .../antrea/templates/agent/daemonset.yaml | 14 + .../templates/controller/clusterrole.yaml | 36 ++ build/charts/antrea/values.yaml | 8 + build/yamls/antrea-aks.yml | 77 ++- build/yamls/antrea-eks.yml | 77 ++- build/yamls/antrea-gke.yml | 77 ++- build/yamls/antrea-ipsec.yml | 87 ++- build/yamls/antrea.yml | 77 ++- cmd/antrea-agent/agent.go | 21 +- cmd/antrea-agent/options.go | 7 + cmd/antrea-controller/controller.go | 31 + cmd/antrea-controller/options.go | 16 +- pkg/agent/agent.go | 69 +- pkg/agent/config/ipsec_authentication_mode.go | 53 ++ pkg/agent/config/node_config.go | 8 +- .../ipsec_certificate_controller.go | 482 ++++++++++++++ .../ipsec_certificate_controller_test.go | 463 ++++++++++++++ .../noderoute/node_route_controller.go | 83 ++- .../noderoute/node_route_controller_test.go | 70 +- pkg/agent/interfacestore/types.go | 9 +- pkg/apis/certificates.go | 22 + pkg/config/agent/config.go | 9 + pkg/config/controller/config.go | 14 + .../approving_controller.go | 169 +++++ .../approving_controller_test.go | 195 ++++++ .../certificatesigningrequest/common.go | 158 +++++ .../ipsec_csr_approver.go | 155 +++++ .../ipsec_csr_approver_test.go | 604 ++++++++++++++++++ .../ipsec_csr_signing_controller.go | 470 ++++++++++++++ .../ipsec_csr_signing_controller_test.go | 139 ++++ pkg/features/antrea_features.go | 5 + pkg/ovs/ovsconfig/interfaces.go | 3 +- pkg/ovs/ovsconfig/ovs_client.go | 72 ++- pkg/ovs/ovsconfig/testing/mock_ovsconfig.go | 22 +- test/e2e/ipsec_test.go | 95 ++- test/integration/ovs/ovs_client_test.go | 29 +- 40 files changed, 3860 insertions(+), 111 deletions(-) create mode 100644 pkg/agent/config/ipsec_authentication_mode.go create mode 100644 pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go create mode 100644 pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go create mode 100644 pkg/apis/certificates.go create mode 100644 pkg/controller/certificatesigningrequest/approving_controller.go create mode 100644 pkg/controller/certificatesigningrequest/approving_controller_test.go create mode 100644 pkg/controller/certificatesigningrequest/common.go create mode 100644 pkg/controller/certificatesigningrequest/ipsec_csr_approver.go create mode 100644 pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go create mode 100644 pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go create mode 100644 pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index d68c2d0faae..0506a97cec2 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -73,6 +73,10 @@ Kubernetes: `>= 1.16.0-0` | flowCollector.idleFlowExportTimeout | string | `"15s"` | timeout after which a flow record is sent to the collector for idle flows. | | hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. | | image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/antrea-ubuntu","tag":"latest"}` | Container image to use for Antrea components. | +| ipsec.authenticationMode | string | `"psk"` | The authentication mode to use for IPsec. Must be one of "psk" or "cert". | +| ipsec.csrSigner | object | `{"autoApprove":true,"selfSignedCA":true}` | CSR signer configuration when the authenticationMode is "cert". | +| ipsec.csrSigner.autoApprove | bool | `true` | - Enable auto approval of Antrea signer for IPsec certificates. | +| ipsec.csrSigner.selfSignedCA | bool | `true` | - Whether or not to use auto-generated self-signed CA. | | ipsec.psk | string | `"changeme"` | Preshared Key (PSK) for IKE authentication. It will be stored in a secret and passed to antrea-agent as an environment variable. | | kubeAPIServerOverride | string | `""` | Address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. | | logVerbosity | int | `0` | | diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index d60c167177e..0527661c0eb 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -50,6 +50,9 @@ featureGates: # Enable mirroring or redirecting the traffic Pods send or receive. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "TrafficControl" "default" false) }} +# Enable certificated-based authentication for IPsec. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} @@ -279,3 +282,13 @@ antreaProxy: # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: {{ .proxyLoadBalancerIPs }} {{- end }} + +# IPsec tunnel related configurations. +ipsec: +{{- with .Values.ipsec }} + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: {{ .authenticationMode | quote }} +{{- end }} diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index f54bf5ce864..2ce21423ae7 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -25,6 +25,9 @@ featureGates: # Enable managing external IPs of Services of LoadBalancer type. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ServiceExternalIP" "default" false) }} +# Enable certificated-based authentication for IPsec. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -71,3 +74,19 @@ nodeIPAM: # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: {{ .nodeCIDRMaskSizeIPv6 }} {{- end }} + +ipsecCSRSigner: +{{- with .Values.ipsec }} + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: {{ .csrSigner.autoApprove }} + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: {{ .csrSigner.selfSignedCA }} +{{- end }} diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 10e847a669b..4520872afdb 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -180,3 +180,12 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml index 3d24e44b45f..9cee03c60dd 100644 --- a/build/charts/antrea/templates/agent/daemonset.yaml +++ b/build/charts/antrea/templates/agent/daemonset.yaml @@ -205,6 +205,11 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true + {{- end }} # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -305,6 +310,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true {{- end }} volumes: - name: antrea-config @@ -322,6 +330,12 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca + optional: true + {{- end }} - name: host-var-run-antrea hostPath: path: /var/run/antrea diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index 2a5f043af35..0e7bacd56c9 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -80,14 +80,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -128,6 +140,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index f7212eb3289..b4a1b90bf67 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -59,9 +59,17 @@ wireGuard: port: 51820 ipsec: + # -- The authentication mode to use for IPsec. Must be one of "psk" or "cert". + authenticationMode: "psk" # -- Preshared Key (PSK) for IKE authentication. It will be stored in a secret # and passed to antrea-agent as an environment variable. psk: "changeme" + # -- CSR signer configuration when the authenticationMode is "cert". + csrSigner: + # --- Enable auto approval of Antrea signer for IPsec certificates. + autoApprove: true + # --- Whether or not to use auto-generated self-signed CA. + selfSignedCA: true egress: # -- CIDR ranges to which outbound Pod traffic will not be SNAT'd by Egresses. diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index d29d3f36159..0957b19a79c 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 7587cc5d41e..aa2c7efdd6a 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-agent @@ -3738,7 +3811,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 7294819af5f..47e83a0ec7c 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e0d8c70d728f9f981756d8238d9b19a9c2321206b09a814a1cdb4ac604b190c + checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e0d8c70d728f9f981756d8238d9b19a9c2321206b09a814a1cdb4ac604b190c + checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 2047b52f63d..7ea65444221 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -121,6 +121,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -325,6 +328,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -376,6 +387,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -417,6 +431,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3009,6 +3037,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3170,14 +3207,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3218,6 +3267,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3509,7 +3582,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5038309e5a226d5d860b167b95e0d8ed55af1914526f52e5ef8600e527695e5 + checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -3639,6 +3712,9 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -3727,6 +3803,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true volumes: - name: antrea-config configMap: @@ -3743,6 +3822,10 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca + optional: true - name: host-var-run-antrea hostPath: path: /var/run/antrea @@ -3785,7 +3868,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5038309e5a226d5d860b167b95e0d8ed55af1914526f52e5ef8600e527695e5 + checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 2b02024377d..3ec7336f8cc 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 03ee0481c65f3ec8ff9e879ff10c3cf65991a347a7aec1c9bc866b019ec470f1 + checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 03ee0481c65f3ec8ff9e879ff10c3cf65991a347a7aec1c9bc866b019ec470f1 + checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 labels: app: antrea component: antrea-controller diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 362e3788996..00a162044de 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/cniserver/ipam" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/controller/egress" + "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/controller/networkpolicy" "antrea.io/antrea/pkg/agent/controller/noderoute" "antrea.io/antrea/pkg/agent/controller/serviceexternalip" @@ -148,12 +149,16 @@ func run(o *Options) error { klog.InfoS("enableIPSecTunnel is deprecated, use trafficEncryptionMode instead.") encryptionMode = config.TrafficEncryptionModeIPSec } + _, ipsecAuthenticationMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) networkConfig := &config.NetworkConfig{ TunnelType: ovsconfig.TunnelType(o.config.TunnelType), TrafficEncapMode: encapMode, TrafficEncryptionMode: encryptionMode, TransportIface: o.config.TransportInterface, TransportIfaceCIDRs: o.config.TransportInterfaceCIDRs, + IPsecConfig: config.IPsecConfig{ + AuthenticationMode: ipsecAuthenticationMode, + }, } wireguardConfig := &config.WireGuardConfig{ @@ -228,6 +233,13 @@ func run(o *Options) error { } nodeConfig := agentInitializer.GetNodeConfig() + var ipsecCertController *ipseccertificate.Controller + + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + ipsecCertController = ipseccertificate.NewIPSecCertificateController(k8sClient, ovsBridgeClient, nodeConfig.Name) + } + nodeRouteController := noderoute.NewNodeRouteController( k8sClient, informerFactory, @@ -238,7 +250,9 @@ func run(o *Options) error { networkConfig, nodeConfig, agentInitializer.GetWireGuardClient(), - o.config.AntreaProxy.ProxyAll) + o.config.AntreaProxy.ProxyAll, + ipsecCertController, + ) var groupCounters []proxytypes.GroupCounter groupIDUpdates := make(chan string, 100) @@ -484,6 +498,11 @@ func run(o *Options) error { go antreaClientProvider.Run(ctx) + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + go ipsecCertController.Run(stopCh) + } + go nodeRouteController.Run(stopCh) go networkPolicyController.Run(stopCh) diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index fbc7c7784d0..7f4da6d3d9e 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -124,6 +124,13 @@ func (o *Options) validate(args []string) error { if !ok { return fmt.Errorf("TrafficEncapMode %s is unknown", o.config.TrafficEncapMode) } + ok, ipsecAuthMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) + if !ok { + return fmt.Errorf("IPsec AuthenticationMode %s is unknown", o.config.TrafficEncapMode) + } + if ipsecAuthMode == config.IPsecAuthenticationModeCert && !features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + return fmt.Errorf("IPsec AuthenticationMode %s requires feature gate %s to be enabled", o.config.TrafficEncapMode, features.IPsecCertAuth) + } // Check if the enabled features are supported on the OS. if err := o.checkUnsupportedFeatures(); err != nil { diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 25b75bca3bb..47c0c553514 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -24,21 +24,28 @@ import ( "time" apiextensionclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" genericopenapi "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/informers" + csrinformers "k8s.io/client-go/informers/certificates/v1" clientset "k8s.io/client-go/kubernetes" + csrlisters "k8s.io/client-go/listers/certificates/v1" + "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" netutils "k8s.io/utils/net" + antreaapis "antrea.io/antrea/pkg/apis" "antrea.io/antrea/pkg/apiserver" "antrea.io/antrea/pkg/apiserver/certificate" "antrea.io/antrea/pkg/apiserver/openapi" "antrea.io/antrea/pkg/apiserver/storage" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/clusteridentity" + "antrea.io/antrea/pkg/controller/certificatesigningrequest" "antrea.io/antrea/pkg/controller/egress" egressstore "antrea.io/antrea/pkg/controller/egress/store" "antrea.io/antrea/pkg/controller/externalippool" @@ -176,6 +183,22 @@ func run(o *Options) error { ) } + var csrApprovingController *certificatesigningrequest.CSRApprovingController + var csrSigningController *certificatesigningrequest.IPsecCSRSigningController + var csrInformer cache.SharedIndexInformer + var csrLister csrlisters.CertificateSigningRequestLister + if features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + csrInformer = csrinformers.NewFilteredCertificateSigningRequestInformer(client, 0, nil, func(listOptions *metav1.ListOptions) { + listOptions.FieldSelector = fields.OneTermEqualSelector("spec.signerName", antreaapis.AntreaIPsecCSRSignerName).String() + }) + csrLister = csrlisters.NewCertificateSigningRequestLister(csrInformer.GetIndexer()) + + if *o.config.IPsecCSRSignerConfig.AutoApprove { + csrApprovingController = certificatesigningrequest.NewCSRApprovingController(client, csrInformer, csrLister) + } + csrSigningController = certificatesigningrequest.NewIPsecCSRSigningController(client, csrInformer, csrLister, *o.config.IPsecCSRSignerConfig.SelfSignedCA) + } + if features.DefaultFeatureGate.Enabled(features.Egress) { egressController = egress.NewEgressController(crdClient, groupEntityIndex, egressInformer, externalIPPoolController, egressGroupStore) } @@ -319,6 +342,14 @@ func run(o *Options) error { go antreaIPAMController.Run(stopCh) } + if features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + go csrInformer.Run(stopCh) + if *o.config.IPsecCSRSignerConfig.AutoApprove { + go csrApprovingController.Run(stopCh) + } + go csrSigningController.Run(stopCh) + } + <-stopCh klog.Info("Stopping Antrea controller") return nil diff --git a/cmd/antrea-controller/options.go b/cmd/antrea-controller/options.go index f10fa9cada6..cf5034bef6b 100644 --- a/cmd/antrea-controller/options.go +++ b/cmd/antrea-controller/options.go @@ -161,12 +161,10 @@ func (o *Options) setDefaults() { o.config.APIPort = apis.AntreaControllerAPIPort } if o.config.EnablePrometheusMetrics == nil { - o.config.EnablePrometheusMetrics = new(bool) - *o.config.EnablePrometheusMetrics = true + o.config.EnablePrometheusMetrics = ptrBool(true) } if o.config.SelfSignedCert == nil { - o.config.SelfSignedCert = new(bool) - *o.config.SelfSignedCert = true + o.config.SelfSignedCert = ptrBool(true) } if o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 = ipamIPv4MaskDefault @@ -175,4 +173,14 @@ func (o *Options) setDefaults() { if o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 = ipamIPv6MaskDefault } + if o.config.IPsecCSRSignerConfig.SelfSignedCA == nil { + o.config.IPsecCSRSignerConfig.SelfSignedCA = ptrBool(true) + } + if o.config.IPsecCSRSignerConfig.AutoApprove == nil { + o.config.IPsecCSRSignerConfig.AutoApprove = ptrBool(true) + } +} + +func ptrBool(value bool) *bool { + return &value } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index c37fabd3e1b..3914ed2acbd 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -74,6 +74,10 @@ var ( getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName ) +// otherConfigKeysForIPsecCertificates are configurations added to OVS bridge when AuthenticationMode is "cert" and +// need to be deleted when changing to "psk". +var otherConfigKeysForIPsecCertificates = []string{"certificate", "private_key", "ca_cert", "remote_cert", "remote_name"} + // Initializer knows how to setup host networking, OpenVSwitch, and Openflow. type Initializer struct { client clientset.Interface @@ -348,17 +352,56 @@ func (i *Initializer) Initialize() error { } // initializeWireGuard must be executed after setupOVSBridge as it requires gateway addresses on the OVS bridge. - switch i.networkConfig.TrafficEncryptionMode { - case config.TrafficEncryptionModeIPSec: - if err := i.initializeIPSec(); err != nil { + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeWireGuard { + if err := i.initializeWireGuard(); err != nil { return err } - case config.TrafficEncryptionModeWireGuard: - if err := i.initializeWireGuard(); err != nil { + } + // TODO: clean up WireGuard related configurations. + + // Initialize for IPsec PSK mode. + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + i.networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModePSK { + if err := i.waitForIPsecMonitorDaemon(); err != nil { + return err + } + if err := i.readIPSecPSK(); err != nil { return err } } + // Initialize for IPsec Certificate mode. + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + i.networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + if err := i.waitForIPsecMonitorDaemon(); err != nil { + return err + } + } else { + configs, err := i.ovsBridgeClient.GetOVSOtherConfig() + if err != nil { + return fmt.Errorf("failed to get OVS other configs: %w", err) + } + // Clean up certificate and private key files. + if configs["certificate"] != "" { + if err := os.Remove(configs["certificate"]); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete unused IPsec certificate", "file", configs["certificate"]) + } + } + if configs["private_key"] != "" { + if err := os.Remove(configs["private_key"]); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete unused IPsec private key", "file", configs["private_key"]) + } + } + toDelete := make(map[string]interface{}) + for _, key := range otherConfigKeysForIPsecCertificates { + toDelete[key] = "" + } + // Clean up stale configs in OVS database. + if err := i.ovsBridgeClient.DeleteOVSOtherConfig(toDelete); err != nil { + return fmt.Errorf("failed to clean up OVS other configs: %w", err) + } + } + wg.Add(1) // routeClient.Initialize() should be after i.setupOVSBridge() which // creates the host gateway interface. @@ -707,7 +750,7 @@ func (i *Initializer) setupDefaultTunnelInterface() error { externalIDs := map[string]interface{}{ interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaTunnel, } - tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", externalIDs) + tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", "", externalIDs) if err != nil { klog.Errorf("Failed to create tunnel port %s type %s on OVS bridge: %v", tunnelPortName, i.networkConfig.TunnelType, err) return err @@ -884,8 +927,8 @@ func (i *Initializer) initNodeLocalConfig() error { return nil } -// initializeIPSec checks if preconditions are met for using IPsec and reads the IPsec PSK value. -func (i *Initializer) initializeIPSec() error { +// waitForIPsecMonitorDaemon checks if preconditions are met for using IPsec. +func (i *Initializer) waitForIPsecMonitorDaemon() error { // At the time the agent is initialized and this code is executed, the // OVS daemons are already running given that we have successfully // connected to OVSDB. Given that the start_ovs script deletes existing @@ -908,10 +951,6 @@ func (i *Initializer) initializeIPSec() error { return fmt.Errorf("IPsec was requested, but the OVS IPsec monitor does not seem to be running") } } - - if err := i.readIPSecPSK(); err != nil { - return err - } return nil } @@ -929,13 +968,13 @@ func (i *Initializer) initializeWireGuard() error { // readIPSecPSK reads the IPsec PSK value from environment variable ANTREA_IPSEC_PSK func (i *Initializer) readIPSecPSK() error { - i.networkConfig.IPSecPSK = os.Getenv(ipsecPSKEnvKey) - if i.networkConfig.IPSecPSK == "" { + i.networkConfig.IPsecConfig.PSK = os.Getenv(ipsecPSKEnvKey) + if i.networkConfig.IPsecConfig.PSK == "" { return fmt.Errorf("IPsec PSK environment variable '%s' is not set or is empty", ipsecPSKEnvKey) } // Usually one does not want to log the secret data. - klog.V(4).Infof("IPsec PSK value: %s", i.networkConfig.IPSecPSK) + klog.V(4).Infof("IPsec PSK value: %s", i.networkConfig.IPsecConfig.PSK) return nil } diff --git a/pkg/agent/config/ipsec_authentication_mode.go b/pkg/agent/config/ipsec_authentication_mode.go new file mode 100644 index 00000000000..6c1ad60cb2c --- /dev/null +++ b/pkg/agent/config/ipsec_authentication_mode.go @@ -0,0 +1,53 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "strings" + +type IPsecAuthenticationMode int + +const ( + IPsecAuthenticationModePSK IPsecAuthenticationMode = iota + IPsecAuthenticationModeCert + IPsecAuthenticationModeInvalid = -1 +) + +var supportedIPsecAuthenticationModeStrs = [...]string{ + "psk", + "cert", +} + +func GetIPsecConfigModes() []IPsecAuthenticationMode { + return []IPsecAuthenticationMode{ + IPsecAuthenticationModePSK, + IPsecAuthenticationModeCert, + } +} + +// String returns value in string. +func (am IPsecAuthenticationMode) String() string { + return supportedIPsecAuthenticationModeStrs[am] +} + +// GetIPsecAuthenticationModeFromStr returns true and IPsecAuthenticationModeType corresponding to input string. +// Otherwise, false and undefined value is returned +func GetIPsecAuthenticationModeFromStr(str string) (bool, IPsecAuthenticationMode) { + for idx, ms := range supportedIPsecAuthenticationModeStrs { + if strings.EqualFold(ms, str) { + return true, IPsecAuthenticationMode(idx) + } + } + return false, IPsecAuthenticationModeInvalid +} diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 5b60993e1b7..2de112bb53c 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -146,12 +146,18 @@ func (n *NodeConfig) String() string { n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig) } +// IPsecConfig includes IPsec related configurations. +type IPsecConfig struct { + AuthenticationMode IPsecAuthenticationMode + PSK string +} + // NetworkConfig includes user provided network configuration parameters. type NetworkConfig struct { TrafficEncapMode TrafficEncapModeType TunnelType ovsconfig.TunnelType TrafficEncryptionMode TrafficEncryptionModeType - IPSecPSK string + IPsecConfig IPsecConfig TransportIface string TransportIfaceCIDRs []string IPv4Enabled bool diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go new file mode 100644 index 00000000000..cca760beaac --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go @@ -0,0 +1,482 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync/atomic" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + certutil "k8s.io/client-go/util/cert" + csrutil "k8s.io/client-go/util/certificate/csr" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/ovs/ovsconfig" +) + +const ( + controllerName = "AntreaAgentIPsecCertificateController" + workerItemKey = "key" + minRetryDelay = 5 * time.Second + maxRetryDelay = 60 * time.Second + + // the mount path for CA certificate in antrea-ipsec container. + // StrongSwan will never reads CA certificates from folders other than `/etc/ipsec.d/cacerts`. + // Though StrongSwan will automatically load CA certificates from the folder, we set the ca_path in other_configs + // to the correct path for better consistency. + caCertificatePath = "/etc/ipsec.d/cacerts/ca.crt" + + ovsConfigCACertificateKey = "ca_cert" + ovsConfigPrivateKeyKey = "private_key" + ovsConfigCertificateKey = "certificate" +) + +// certificateWaitTimeout controls the amount of time we wait for certificate +// approval in one iteration. +var certificateWaitTimeout = 15 * time.Minute +var defaultCertificatesPath = "/var/run/openvswitch" + +// Controller is responsible for requesting certificates by CertificateSigningRequest and configure them to OVS +type Controller struct { + kubeClient clientset.Interface + ovsBridgeClient ovsconfig.OVSBridgeClient + nodeName string + queue workqueue.RateLimitingInterface + + rotateCertificate func() (*certificateKeyPair, error) + certificateKeyPair *certificateKeyPair + + // caPath and is initialized with NewIPSecCertificateController and should not + // be changed once Controller starts. + caPath string + // certificateFolderPath is the folder to store private keys and issued certificates. + // defaults to defaultCertificatesPath. + certificateFolderPath string + + syncedOnce uint32 +} + +// Manager is an interface to track the status of the IPsec certificate controller. +type Manager interface { + HasSynced() bool +} + +var _ Manager = (*Controller)(nil) + +func NewIPSecCertificateController( + kubeClient clientset.Interface, + ovsBridgeClient ovsconfig.OVSBridgeClient, + nodeName string, +) *Controller { + controller := &Controller{ + kubeClient: kubeClient, + ovsBridgeClient: ovsBridgeClient, + nodeName: nodeName, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "IPsecCertificateController"), + caPath: filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), + certificateFolderPath: defaultCertificatesPath, + } + controller.rotateCertificate = controller.newCertificateKeyPair + return controller +} + +// worker is a long-running function that will continually call the processNextWorkItem function in +// order to read and process a message on the workqueue. +func (c *Controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *Controller) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.ErrorS(nil, "Unexpected object in work queue", "object", obj) + return true + } else if err := c.syncConfigurations(); err == nil { + c.queue.Forget(key) + } else { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing IPSec certificates, requeuing") + } + return true +} + +type certificateKeyPair struct { + caCertificate []*x509.Certificate + certificate []*x509.Certificate + privateKey crypto.Signer + certificatePath string + privateKeyPath string + rotationDeadline time.Time +} + +func (pair *certificateKeyPair) validate() error { + if pair == nil { + return fmt.Errorf("certificate and key pair is nil") + } + if len(pair.caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + if len(pair.certificate) == 0 { + return fmt.Errorf("certificate is empty") + } + if pair.privateKey == nil { + return fmt.Errorf("private key is empty") + } + roots := x509.NewCertPool() + for _, r := range pair.caCertificate { + roots.AddCert(r) + } + certificate := pair.certificate[0] + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + if _, err := certificate.Verify(verifyOptions); err != nil { + return err + } + switch pub := certificate.PublicKey.(type) { + //TODO: support key types other than RSA such as *ecdsa.PublicKey. + case *rsa.PublicKey: + priv, ok := pair.privateKey.(*rsa.PrivateKey) + if !ok { + return fmt.Errorf("private key type does not match public key type") + } + if pub.N.Cmp(priv.N) != 0 { + return fmt.Errorf("private key does not match public key") + } + default: + return fmt.Errorf("unrecognized certificate public key type") + } + return nil +} + +// cleanup deletes the files of certificate and private key. +func (pair *certificateKeyPair) cleanup() { + if pair.certificatePath != "" { + // Delete the old certificate file. + if err := os.Remove(pair.certificatePath); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete old certificate", "file", pair.certificatePath) + } + } + if pair.privateKeyPath != "" { + // Delete the old private key file. + if err := os.Remove(pair.privateKeyPath); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete old private key", "file", pair.privateKeyPath) + } + } +} + +// jitteryDuration returns a duration in [totalDuration * 0.7, totalDuration * 0.9]. +var jitteryDuration = func(totalDuration time.Duration) time.Duration { + // wait.Jitter returns a duration in [totalDuration, totalDuration * 1.2]. + return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(float64(totalDuration)*0.3) +} + +// nextRotationDeadline returns a value for the threshold at which the +// current certificate should be rotated, 80%+/-10% of the expiration of the +// certificate. The deadline will not change once calculated. +// This function is not thread-safe. +func (pair *certificateKeyPair) nextRotationDeadline() time.Time { + // Return the previous calculated rotation deadline if applicable. + if !pair.rotationDeadline.IsZero() { + return pair.rotationDeadline + } + notAfter := pair.certificate[0].NotAfter + totalDuration := notAfter.Sub(pair.certificate[0].NotBefore) + deadline := pair.certificate[0].NotBefore.Add(jitteryDuration(totalDuration)) + klog.InfoS("Calculated certificate rotation deadline", "expiration", notAfter, "deadline", deadline) + pair.rotationDeadline = deadline + return deadline +} + +func loadCertAndKeyFromFiles(caPath, certPath, keyPath string) (*certificateKeyPair, error) { + ca, err := loadRootCA(caPath) + if err != nil { + return nil, err + } + key, err := loadPrivateKey(keyPath) + if err != nil { + return nil, err + } + cert, err := loadCertificate(certPath) + if err != nil { + return nil, err + } + pair := &certificateKeyPair{ + certificatePath: certPath, + privateKeyPath: keyPath, + caCertificate: ca, + certificate: cert, + privateKey: key, + } + return pair, nil +} + +func (c *Controller) syncConfigurations() error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing IPsec certificate configurations", "duration", d) + }() + + var deadline time.Time + // Validate the existing certificate and key pair. + if err := c.certificateKeyPair.validate(); err != nil { + klog.ErrorS(err, "Verifying current certificate configurations failed") + deadline = time.Now() + } else { + deadline = c.certificateKeyPair.nextRotationDeadline() + } + // Current certificate is about to expire. + if sleepInterval := time.Until(deadline); sleepInterval <= 0 { + klog.InfoS("Start rotating IPsec certificate") + newCertKeyPair, err := c.rotateCertificate() + if err != nil { + return fmt.Errorf("failed to rotate certificate: %w", err) + } + if err := newCertKeyPair.validate(); err != nil { + newCertKeyPair.cleanup() + return fmt.Errorf("failed to validate new certificate: %w", err) + } + // Clean up old certificate and key pair. + if c.certificateKeyPair != nil { + c.certificateKeyPair.cleanup() + } + // Save the known good certificate and key pair. + c.certificateKeyPair = newCertKeyPair + // Calculate the rotation deadline of new certificate. + deadline = c.certificateKeyPair.nextRotationDeadline() + } + // Re-queue after the interval to renew the certificate. + c.queue.AddAfter(workerItemKey, time.Until(deadline)) + // Sync OVS bridge configurations. + if err := c.syncOVSConfigurations(c.certificateKeyPair.certificatePath, + c.certificateKeyPair.privateKeyPath, caCertificatePath); err != nil { + return err + } + atomic.StoreUint32(&c.syncedOnce, 1) + return nil +} + +// SyncedOnce returns true if the controller has configured certificate successfully +// at least once. +func (c *Controller) SyncedOnce() bool { + return atomic.LoadUint32(&c.syncedOnce) == 1 +} + +// HasSynced implements the Manager interface. +func (c *Controller) HasSynced() bool { + if c == nil { + return true + } + return c.SyncedOnce() +} + +func loadRootCA(caPath string) ([]*x509.Certificate, error) { + pemBlock, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, err + } + certs, err := certutil.ParseCertsPEM(pemBlock) + if err != nil { + return nil, fmt.Errorf("error reading root CA %s: %w", caPath, err) + } + return certs, nil +} + +func newRSAPrivateKey() (crypto.Signer, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate new RSA private key: %v", err) + } + bs, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal private key: %v", err) + } + return key, bs, nil +} + +func loadPrivateKey(privateKeyPath string) (crypto.Signer, error) { + var keyPEMBytes []byte + _, err := os.Stat(privateKeyPath) + if err == nil { + // Load the private key contents from file. + keyPEMBytes, err = ioutil.ReadFile(privateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read key file %s: %v", privateKeyPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to stat key file %s: %v", privateKeyPath, err) + } + if len(keyPEMBytes) > 0 { + // Try to parse private key from existing file. + parsed, err := keyutil.ParsePrivateKeyPEM(keyPEMBytes) + privateKey, ok := parsed.(crypto.Signer) + if err != nil || !ok { + klog.ErrorS(err, "Parse key from file error", "file", privateKeyPath) + } else { + return privateKey, nil + } + } + return nil, nil +} + +func loadCertificate(certPath string) ([]*x509.Certificate, error) { + var certPEMBytes []byte + _, err := os.Stat(certPath) + if err == nil { + // Load the certificate from file. + certPEMBytes, err = ioutil.ReadFile(certPath) + if err != nil { + return nil, fmt.Errorf("failed to read certificate file %s: %w", certPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to stat certificate file %s: %w", certPath, err) + } + if len(certPEMBytes) > 0 { + // Try to parse the certificate from the existing file. + certificates, err := certutil.ParseCertsPEM(certPEMBytes) + if err != nil { + klog.ErrorS(err, "Parse certificate from file error", "file", certPath) + } else { + return certificates, nil + } + } + return nil, nil +} + +func (c *Controller) syncOVSConfigurations(certPath, keyPath, caPath string) error { + ovsConfig := map[string]interface{}{ + ovsConfigCertificateKey: certPath, + ovsConfigPrivateKeyKey: keyPath, + ovsConfigCACertificateKey: caPath, + } + klog.InfoS("Updating OVS configurations for IPsec certificates") + return c.ovsBridgeClient.UpdateOVSOtherConfig(ovsConfig) +} + +func newCSR(csrNamePrefix, commonName string, privateKey crypto.Signer) (*certificatesv1.CertificateSigningRequest, error) { + subject := &pkix.Name{ + CommonName: commonName, + Organization: []string{antreaapis.AntreaOrganizationName}, + } + csrBytes, err := certutil.MakeCSR(privateKey, subject, []string{commonName}, nil) + if err != nil { + return nil, err + } + return &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: csrNamePrefix, + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: csrBytes, + SignerName: antreaapis.AntreaIPsecCSRSignerName, + Usages: []certificatesv1.KeyUsage{certificatesv1.UsageIPsecTunnel}, + }, + }, nil +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting " + controllerName) + defer klog.InfoS("Shutting down " + controllerName) + + // Load the previous configured certificate path from OVS database. + config, ovsErr := c.ovsBridgeClient.GetOVSOtherConfig() + if ovsErr != nil { + klog.ErrorS(ovsErr, "Failed to get OVS bridge other configs") + } + + certificatePath := config[ovsConfigCertificateKey] + privateKeyPath := config[ovsConfigPrivateKeyKey] + if certificatePath != "" && privateKeyPath != "" { + pair, err := loadCertAndKeyFromFiles(c.caPath, certificatePath, privateKeyPath) + if err != nil { + klog.ErrorS(err, "Failed to load IPsec certificate and private key from existing files", + "ca", c.caPath, "cert", certificatePath, "key", privateKeyPath) + } else { + c.certificateKeyPair = pair + } + } + + c.queue.Add(workerItemKey) + go wait.Until(c.worker, time.Second, stopCh) + <-stopCh +} + +func (c *Controller) newCertificateKeyPair() (*certificateKeyPair, error) { + key, rawKey, err := newRSAPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to generate new private key: %w", err) + } + // Always create a new CSR for certificate rotation. The old ones will be GCed automatically. + csrNamePrefix := fmt.Sprintf("%s-", c.nodeName) + csr, err := newCSR(csrNamePrefix, c.nodeName, key) + if err != nil { + return nil, err + } + csr, err = c.kubeClient.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), csr, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), certificateWaitTimeout) + defer cancel() + rawCert, err := csrutil.WaitForCertificate(ctx, c.kubeClient, csr.Name, csr.UID) + if err != nil { + return nil, err + } + // Use the hash of new certificate and key as the filename suffix. + hasher := sha256.New() + hasher.Write(rawCert) + hasher.Write(rawKey) + hash := hasher.Sum(nil) + certPath := filepath.Join(c.certificateFolderPath, fmt.Sprintf("%s-%x.crt", c.nodeName, hash[:5])) + keyPath := filepath.Join(c.certificateFolderPath, fmt.Sprintf("%s-%x.key", c.nodeName, hash[:5])) + if err := certutil.WriteCert(certPath, rawCert); err != nil { + return nil, err + } + if err := keyutil.WriteKey(keyPath, rawKey); err != nil { + return nil, err + } + klog.InfoS("Created new certificate and key for IPSec", "cert", certPath, "key", keyPath) + + return loadCertAndKeyFromFiles(c.caPath, certPath, keyPath) +} diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go new file mode 100644 index 00000000000..2128b838246 --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go @@ -0,0 +1,463 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + certutil "k8s.io/client-go/util/cert" + + ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" +) + +const fakeNodeName = "fake-node-1" + +type fakeController struct { + *Controller + mockController *gomock.Controller + mockBridgeClient *ovsconfigtest.MockOVSBridgeClient + rawCAcert []byte + caCert *x509.Certificate + caKey crypto.Signer +} + +func newFakeController(t *testing.T) *fakeController { + mockController := gomock.NewController(t) + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(mockController) + fakeClient := fake.NewSimpleClientset() + listCSRAction := k8stesting.NewRootListAction(certificatesv1.SchemeGroupVersion.WithResource("certificatesigningrequests"), certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest"), metav1.ListOptions{}) + + // add an reactor to fill the Name and UID in the Create request. + fakeClient.PrependReactor("create", "certificatesigningrequests", k8stesting.ReactionFunc( + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + csr := action.(k8stesting.CreateAction).GetObject().(*certificatesv1.CertificateSigningRequest) + if csr.ObjectMeta.GenerateName != "" { + csr.ObjectMeta.Name = fmt.Sprintf("%s%s", csr.ObjectMeta.GenerateName, utilrand.String(8)) + csr.ObjectMeta.GenerateName = "" + csr.UID = uuid.NewUUID() + csr.CreationTimestamp = metav1.Now() + } + return false, csr, nil + }), + ) + // add an reactor to honor the fieldsSelector in the List request. + fakeClient.PrependReactor("list", "certificatesigningrequests", func(action k8stesting.Action) (bool, runtime.Object, error) { + var csrList *certificatesv1.CertificateSigningRequestList + // list CSRs using the original reactors. + for _, reactor := range fakeClient.Fake.ReactionChain[1:] { + if !reactor.Handles(listCSRAction) { + continue + } + handled, ret, err := reactor.React(listCSRAction) + if !handled { + continue + } + if err != nil { + return false, nil, err + } + csrList = ret.(*certificatesv1.CertificateSigningRequestList) + } + actionList, ok := action.(k8stesting.ListActionImpl) + if !ok { + return true, nil, fmt.Errorf("unexpected action type, expected %T, got %T", k8stesting.ListActionImpl{}, action) + } + listFieldsSelector := actionList.GetListRestrictions().Fields + var filtered []certificatesv1.CertificateSigningRequest + for _, c := range csrList.Items { + csrSpecificFieldsSet := make(fields.Set) + csrSpecificFieldsSet["metadata.name"] = c.Name + if listFieldsSelector.Matches(csrSpecificFieldsSet) { + filtered = append(filtered, c) + } + } + return true, &certificatesv1.CertificateSigningRequestList{ + Items: filtered, + }, nil + }) + + originDefaultPath := defaultCertificatesPath + cfg := certutil.Config{ + CommonName: "antrea-ipsec-ca", + Organization: []string{"antrea.io"}, + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + rootCA, err := certutil.NewSelfSignedCACert(cfg, key) + require.NoError(t, err) + tempDir, err := ioutil.TempDir("", "antrea-ipsec-test") + require.NoError(t, err) + defaultCertificatesPath = tempDir + defer func() { + defaultCertificatesPath = originDefaultPath + }() + caData, err := certutil.EncodeCertificates(rootCA) + require.NoError(t, err) + err = certutil.WriteCert(filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), caData) + require.NoError(t, err) + + c := NewIPSecCertificateController(fakeClient, mockOVSBridgeClient, fakeNodeName) + return &fakeController{ + Controller: c, + mockController: mockController, + mockBridgeClient: mockOVSBridgeClient, + rawCAcert: caData, + caCert: rootCA, + caKey: key, + } +} + +func TestController_syncConfigurations(t *testing.T) { + t.Run("rotate certificate if current certificates are empty", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + return nil, fmt.Errorf("unable to rotate certificate") + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + assert.Nil(t, fakeController.certificateKeyPair) + <-ch + }) + t.Run("should not touch existing certificate if rotate certificate failed", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + fakeController.certificateKeyPair = &certificateKeyPair{ + certificatePath: "cert.crt", + privateKeyPath: "key.key", + } + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + return nil, fmt.Errorf("unable to rotate certificate") + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + assert.NotNil(t, fakeController.certificateKeyPair) + assert.Equal(t, "cert.crt", fakeController.certificateKeyPair.certificatePath) + assert.Equal(t, "key.key", fakeController.certificateKeyPair.privateKeyPath) + <-ch + }) + t.Run("should clean up new certificate if it is not valid", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + certPath := filepath.Join(fakeController.certificateFolderPath, "cert-1.crt") + keyPath := filepath.Join(fakeController.certificateFolderPath, "key-1.key") + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + require.NoError(t, ioutil.WriteFile(certPath, nil, 0600)) + require.NoError(t, ioutil.WriteFile(keyPath, nil, 0400)) + return &certificateKeyPair{ + certificatePath: certPath, + privateKeyPath: keyPath, + }, nil + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + _, err = os.Stat(certPath) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(keyPath) + assert.True(t, os.IsNotExist(err)) + <-ch + }) + t.Run("request and configure new certificates", func(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + assert.Equal(t, 0, fakeController.queue.Len()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // start a fake signer in background to sign CSRs. + go func() { + watcher, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for ev := range watcher.ResultChan() { + switch ev.Type { + case watch.Added: + csr, ok := ev.Object.(*certificatesv1.CertificateSigningRequest) + assert.True(t, ok) + signCSR(t, fakeController, csr, time.Hour*24) + } + } + }() + originalRotateCertificate := fakeController.rotateCertificate + newCertDst := filepath.Join(fakeController.certificateFolderPath, "newcert.crt") + newKeyDst := filepath.Join(fakeController.certificateFolderPath, "newkey.key") + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + pair, err := originalRotateCertificate() + assert.NoError(t, err) + os.Link(pair.certificatePath, newCertDst) + os.Link(pair.privateKeyPath, newKeyDst) + pair.certificatePath = newCertDst + pair.privateKeyPath = newKeyDst + return pair, nil + } + // should configure OVS properly in syncConfigurations() + expectedOVSConfig := map[string]interface{}{ + "certificate": newCertDst, + "private_key": newKeyDst, + "ca_cert": caCertificatePath, + } + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(expectedOVSConfig) + // syncConfigurations should not block and get signed certificates from CSR successfully. + err := fakeController.syncConfigurations() + assert.NoError(t, err) + list, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 1) + assert.NotEmpty(t, fakeController.caPath) + + rotationDeadline := fakeController.certificateKeyPair.nextRotationDeadline() + assert.False(t, rotationDeadline.IsZero()) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + t.Error("unexpected call rotateCertificate") + return nil, nil + } + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(expectedOVSConfig) + // syncConfigurations again should not request new certificates. + err = fakeController.syncConfigurations() + assert.NoError(t, err) + // rotation deadline should not be changed. + assert.Equal(t, fakeController.certificateKeyPair.nextRotationDeadline(), rotationDeadline) + }) +} + +func TestController_RotateCertificates(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + assert.Equal(t, 0, fakeController.queue.Len()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // start a fake signer in background to sign CSRs. + signCh := make(chan struct{}) + go func() { + defer close(signCh) + counter := 0 + watcher, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for ev := range watcher.ResultChan() { + switch ev.Type { + case watch.Added: + csr, ok := ev.Object.(*certificatesv1.CertificateSigningRequest) + assert.True(t, ok) + // issue a certificate with lifetime of 10 seconds. + signCSR(t, fakeController, csr, time.Second*10) + counter++ + if counter == 2 { + return + } + } + } + }() + fakeController.mockBridgeClient.EXPECT().GetOVSOtherConfig().Times(1) + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(gomock.Any()).MinTimes(1) + go fakeController.Run(ch) + // wait for the signer to finish signing two CSRs. + <-signCh + list, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{}) + assert.NoError(t, err) + assert.Len(t, list.Items, 2) + delta := list.Items[0].CreationTimestamp.Sub(list.Items[1].CreationTimestamp.Time) + if delta < 0 { + delta = -delta + } + // the rotation interval should be in [7s, 9s], but it takes time to process the CSR request, + // so add one second to the upper bound. + assert.Less(t, delta, time.Second*10) + assert.LessOrEqual(t, time.Second*7, delta) +} + +func newIPsecCertTemplate(t *testing.T, nodeName string, notBefore, notAfter time.Time) *x509.Certificate { + return &x509.Certificate{ + Subject: pkix.Name{ + CommonName: nodeName, + Organization: []string{"antrea.io"}, + }, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: notBefore, + NotAfter: notAfter, + SerialNumber: big.NewInt(12345), + DNSNames: []string{nodeName}, + BasicConstraintsValid: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } +} + +func createCertificate(t *testing.T, nodeName string, caCert *x509.Certificate, caKey crypto.Signer, + publicKey crypto.PublicKey, notBefore time.Time, expirationDuration time.Duration) []byte { + template := newIPsecCertTemplate(t, nodeName, notBefore, notBefore.Add(expirationDuration)) + derBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, publicKey, caKey) + require.NoError(t, err) + certs, err := x509.ParseCertificates(derBytes) + require.NoError(t, err) + assert.Len(t, certs, 1) + encoded, err := certutil.EncodeCertificates(certs...) + require.NoError(t, err) + return encoded +} + +func signCSR(t *testing.T, controller *fakeController, + csr *certificatesv1.CertificateSigningRequest, expirationDuration time.Duration) { + assert.Empty(t, csr.Status.Certificate) + // use the CreationTimestamp as the NotBefore field of issued certificates for testing. + assert.False(t, csr.CreationTimestamp.IsZero()) + block, remain := pem.Decode(csr.Spec.Request) + assert.Empty(t, remain) + req, err := x509.ParseCertificateRequest(block.Bytes) + assert.NoError(t, err) + newCert := createCertificate(t, req.Subject.CommonName, controller.caCert, + controller.caKey, req.PublicKey, csr.CreationTimestamp.Time, expirationDuration) + toUpdate := csr.DeepCopy() + toUpdate.Status.Conditions = append(toUpdate.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + toUpdate, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateApproval(context.TODO(), csr.Name, toUpdate, metav1.UpdateOptions{}) + assert.NoError(t, err) + + toUpdate = toUpdate.DeepCopy() + toUpdate.Status.Certificate = newCert + _, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}) + assert.NoError(t, err) +} + +func Test_jitteryDuration(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedLowerBound, expectedUpperBound time.Duration + }{ + { + name: "10 seconds", + duration: 10 * time.Second, + expectedLowerBound: 7 * time.Second, + expectedUpperBound: 9 * time.Second, + }, { + name: "10 minutes", + duration: 10 * time.Minute, + expectedLowerBound: 7 * time.Minute, + expectedUpperBound: 9 * time.Minute, + }, + { + name: "10 hours", + duration: 10 * time.Hour, + expectedLowerBound: 7 * time.Hour, + expectedUpperBound: 9 * time.Hour, + }, + { + name: "10 days", + duration: 10 * time.Hour * 24, + expectedLowerBound: 7 * time.Hour * 24, + expectedUpperBound: 9 * time.Hour * 24, + }, + { + name: "100 days", + duration: 100 * time.Hour * 24, + expectedLowerBound: 70 * time.Hour * 24, + expectedUpperBound: 90 * time.Hour * 24, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := jitteryDuration(tt.duration) + assert.LessOrEqual(t, tt.expectedLowerBound, d) + assert.LessOrEqual(t, d, tt.expectedUpperBound) + }) + } +} + +func Test_certificateKeyPair_nextRotationDeadline(t *testing.T) { + tests := []struct { + name string + notBefore, notAfter string + want string + }{ + { + "10 hours certificate should be rotated at notBefore + 8h", + "2022-05-20T00:00:00Z", + "2022-05-20T10:00:00Z", + "2022-05-20T08:00:00Z", + }, + { + "10 days certificate should be rotated at notBefore + 8d", + "2022-05-20T00:00:00Z", + "2022-05-30T00:00:00Z", + "2022-05-28T00:00:00Z", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before, err := time.Parse(time.RFC3339, tt.notBefore) + require.NoError(t, err) + after, err := time.Parse(time.RFC3339, tt.notAfter) + require.NoError(t, err) + pair := &certificateKeyPair{ + certificate: []*x509.Certificate{{ + NotBefore: before, + NotAfter: after, + }}, + } + originJitterDuration := jitteryDuration + jitteryDuration = func(totalDuration time.Duration) time.Duration { + return time.Duration(float64(totalDuration) * 0.8) + } + defer func() { + jitteryDuration = originJitterDuration + }() + deadline := pair.nextRotationDeadline() + expectedDeadline, err := time.Parse(time.RFC3339, tt.want) + require.NoError(t, err) + assert.Equal(t, expectedDeadline, deadline) + newDeadline := pair.nextRotationDeadline() + assert.Equal(t, deadline, newDeadline, "rotation deadline should not change") + }) + } +} diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index ece61e3c475..d96e49d486c 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -32,6 +32,7 @@ import ( "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/route" @@ -79,6 +80,10 @@ type Controller struct { installedNodes cache.Indexer wireGuardClient wireguard.Interface proxyAll bool + // ipsecCertificateManager is useful for determining whether the ipsec certificate has been configured + // or not when IPsec is enabled with "cert" mode. The NodeRouteController must wait for the certificate + // to be configured before installing routes/flows to peer Nodes to prevent unencrypted traffic across Nodes. + ipsecCertificateManager ipseccertificate.Manager } // NewNodeRouteController instantiates a new Controller object which will process Node events @@ -94,25 +99,27 @@ func NewNodeRouteController( nodeConfig *config.NodeConfig, wireguardClient wireguard.Interface, proxyAll bool, + ipsecCertificateManager ipseccertificate.Manager, ) *Controller { nodeInformer := informerFactory.Core().V1().Nodes() svcLister := informerFactory.Core().V1().Services() controller := &Controller{ - kubeClient: kubeClient, - ovsBridgeClient: ovsBridgeClient, - ofClient: client, - routeClient: routeClient, - interfaceStore: interfaceStore, - networkConfig: networkConfig, - nodeConfig: nodeConfig, - nodeInformer: nodeInformer, - nodeLister: nodeInformer.Lister(), - nodeListerSynced: nodeInformer.Informer().HasSynced, - svcLister: svcLister.Lister(), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), - installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), - wireGuardClient: wireguardClient, - proxyAll: proxyAll, + kubeClient: kubeClient, + ovsBridgeClient: ovsBridgeClient, + ofClient: client, + routeClient: routeClient, + interfaceStore: interfaceStore, + networkConfig: networkConfig, + nodeConfig: nodeConfig, + nodeInformer: nodeInformer, + nodeLister: nodeInformer.Lister(), + nodeListerSynced: nodeInformer.Informer().HasSynced, + svcLister: svcLister.Lister(), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), + installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), + wireGuardClient: wireguardClient, + proxyAll: proxyAll, + ipsecCertificateManager: ipsecCertificateManager, } nodeInformer.Informer().AddEventHandlerWithResyncPeriod( cache.ResourceEventHandlerFuncs{ @@ -252,10 +259,17 @@ func (c *Controller) removeStaleTunnelPorts() error { klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err) continue } - + var remoteName, psk string + // remote_name and psk are mutually exclusive. + switch c.networkConfig.IPsecConfig.AuthenticationMode { + case config.IPsecAuthenticationModeCert: + remoteName = node.Name + case config.IPsecAuthenticationModePSK: + psk = c.networkConfig.IPsecConfig.PSK + } ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name) ifaceName := util.GenerateNodeTunnelInterfaceName(node.Name) - if c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv4, ifaceName) || c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv6, ifaceName) { + if c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv4, psk, remoteName, ifaceName) || c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv6, psk, remoteName, ifaceName) { desiredInterfaces[ifaceID] = true } } @@ -289,9 +303,10 @@ func (c *Controller) removeStaleTunnelPorts() error { } func (c *Controller) compareInterfaceConfig(interfaceConfig *interfacestore.InterfaceConfig, - peerNodeIP net.IP, interfaceName string) bool { + peerNodeIP net.IP, psk, remoteName, interfaceName string) bool { return interfaceConfig.InterfaceName == interfaceName && - interfaceConfig.PSK == c.networkConfig.IPSecPSK && + interfaceConfig.PSK == psk && + interfaceConfig.RemoteName == remoteName && interfaceConfig.RemoteIP.Equal(peerNodeIP) && interfaceConfig.TunnelInterfaceConfig.Type == c.networkConfig.TunnelType } @@ -347,7 +362,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) { klog.Infof("Starting %s", controllerName) defer klog.Infof("Shutting down %s", controllerName) - if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.nodeListerSynced) { + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.nodeListerSynced, c.ipsecCertificateManager.HasSynced) { return } @@ -626,11 +641,20 @@ func getPodCIDRsOnNode(node *corev1.Node) []string { func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int32, error) { portName := util.GenerateNodeTunnelInterfaceName(nodeName) interfaceConfig, exists := c.interfaceStore.GetNodeTunnelInterface(nodeName) - // check if Node IP, PSK, or tunnel type changes. This can + + var remoteName, psk string + // remote_name and psk are mutually exclusive. + switch c.networkConfig.IPsecConfig.AuthenticationMode { + case config.IPsecAuthenticationModeCert: + remoteName = nodeName + case config.IPsecAuthenticationModePSK: + psk = c.networkConfig.IPsecConfig.PSK + } + // check if Node IP, PSK, remote name, or tunnel type changes. This can // happen if removeStaleTunnelPorts fails to remove a "stale" // tunnel port for which the configuration has changed, return error to requeue the Node. if exists { - if !c.compareInterfaceConfig(interfaceConfig, nodeIP, portName) { + if !c.compareInterfaceConfig(interfaceConfig, nodeIP, psk, remoteName, portName) { klog.InfoS("IPsec tunnel interface config doesn't match the cached one, deleting the stale IPsec tunnel port", "node", nodeName, "interface", interfaceConfig.InterfaceName) if err := c.ovsBridgeClient.DeletePort(interfaceConfig.PortUUID); err != nil { return 0, fmt.Errorf("fail to delete the stale IPsec tunnel port %s: %v", interfaceConfig.InterfaceName, err) @@ -653,7 +677,8 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 false, "", nodeIP.String(), - c.networkConfig.IPSecPSK, + remoteName, + psk, ovsExternalIDs) if err != nil { return 0, fmt.Errorf("failed to create IPsec tunnel port for Node %s", nodeName) @@ -666,7 +691,9 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 c.networkConfig.TunnelType, nodeName, nodeIP, - c.networkConfig.IPSecPSK) + psk, + remoteName, + ) interfaceConfig.OVSPortConfig = ovsPortConfig c.interfaceStore.AddInterface(interfaceConfig) } @@ -693,20 +720,22 @@ func ParseTunnelInterfaceConfig( klog.V(2).Infof("OVS port %s has no options", portData.Name) return nil } - remoteIP, localIP, psk, csum := ovsconfig.ParseTunnelInterfaceOptions(portData) + remoteIP, localIP, psk, remoteName, csum := ovsconfig.ParseTunnelInterfaceOptions(portData) var interfaceConfig *interfacestore.InterfaceConfig var nodeName string if portData.ExternalIDs != nil { nodeName = portData.ExternalIDs[ovsExternalIDNodeName] } - if psk != "" { + if psk != "" || remoteName != "" { interfaceConfig = interfacestore.NewIPSecTunnelInterface( portData.Name, ovsconfig.TunnelType(portData.IFType), nodeName, remoteIP, - psk) + psk, + remoteName, + ) } else { interfaceConfig = interfacestore.NewTunnelInterface(portData.Name, ovsconfig.TunnelType(portData.IFType), localIP, csum) } diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index 7932c66b05e..8009d61e572 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -60,6 +60,12 @@ type fakeController struct { interfaceStore interfacestore.InterfaceStore } +type fakeIPsecCertificateManager struct{} + +func (f *fakeIPsecCertificateManager) HasSynced() bool { + return true +} + func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeController, func()) { clientset := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(clientset, 12*time.Hour) @@ -68,11 +74,11 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont ovsClient := ovsconfigtest.NewMockOVSBridgeClient(ctrl) routeClient := routetest.NewMockInterface(ctrl) interfaceStore := interfacestore.NewInterfaceStore() - + ipsecCertificateManager := &fakeIPsecCertificateManager{} c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ IPv4: nil, MAC: gatewayMAC, - }}, nil, false) + }}, nil, false, ipsecCertificateManager) return &fakeController{ Controller: c, clientset: clientset, @@ -229,12 +235,15 @@ func TestIPInPodSubnets(t *testing.T) { assert.Equal(t, false, c.Controller.IPInPodSubnets(net.ParseIP("8.8.8.8"))) } -func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig) (*fakeController, func()) { +func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig, authenticationMode config.IPsecAuthenticationMode) (*fakeController, func()) { c, closeFn := newController(t, &config.NetworkConfig{ TrafficEncapMode: 0, TunnelType: ovsconfig.TunnelType("vxlan"), TrafficEncryptionMode: config.TrafficEncryptionModeIPSec, - IPSecPSK: "changeme", + IPsecConfig: config.IPsecConfig{ + PSK: "changeme", + AuthenticationMode: authenticationMode, + }, }) for _, i := range ifaces { c.interfaceStore.AddInterface(i) @@ -257,7 +266,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { PortUUID: "123", }, }, - }) + }, config.IPsecAuthenticationModePSK) defer closeFn() defer c.queue.ShutDown() @@ -290,7 +299,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { assert.NoError(t, err) } -func TestCreateIPSecTunnelPort(t *testing.T) { +func TestCreateIPSecTunnelPortPSK(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { Type: interfacestore.TunnelInterface, @@ -319,7 +328,7 @@ func TestCreateIPSecTunnelPort(t *testing.T) { OFPort: int32(5), }, }, - }) + }, config.IPsecAuthenticationModePSK) defer closeFn() defer c.queue.ShutDown() @@ -332,11 +341,11 @@ func TestCreateIPSecTunnelPort(t *testing.T) { node2PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-2") c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP1.String(), "changeme", + false, "", nodeIP1.String(), "", "changeme", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) c.ovsClient.EXPECT().CreateTunnelPortExt( node2PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP2.String(), "changeme", + false, "", nodeIP2.String(), "", "changeme", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2"}).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) c.ovsClient.EXPECT().GetOFPort(node2PortName, false).Return(int32(2), nil) @@ -381,3 +390,46 @@ func TestCreateIPSecTunnelPort(t *testing.T) { }) } } + +func TestCreateIPSecTunnelPortCert(t *testing.T) { + c, closeFn := setup(t, nil, config.IPsecAuthenticationModeCert) + + defer closeFn() + defer c.queue.ShutDown() + stopCh := make(chan struct{}) + defer close(stopCh) + c.informerFactory.Start(stopCh) + c.informerFactory.WaitForCacheSync(stopCh) + + node1PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1") + c.ovsClient.EXPECT().CreateTunnelPortExt( + node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), + false, "", nodeIP1.String(), "xyz-k8s-0-1", "", + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + + tests := []struct { + name string + nodeName string + peerNodeIP net.IP + wantErr bool + want int32 + }{ + { + name: "create new port", + nodeName: "xyz-k8s-0-1", + peerNodeIP: nodeIP1, + wantErr: false, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := c.createIPSecTunnelPort(tt.nodeName, tt.peerNodeIP) + hasErr := err != nil + assert.Equal(t, tt.wantErr, hasErr) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index a443dd6dd6d..dd4131398ba 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -68,7 +68,10 @@ type TunnelInterfaceConfig struct { LocalIP net.IP // IP address of the remote Node. RemoteIP net.IP - PSK string + // CommonName of the remote Name for certificate based authentication. + RemoteName string + // Pre-shard key for authentication. + PSK string // Whether options:csum is set for this tunnel interface. // If true, encapsulation header UDP checksums will be computed on outgoing packets. Csum bool @@ -144,8 +147,8 @@ func NewTunnelInterface(tunnelName string, tunnelType ovsconfig.TunnelType, loca // NewIPSecTunnelInterface creates InterfaceConfig for the IPsec tunnel to the // Node. -func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk string) *InterfaceConfig { - tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk} +func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk, remoteName string) *InterfaceConfig { + tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk, RemoteName: remoteName} return &InterfaceConfig{InterfaceName: interfaceName, Type: TunnelInterface, TunnelInterfaceConfig: tunnelConfig} } diff --git a/pkg/apis/certificates.go b/pkg/apis/certificates.go new file mode 100644 index 00000000000..ea699db1a8c --- /dev/null +++ b/pkg/apis/certificates.go @@ -0,0 +1,22 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apis + +const ( + // AntreaOrganizationName is the organization name of Antrea managed certificates. + AntreaOrganizationName = "antrea.io" + // AntreaIPsecCSRSignerName is the signer name for signing IPsec certificates for antrea-agents. + AntreaIPsecCSRSignerName = "antrea.io/antrea-agent-ipsec-tunnel" +) diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index 0c913d191f0..4604434c930 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -192,6 +192,8 @@ type AgentConfig struct { AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"` // Egress related configurations. Egress EgressConfig `yaml:"egress"` + // IPsec related configurations. + IPsec IPsecConfig `yaml:"ipsec"` } type AntreaProxyConfig struct { @@ -246,3 +248,10 @@ type MulticastConfig struct { type EgressConfig struct { ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"` } + +type IPsecConfig struct { + // The authentication mode of IPsec tunnel. It has the following options: + // - psk (default): Use pre-shared key (PSK) for IKE authentication. + // - cert: Use CA-signed certificates for IKE authentication. + AuthenticationMode string `yaml:"authenticationMode,omitempty"` +} diff --git a/pkg/config/controller/config.go b/pkg/config/controller/config.go index 643d7e476cb..b0004b7e5a8 100644 --- a/pkg/config/controller/config.go +++ b/pkg/config/controller/config.go @@ -65,4 +65,18 @@ type ControllerConfig struct { LegacyCRDMirroring *bool `yaml:"legacyCRDMirroring,omitempty"` // NodeIPAM Configuration NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"` + // IPsec CSR signer configuration + IPsecCSRSignerConfig IPsecCSRSignerConfig `yaml:"ipsecCSRSigner"` +} + +type IPsecCSRSignerConfig struct { + // Indicates whether to use auto-generated self-signed CA certificate. + // If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + // tls.crt: + // tls.key: + // Defaults to true. + SelfSignedCA *bool `yaml:"selfSignedCA,omitempty"` + // Antrea signer auto approve policy. + // Defaults to true. + AutoApprove *bool `yaml:"autoApprove,omitempty"` } diff --git a/pkg/controller/certificatesigningrequest/approving_controller.go b/pkg/controller/certificatesigningrequest/approving_controller.go new file mode 100644 index 00000000000..1f26a26a371 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approving_controller.go @@ -0,0 +1,169 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "fmt" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + csrlisters "k8s.io/client-go/listers/certificates/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" +) + +const ( + approvingControllerName = "CertificateSigningRequestApprovingController" +) + +type approver interface { + recognize(csr *certificatesv1.CertificateSigningRequest) bool + verify(csr *certificatesv1.CertificateSigningRequest) (bool, error) + name() string +} + +// CSRApprovingController is responsible for approving CertificateSigningRequests. +type CSRApprovingController struct { + client clientset.Interface + csrInformer cache.SharedIndexInformer + csrLister csrlisters.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + queue workqueue.RateLimitingInterface + approvers []approver +} + +// NewCSRApprovingController returns a new *CSRApprovingController. +func NewCSRApprovingController(client clientset.Interface, csrInformer cache.SharedIndexInformer, csrLister csrlisters.CertificateSigningRequestLister) *CSRApprovingController { + c := &CSRApprovingController{ + client: client, + csrInformer: csrInformer, + csrLister: csrLister, + csrListerSynced: csrInformer.HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + approvers: []approver{ + &ipsecCSRApprover{ + client: client, + }, + }, + } + csrInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + }, + resyncPeriod, + ) + return c +} + +// Run begins watching and syncing of the CSRApprovingController. +func (c *CSRApprovingController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting " + approvingControllerName) + defer klog.InfoS("Shutting down " + approvingControllerName) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced} + if !cache.WaitForNamedCacheSync(approvingControllerName, stopCh, cacheSyncs...) { + return + } + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +func (c *CSRApprovingController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *CSRApprovingController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + return + } + c.queue.Add(csr.Name) +} + +func (c *CSRApprovingController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing CertificateSigningRequest", "name", key, "duration", d) + }() + + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + // The spec will not be updated by antrea-agent once it is approved or denied. + if approved, denied := getCertApprovalCondition(&csr.Status); approved || denied { + return nil + } + for _, a := range c.approvers { + if a.recognize(csr) { + approved, err := a.verify(csr) + if err != nil { + return err + } + if approved { + toUpdate := csr.DeepCopy() + appendApprovalCondition(toUpdate, fmt.Sprintf("Automatically approved by %s", a.name())) + _, err = c.client.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.Background(), toUpdate.Name, toUpdate, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error updating approval for csr: %w", err) + } + return nil + } + } + } + return nil +} + +func appendApprovalCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: message, + }) +} + +func (c *CSRApprovingController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} diff --git a/pkg/controller/certificatesigningrequest/approving_controller_test.go b/pkg/controller/certificatesigningrequest/approving_controller_test.go new file mode 100644 index 00000000000..714a9712b6b --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approving_controller_test.go @@ -0,0 +1,195 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +type fakeApprover struct { + approverName string + recognized, verified bool +} + +var _ approver = (*fakeApprover)(nil) + +func (f *fakeApprover) recognize(_ *certificatesv1.CertificateSigningRequest) bool { + return f.recognized +} + +func (f *fakeApprover) verify(_ *certificatesv1.CertificateSigningRequest) (bool, error) { + return f.verified, nil +} + +func (f *fakeApprover) name() string { + return f.approverName +} + +func TestCSRApprovingController_syncCSR(t *testing.T) { + tests := []struct { + name string + approvers []approver + expectErr bool + expectApproved, expectedDenied bool + csrToSync *certificatesv1.CertificateSigningRequest + }{ + { + name: "recognized and approved", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "not approved by any approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: false, + verified: false, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: false, + verified: false, + }, + }, + expectErr: false, + expectApproved: false, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "recognized by both approver and approved by the second approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: false, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "recognized by and approved by the second approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: false, + verified: true, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "do not approve denied CSR", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: false, + expectedDenied: true, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + { + Type: certificatesv1.CertificateDenied, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tt.csrToSync) + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + stopCh := make(chan struct{}) + defer close(stopCh) + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + controller := NewCSRApprovingController(clientset, csrInformer.Informer(), csrInformer.Lister()) + controller.csrInformer = csrInformer.Informer() + controller.csrLister = csrInformer.Lister() + controller.approvers = tt.approvers + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + + err := controller.syncCSR(tt.csrToSync.Name) + assert.Equal(t, tt.expectErr, err != nil) + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csrToSync.Name, metav1.GetOptions{}) + require.NoError(t, err) + approved, denied := getCertApprovalCondition(&csr.Status) + assert.Equal(t, tt.expectApproved, approved) + assert.Equal(t, tt.expectedDenied, denied) + }) + } +} diff --git a/pkg/controller/certificatesigningrequest/common.go b/pkg/controller/certificatesigningrequest/common.go new file mode 100644 index 00000000000..9909863cb94 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/common.go @@ -0,0 +1,158 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "sort" + "time" + + certificates "k8s.io/api/certificates/v1" + sautil "k8s.io/apiserver/pkg/authentication/serviceaccount" + certutil "k8s.io/client-go/util/cert" + + antreaapis "antrea.io/antrea/pkg/apis" +) + +const ( + // Set resyncPeriod to 0 to disable resyncing. + resyncPeriod time.Duration = 0 + // How long to wait before retrying the processing of a CertificateSigningRequest change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing a CertificateSigningRequest change. + defaultWorkers = 2 +) + +var ( + errOrganizationNotAntrea = fmt.Errorf("subject organization is not %s", antreaapis.AntreaOrganizationName) + errDNSSANNotMatchCommonName = fmt.Errorf("DNS subjectAltNames do not match subject common name") + errEmailSANNotAllowed = fmt.Errorf("email subjectAltNames are not allowed") + errIPSANNotAllowed = fmt.Errorf("IP subjectAltNames are not allowed") + errURISANNotAllowed = fmt.Errorf("URI subjectAltNames are not allowed") + errCommonNameRequired = fmt.Errorf("subject common name is required") + errExtraFieldsRequired = fmt.Errorf("extra values must contain %q and %q", sautil.PodNameKey, sautil.PodUIDKey) + errPodUIDMismatch = fmt.Errorf("Pod UID does not match") + errPodNotOnNode = fmt.Errorf("Pod is not on requested Node") + errUserUnauthorized = fmt.Errorf("Unrecognized username") +) + +// isCertificateRequestApproved returns true if a certificate request has the +// "Approved" condition and no "Denied" conditions; false otherwise. +func isCertificateRequestApproved(csr *certificates.CertificateSigningRequest) bool { + approved, denied := getCertApprovalCondition(&csr.Status) + return approved && !denied +} + +func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != certutil.CertificateRequestBlockType { + err := fmt.Errorf("PEM block type must be %s", certutil.CertificateRequestBlockType) + return nil, err + } + return x509.ParseCertificateRequest(block.Bytes) +} + +type transientError struct { + error +} + +func getCertApprovalCondition(status *certificates.CertificateSigningRequestStatus) (bool, bool) { + var approved, denied bool + for _, c := range status.Conditions { + if c.Type == certificates.CertificateApproved { + approved = true + } + if c.Type == certificates.CertificateDenied { + denied = true + } + } + return approved, denied +} + +var keyUsageDict = map[certificates.KeyUsage]x509.KeyUsage{ + certificates.UsageSigning: x509.KeyUsageDigitalSignature, + certificates.UsageDigitalSignature: x509.KeyUsageDigitalSignature, + certificates.UsageContentCommitment: x509.KeyUsageContentCommitment, + certificates.UsageKeyEncipherment: x509.KeyUsageKeyEncipherment, + certificates.UsageKeyAgreement: x509.KeyUsageKeyAgreement, + certificates.UsageDataEncipherment: x509.KeyUsageDataEncipherment, + certificates.UsageCertSign: x509.KeyUsageCertSign, + certificates.UsageCRLSign: x509.KeyUsageCRLSign, + certificates.UsageEncipherOnly: x509.KeyUsageEncipherOnly, + certificates.UsageDecipherOnly: x509.KeyUsageDecipherOnly, +} + +var extKeyUsageDict = map[certificates.KeyUsage]x509.ExtKeyUsage{ + certificates.UsageAny: x509.ExtKeyUsageAny, + certificates.UsageServerAuth: x509.ExtKeyUsageServerAuth, + certificates.UsageClientAuth: x509.ExtKeyUsageClientAuth, + certificates.UsageCodeSigning: x509.ExtKeyUsageCodeSigning, + certificates.UsageEmailProtection: x509.ExtKeyUsageEmailProtection, + certificates.UsageSMIME: x509.ExtKeyUsageEmailProtection, + certificates.UsageIPsecEndSystem: x509.ExtKeyUsageIPSECEndSystem, + certificates.UsageIPsecTunnel: x509.ExtKeyUsageIPSECTunnel, + certificates.UsageIPsecUser: x509.ExtKeyUsageIPSECUser, + certificates.UsageTimestamping: x509.ExtKeyUsageTimeStamping, + certificates.UsageOCSPSigning: x509.ExtKeyUsageOCSPSigning, + certificates.UsageMicrosoftSGC: x509.ExtKeyUsageMicrosoftServerGatedCrypto, + certificates.UsageNetscapeSGC: x509.ExtKeyUsageNetscapeServerGatedCrypto, +} + +// keyUsagesFromStrings will translate a slice of usage strings from the +// certificates API ("pkg/apis/certificates".KeyUsage) to x509.KeyUsage and +// x509.ExtKeyUsage types. +func keyUsagesFromStrings(usages []certificates.KeyUsage) (x509.KeyUsage, []x509.ExtKeyUsage, error) { + var keyUsage x509.KeyUsage + var unrecognized []certificates.KeyUsage + extKeyUsages := make(map[x509.ExtKeyUsage]struct{}) + for _, usage := range usages { + if val, ok := keyUsageDict[usage]; ok { + keyUsage |= val + } else if val, ok := extKeyUsageDict[usage]; ok { + extKeyUsages[val] = struct{}{} + } else { + unrecognized = append(unrecognized, usage) + } + } + + var sorted sortedExtKeyUsage + for eku := range extKeyUsages { + sorted = append(sorted, eku) + } + sort.Sort(sorted) + + if len(unrecognized) > 0 { + return 0, nil, fmt.Errorf("unrecognized usage values: %q", unrecognized) + } + + return keyUsage, sorted, nil +} + +type sortedExtKeyUsage []x509.ExtKeyUsage + +func (s sortedExtKeyUsage) Len() int { + return len(s) +} + +func (s sortedExtKeyUsage) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedExtKeyUsage) Less(i, j int) bool { + return s[i] < s[j] +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go b/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go new file mode 100644 index 00000000000..0e909ff0c21 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go @@ -0,0 +1,155 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "fmt" + "reflect" + "strings" + + certificatesv1 "k8s.io/api/certificates/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + sautil "k8s.io/apiserver/pkg/authentication/serviceaccount" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/util/env" +) + +const ( + ipsecCSRApproverName = "AntreaIPsecCSRApprover" +) + +var ( + antreaAgentServiceAccountName = strings.Join([]string{ + "system", "serviceaccount", env.GetAntreaNamespace(), "antrea-agent", + }, ":") +) + +type ipsecCSRApprover struct { + client clientset.Interface +} + +var ipsecTunnelUsages = sets.NewString( + string(certificatesv1.UsageIPsecTunnel), +) + +var _ approver = (*ipsecCSRApprover)(nil) + +func (ic *ipsecCSRApprover) recognize(csr *certificatesv1.CertificateSigningRequest) bool { + return csr.Spec.SignerName == antreaapis.AntreaIPsecCSRSignerName +} + +func (ic *ipsecCSRApprover) verify(csr *certificatesv1.CertificateSigningRequest) (bool, error) { + var failedReasons []string + cr, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + return false, err + } + if err := ic.verifyCertificateRequest(cr, csr.Spec.Usages); err != nil { + if _, ok := err.(*transientError); ok { + return false, err + } else { + failedReasons = append(failedReasons, err.Error()) + } + } + if err := ic.verifyIdentity(cr.Subject.CommonName, csr); err != nil { + if _, ok := err.(*transientError); ok { + return false, err + } + failedReasons = append(failedReasons, err.Error()) + } + + if len(failedReasons) > 0 { + klog.InfoS("Verifing CertificateSigningRequest for IPsec failed", "reasons", failedReasons, "CSR", csr.Name) + return false, nil + } + return true, nil +} + +func (ic *ipsecCSRApprover) name() string { + return ipsecCSRApproverName +} + +func (ic *ipsecCSRApprover) verifyCertificateRequest(req *x509.CertificateRequest, usages []certificatesv1.KeyUsage) error { + if !reflect.DeepEqual(req.Subject.Organization, []string{antreaapis.AntreaOrganizationName}) { + return errOrganizationNotAntrea + } + if req.Subject.CommonName == "" { + return errCommonNameRequired + } + if len(req.URIs) > 0 { + return errURISANNotAllowed + } + if len(req.IPAddresses) > 0 { + return errIPSANNotAllowed + } + if len(req.EmailAddresses) > 0 { + return errEmailSANNotAllowed + } + if !reflect.DeepEqual([]string{req.Subject.CommonName}, req.DNSNames) { + return errDNSSANNotMatchCommonName + } + for _, u := range usages { + if !ipsecTunnelUsages.Has(string(u)) { + return fmt.Errorf("unsupported key usage: %v", u) + } + } + _, err := ic.client.CoreV1().Nodes().Get(context.TODO(), req.Subject.CommonName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("requested Node %s not found", req.Subject.CommonName) + } else if err != nil { + return &transientError{err} + } + return nil +} + +func (ic *ipsecCSRApprover) verifyIdentity(nodeName string, csr *certificatesv1.CertificateSigningRequest) error { + if csr.Spec.Username != antreaAgentServiceAccountName { + return errUserUnauthorized + } + podNameValues, podUIDValues := csr.Spec.Extra[sautil.PodNameKey], csr.Spec.Extra[sautil.PodUIDKey] + if len(podNameValues) == 0 && len(podUIDValues) == 0 { + klog.Warning("Could not determine Pod identity from CertificateSigningRequest.", + " Enable K8s BoundServiceAccountTokenVolume feature gate to provide maximum security.") + return nil + } + if len(podNameValues) == 0 || len(podUIDValues) == 0 { + return errExtraFieldsRequired + } + podName, podUID := podNameValues[0], podUIDValues[0] + if podName == "" || podUID == "" { + return errExtraFieldsRequired + } + pod, err := ic.client.CoreV1().Pods(env.GetAntreaNamespace()).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("Pod %s not found", podName) + } else if err != nil { + return &transientError{err} + } + if pod.ObjectMeta.UID != types.UID(podUID) { + return errPodUIDMismatch + } + if pod.Spec.NodeName != nodeName { + return errPodNotOnNode + } + return nil +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go b/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go new file mode 100644 index 00000000000..f4aaec4c854 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go @@ -0,0 +1,604 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "net" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func Test_validIPSecCSR(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + cr *x509.CertificateRequest + keyUsages []certificatesv1.KeyUsage + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: nil, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Organization missing", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errOrganizationNotAntrea, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "requested Node not found", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("requested Node worker-node-1 not found"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "DNS SAN not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + }, + expectedErr: errDNSSANNotMatchCommonName, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "key usages not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("unsupported key usage: client auth"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageClientAuth, + }, + }, + { + name: "IP SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + IPAddresses: []net.IP{net.ParseIP("1.2.3.4")}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errIPSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "URI SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + URIs: []*url.URL{ + {Host: "antrea.io"}, + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errURISANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Email SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + EmailAddresses: []string{"user@antrea.io"}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errEmailSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + err := ic.verifyCertificateRequest(tt.cr, tt.keyUsages) + if tt.expectedErr == nil { + assert.NoError(t, err, "validIPSecCSR should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "validIPSecCSR should return an error") + } + }) + } +} + +func Test_verifyIdentity(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + nodeName string + csr *certificatesv1.CertificateSigningRequest + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: nil, + }, + { + name: "invalid username", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:my-sa", + }, + }, + nodeName: "worker-node-1", + expectedErr: errUserUnauthorized, + }, + { + name: "Pod UID mismatch", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"7afec259-ba03-441d-adeb-be163da2da2c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodUIDMismatch, + }, + { + name: "extra fields missing", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{}, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: nil, + }, + { + name: "Pod is not on requested Node", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-2", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodNotOnNode, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + err := ic.verifyIdentity(tt.nodeName, tt.csr) + if tt.expectedErr == nil { + assert.NoError(t, err, "verifyPodOnNode should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "verifyPodOnNode should return an error") + } + }) + } +} + +func Test_ipsecCertificateApprover_recognize(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedResult bool + }{ + { + name: "valid IPsec CSR", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/antrea-agent-ipsec-tunnel", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + expectedResult: true, + }, + { + name: "Unknown signer name", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "k8s.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + expectedResult: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + recognized := ic.recognize(tt.csr) + assert.Equal(t, tt.expectedResult, recognized) + }) + } +} + +func x509CRtoPEM(t *testing.T, cr *x509.CertificateRequest) (crypto.PrivateKey, []byte) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + crDer, err := x509.CreateCertificateRequest(rand.Reader, cr, privateKey) + require.NoError(t, err) + csrPemBlock := &pem.Block{ + Type: certutil.CertificateRequestBlockType, + Bytes: crDer, + } + crBytes := pem.EncodeToMemory(csrPemBlock) + assert.NotEmpty(t, crBytes) + return privateKey, crBytes +} + +func Test_ipsecCertificateApprover_verify(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedError error + expectedApproved bool + }{ + { + name: "valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + }, + { + name: "IPsec CSR with unknown username", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "my-pod-1", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipsec-unknown-user", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"my-pod-1"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:user-1", + }, + }, + expectedApproved: false, + }, + { + name: "CSR missing ExtraValue", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{}, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objs := append(tt.objects, tt.csr) + client := fake.NewSimpleClientset(objs...) + ic := &ipsecCSRApprover{ + client: client, + } + approved, err := ic.verify(tt.csr) + if tt.expectedError != nil { + assert.EqualError(t, err, tt.expectedError.Error()) + } else { + assert.Equal(t, tt.expectedApproved, approved) + } + }) + } +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go new file mode 100644 index 00000000000..e6fc51bc254 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go @@ -0,0 +1,470 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "reflect" + "sync/atomic" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + corev1informers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + csrlister "k8s.io/client-go/listers/certificates/v1" + csrlisters "k8s.io/client-go/listers/certificates/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/util/env" +) + +const ( + ipsecRootCAName = "antrea-ipsec-ca" + ipsecCSRSigningControllerName = "IPsecCertificateSigningRequestSigningController" + workerItemKey = "key" + rootCACertKey = "ca.crt" + + duration365d = time.Hour * 24 * 365 + duration10y = duration365d * 10 +) + +// IPsecCSRSigningController is responsible for signing CertificateSigningRequests. +type IPsecCSRSigningController struct { + client clientset.Interface + csrInformer cache.SharedIndexInformer + csrLister csrlister.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + + configMapInformer cache.SharedIndexInformer + configMapLister corev1listers.ConfigMapLister + configMapListerSynced cache.InformerSynced + + selfSignedCA bool + + // saved CertificateAuthority + certificateAuthority atomic.Value + + queue workqueue.RateLimitingInterface + fixturesQueue workqueue.RateLimitingInterface +} + +// certificateAuthority implements a certificate authority and used by the signing controller. +type certificateAuthority struct { + // RawCert is an optional field to determine if signing cert/key pairs have changed + RawCert []byte + // RawKey is an optional field to determine if signing cert/key pairs have changed + RawKey []byte + + Certificate *x509.Certificate + PrivateKey crypto.Signer +} + +func (c *certificateAuthority) signCSR(template *x509.Certificate, requestKey crypto.PublicKey) (*x509.Certificate, error) { + if len(c.RawCert) == 0 || len(c.RawKey) == 0 { + return nil, fmt.Errorf("certificate authority is not valid") + } + derBytes, err := x509.CreateCertificate(rand.Reader, template, c.Certificate, requestKey, c.PrivateKey) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(derBytes) + if err != nil { + return nil, err + } + if len(certs) != 1 { + return nil, fmt.Errorf("expect a single certificate, got %d", len(certs)) + } + return certs[0], nil +} + +// NewIPsecCSRSigningController returns a new *IPsecCSRSigningController. +func NewIPsecCSRSigningController(client clientset.Interface, csrInformer cache.SharedIndexInformer, csrLister csrlisters.CertificateSigningRequestLister, selfSignedCA bool) *IPsecCSRSigningController { + + caConfigMapInformer := corev1informers.NewFilteredConfigMapInformer(client, env.GetAntreaNamespace(), resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *metav1.ListOptions) { + listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", ipsecRootCAName).String() + }) + + configMapLister := corev1listers.NewConfigMapLister(caConfigMapInformer.GetIndexer()) + + c := &IPsecCSRSigningController{ + client: client, + csrInformer: csrInformer, + csrLister: csrLister, + csrListerSynced: csrInformer.HasSynced, + configMapInformer: caConfigMapInformer, + configMapLister: configMapLister, + configMapListerSynced: caConfigMapInformer.HasSynced, + selfSignedCA: selfSignedCA, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + fixturesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + } + + csrInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + UpdateFunc: func(old, cur interface{}) { + c.enqueueCertificateSigningRequest(cur) + }, + }, + resyncPeriod, + ) + + caConfigMapInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + UpdateFunc: func(old, cur interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + DeleteFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + }, + resyncPeriod, + ) + + return c +} + +// Run begins watching and syncing of the IPsecCSRSigningController. +func (c *IPsecCSRSigningController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.Infof("Starting %s", ipsecCSRSigningControllerName) + defer klog.Infof("Shutting down %s", ipsecCSRSigningControllerName) + + go c.configMapInformer.Run(stopCh) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced, c.configMapListerSynced} + if !cache.WaitForNamedCacheSync(ipsecCSRSigningControllerName, stopCh, cacheSyncs...) { + return + } + c.fixturesQueue.Add(workerItemKey) + + go wait.Until(c.fixturesWorker, time.Second, stopCh) + + go wait.NonSlidingUntil(func() { + if err := c.watchSecretChanges(stopCh); err != nil { + klog.ErrorS(err, "Watch Secret error", "secret", ipsecRootCAName) + } + }, time.Second*10, stopCh) + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.csrWorker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *IPsecCSRSigningController) syncRootCertificateAndKey() error { + var caBytes, caKeyBytes []byte + caSecret, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Get(context.TODO(), ipsecRootCAName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + if !c.selfSignedCA { + klog.InfoS("Self-signed CA is disabled. Ensure CA Secret exists", "name", ipsecRootCAName, "namespace", env.GetAntreaNamespace()) + return nil + } + caBytes, caKeyBytes, err = generateSelfSignedRootCertificate(ipsecRootCAName) + if err != nil { + return err + } + caSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: ipsecRootCAName, + Namespace: env.GetAntreaNamespace(), + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: caBytes, + corev1.TLSPrivateKeyKey: caKeyBytes, + }, + } + caSecret, err = c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Create(context.TODO(), caSecret, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.Info("Created Secret for self-signed IPsec root CA") + } + caCertificate, err := certutil.ParseCertsPEM(caSecret.Data[corev1.TLSCertKey]) + if err != nil { + return err + } + if len(caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + privateKey, err := keyutil.ParsePrivateKeyPEM(caSecret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return err + } + priv, ok := privateKey.(crypto.Signer) + if !ok { + return fmt.Errorf("error reading CA: key did not implement crypto.Signer") + } + ca := &certificateAuthority{ + RawCert: caSecret.Data[corev1.TLSCertKey], + RawKey: caSecret.Data[corev1.TLSPrivateKeyKey], + Certificate: caCertificate[0], + PrivateKey: priv, + } + c.certificateAuthority.Store(ca) + desiredConfigMapData := map[string]string{ + rootCACertKey: string(caSecret.Data[corev1.TLSCertKey]), + } + caConfigMap, err := c.configMapLister.ConfigMaps(env.GetAntreaNamespace()).Get(ipsecRootCAName) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + caConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ipsecRootCAName, + Namespace: env.GetAntreaNamespace(), + }, + Data: desiredConfigMapData, + } + caConfigMap, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Create(context.TODO(), caConfigMap, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.InfoS("Created ConfigMap for self-signed IPsec root CA") + } + if !reflect.DeepEqual(desiredConfigMapData, caConfigMap.Data) { + toUpdate := caConfigMap.DeepCopy() + toUpdate.Data = desiredConfigMapData + _, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Update(context.TODO(), toUpdate, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (c *IPsecCSRSigningController) csrWorker() { + for c.processNextWorkItem() { + } +} + +// watchSecretChanges uses watch API directly to watch for Secret changes. +// Antrea Controller should not have List permission for Secrets. +func (c *IPsecCSRSigningController) watchSecretChanges(endCh <-chan struct{}) error { + watcher, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Watch(context.TODO(), metav1.SingleObject(metav1.ObjectMeta{ + Namespace: env.GetAntreaNamespace(), + Name: ipsecRootCAName, + })) + if err != nil { + return fmt.Errorf("failed to create Secret watcher: %v", err) + } + // re-queue in case of missing events before watcher starts. + c.fixturesQueue.Add(workerItemKey) + ch := watcher.ResultChan() + defer watcher.Stop() + for { + select { + case _, ok := <-ch: + if !ok { + return nil + } + // we do not care the actual Event. + c.fixturesQueue.Add(workerItemKey) + case <-endCh: + return nil + } + } +} + +func (c *IPsecCSRSigningController) fixturesWorker() { + for c.processNextFixtureWorkItem() { + } +} + +func (c *IPsecCSRSigningController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + return + } + c.queue.Add(csr.Name) +} + +func (c *IPsecCSRSigningController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing CertificateSigningRequest", "name", key, "duration", d) + }() + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if csr.Spec.SignerName != antreaapis.AntreaIPsecCSRSignerName { + return nil + } + if len(csr.Status.Certificate) != 0 { + klog.V(2).InfoS("CertificateSigningRequest is already signed", "CertificateSigningRequest", csr.Name) + return nil + } + if !isCertificateRequestApproved(csr) { + klog.V(2).InfoS("CertificateSigningRequest is not approved", "CertificateSigningRequest", csr.Name) + return nil + } + req, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + klog.ErrorS(err, "Failed to decode CertificateSigningRequest", "CertificateSigningRequest", csr.Name) + return nil + } + template, err := newCertificateTemplate(req, csr.Spec.Usages) + if err != nil { + return err + } + currCA, ok := c.certificateAuthority.Load().(*certificateAuthority) + if !ok || currCA == nil { + return fmt.Errorf("certificate authority is not initialized") + } + signed, err := currCA.signCSR(template, req.PublicKey) + if err != nil { + return err + } + bs, err := certutil.EncodeCertificates(signed) + if err != nil { + return err + } + toUpdate := csr.DeepCopy() + toUpdate.Status.Certificate = bs + _, err = c.client.CertificatesV1().CertificateSigningRequests().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil +} + +func newCertificateTemplate(certReq *x509.CertificateRequest, usage []certificatesv1.KeyUsage) (*x509.Certificate, error) { + var sn big.Int + snBytes := make([]byte, 18) + _, err := rand.Read(snBytes) + if err != nil { + return nil, err + } + sn.SetBytes(snBytes) + keyUsage, extKeyUsage, err := keyUsagesFromStrings(usage) + if err != nil { + return nil, err + } + template := &x509.Certificate{ + Subject: certReq.Subject, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(duration365d), // defaults to 1 year + SerialNumber: &sn, + DNSNames: certReq.DNSNames, + BasicConstraintsValid: true, + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + } + return template, nil +} + +func (c *IPsecCSRSigningController) processNextFixtureWorkItem() bool { + key, quit := c.fixturesQueue.Get() + if quit { + return false + } + defer c.fixturesQueue.Done(key) + err := c.syncRootCertificateAndKey() + if err != nil { + c.fixturesQueue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync root CA and private key") + return true + } + c.fixturesQueue.Forget(key) + return true +} + +func (c *IPsecCSRSigningController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} + +// generateSelfSignedRootCertificate creates self-signed CA certificates and returns the PEM encoded +// certificates and private key. +func generateSelfSignedRootCertificate(commonName string) ([]byte, []byte, error) { + validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + caTemplate := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: commonName, + Organization: []string{antreaapis.AntreaOrganizationName}, + }, + NotBefore: validFrom, + NotAfter: validFrom.Add(duration10y), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + + caDERBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{Type: certutil.CertificateBlockType, Bytes: caDERBytes}); err != nil { + return nil, nil, err + } + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(caKey)}); err != nil { + return nil, nil, err + } + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go new file mode 100644 index 00000000000..5100f5bc63a --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go @@ -0,0 +1,139 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func TestIPsecCertificateApproverAndSigner(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedError error + expectedApproved, expectedDenied bool + }{ + { + name: "verify and sign valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/antrea-agent-ipsec-tunnel", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + expectedDenied: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tt.objects...) + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + stopCh := make(chan struct{}) + defer close(stopCh) + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + + approvingController := NewCSRApprovingController(clientset, csrInformer.Informer(), csrInformer.Lister()) + signingController := NewIPsecCSRSigningController(clientset, csrInformer.Informer(), csrInformer.Lister(), true) + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + + go approvingController.Run(stopCh) + go signingController.Run(stopCh) + + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), tt.csr, metav1.CreateOptions{}) + require.NoError(t, err) + err = wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csr.Name, metav1.GetOptions{}) + require.NoError(t, err) + if !isCertificateRequestApproved(csr) { + return false, nil + } + if len(csr.Status.Certificate) == 0 { + return false, nil + } + return true, nil + }) + require.NoError(t, err) + issued := csr.Status.Certificate + parsed, err := certutil.ParseCertsPEM(issued) + assert.NoError(t, err) + require.Len(t, parsed, 1) + roots := x509.NewCertPool() + roots.AddCert(signingController.certificateAuthority.Load().(*certificateAuthority).Certificate) + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + _, err = parsed[0].Verify(verifyOptions) + assert.NoError(t, err) + }) + } +} diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 73a7d15f461..aa3dc1fe1f8 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -95,6 +95,10 @@ const ( // alpha: v1.7 // Enable mirroring or redirecting the traffic Pods send or receive. TrafficControl featuregate.Feature = "TrafficControl" + + // alpha: v1.7 + // Enable certificated-based authentication for IPsec. + IPsecCertAuth featuregate.Feature = "IPsecCertAuth" ) var ( @@ -123,6 +127,7 @@ var ( SecondaryNetwork: {Default: false, PreRelease: featuregate.Alpha}, ServiceExternalIP: {Default: false, PreRelease: featuregate.Alpha}, TrafficControl: {Default: false, PreRelease: featuregate.Alpha}, + IPsecCertAuth: {Default: false, PreRelease: featuregate.Alpha}, } // UnsupportedFeaturesOnWindows records the features not supported on diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index f4b2bd07115..0f6907f5061 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -40,7 +40,7 @@ type OVSBridgeClient interface { CreateAccessPort(name, ifDev string, externalIDs map[string]interface{}, vlanID uint16) (string, Error) CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) - CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, psk string, externalIDs map[string]interface{}) (string, Error) + CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) DeletePort(portUUID string) Error DeletePorts(portUUIDList []string) Error @@ -51,6 +51,7 @@ type OVSBridgeClient interface { GetOVSVersion() (string, Error) AddOVSOtherConfig(configs map[string]interface{}) Error GetOVSOtherConfig() (map[string]string, Error) + UpdateOVSOtherConfig(configs map[string]interface{}) Error DeleteOVSOtherConfig(configs map[string]interface{}) Error AddBridgeOtherConfig(configs map[string]interface{}) Error SetBridgeMcastSnooping(enabled bool) Error diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index 0d0f01ac3b0..03e09027ad9 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -373,7 +373,7 @@ func (br *OVSBridge) CreateInternalPort(name string, ofPortRequest int32, extern // the bridge. // If ofPortRequest is not zero, it will be passed to the OVS port creation. func (br *OVSBridge) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) { - return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", nil) + return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", "", nil) } // CreateTunnelPortExt creates a tunnel port with the specified name and type @@ -393,12 +393,16 @@ func (br *OVSBridge) CreateTunnelPortExt( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { if psk != "" && remoteIP == "" { return "", newInvalidArgumentsError("IPsec tunnel can not be flow based. remoteIP must be set") } - return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, psk, externalIDs) + if psk != "" && remoteName != "" { + return "", newInvalidArgumentsError("Cannot set psk and remoteName together") + } + return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, remoteName, psk, externalIDs) } func (br *OVSBridge) createTunnelPort( @@ -408,6 +412,7 @@ func (br *OVSBridge) createTunnelPort( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { @@ -429,7 +434,9 @@ func (br *OVSBridge) createTunnelPort( if localIP != "" { options["local_ip"] = localIP } - + if remoteName != "" { + options["remote_name"] = remoteName + } if psk != "" { options["psk"] = psk } @@ -481,13 +488,13 @@ func (br *OVSBridge) SetInterfaceOptions(name string, options map[string]interfa // ParseTunnelInterfaceOptions reads remote IP, local IP, IPsec PSK, and csum // from the tunnel interface options and returns them. -func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, bool) { +func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, string, bool) { if portData.Options == nil { - return nil, nil, "", false + return nil, nil, "", "", false } var ok bool - var remoteIPStr, localIPStr, psk string + var remoteIPStr, localIPStr, psk, remoteName string var remoteIP, localIP net.IP var csum bool @@ -504,7 +511,8 @@ func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, if csumStr, ok := portData.Options["csum"]; ok { csum, _ = strconv.ParseBool(csumStr) } - return remoteIP, localIP, psk, csum + remoteName = portData.Options["remote_name"] + return remoteIP, localIP, psk, remoteName, csum } // CreateUplinkPort creates uplink port. @@ -860,17 +868,57 @@ func (br *OVSBridge) GetOVSOtherConfig() (map[string]string, Error) { return buildMapFromOVSDBMap(otherConfigs), nil } +// UpdateOVSOtherConfig updates the given configs to the "other_config" column of +// the single record in the "Open_vSwitch" table. +// For each config, it will be updated if the existing value does not match the given one, +// and it will be added if its key does not exist. +// It the configs are already up to date, this function will be a no-op. +func (br *OVSBridge) UpdateOVSOtherConfig(configs map[string]interface{}) Error { + tx := br.ovsdb.Transaction(openvSwitchSchema) + list := make([]string, 0, len(configs)) + for k := range configs { + list = append(list, k) + } + deleteSet := makeOVSDBSetFromList(list) + insertSet := helpers.MakeOVSDBMap(configs) + tx.Mutate(dbtransaction.Mutate{ + Table: "Open_vSwitch", + Mutations: [][]interface{}{ + {"other_config", "delete", deleteSet}, + {"other_config", "insert", insertSet}, + }}) + _, err, temporary := tx.Commit() + if err != nil { + klog.Error("Transaction failed: ", err) + return NewTransactionError(err, temporary) + } + return nil +} + // DeleteOVSOtherConfig deletes the given configs from the "other_config" column of // the single record in the "Open_vSwitch" table. -// For each config, it will only be deleted if its key exists and its value matches the stored one. -// No error is returned if configs don't exist or don't match. +// For each config, it will be deleted if its key exists and the given value is empty string or +// its value matches the given one. No error is returned if configs don't exist or don't match. func (br *OVSBridge) DeleteOVSOtherConfig(configs map[string]interface{}) Error { tx := br.ovsdb.Transaction(openvSwitchSchema) - mutateSet := helpers.MakeOVSDBMap(configs) + mapToDelete := make(map[string]interface{}) + listToDelete := []string{} + for k, v := range configs { + if v.(string) != "" { + mapToDelete[k] = v + } else { + listToDelete = append(listToDelete, k) + } + } + mutateMap := helpers.MakeOVSDBMap(mapToDelete) + mutateSet := makeOVSDBSetFromList(listToDelete) tx.Mutate(dbtransaction.Mutate{ - Table: "Open_vSwitch", - Mutations: [][]interface{}{{"other_config", "delete", mutateSet}}, + Table: "Open_vSwitch", + Mutations: [][]interface{}{ + {"other_config", "delete", mutateSet}, + {"other_config", "delete", mutateMap}, + }, }) _, err, temporary := tx.Commit() diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index 882fb72bcf8..35b6dee874f 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -151,18 +151,18 @@ func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPort(arg0, arg1, arg2 int } // CreateTunnelPortExt mocks base method -func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6 string, arg7 map[string]interface{}) (string, ovsconfig.Error) { +func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6, arg7 string, arg8 map[string]interface{}) (string, ovsconfig.Error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) ret0, _ := ret[0].(string) ret1, _ := ret[1].(ovsconfig.Error) return ret0, ret1 } // CreateTunnelPortExt indicates an expected call of CreateTunnelPortExt -func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { +func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } // CreateUplinkPort mocks base method @@ -480,3 +480,17 @@ func (mr *MockOVSBridgeClientMockRecorder) SetPortExternalIDs(arg0, arg1 interfa mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPortExternalIDs", reflect.TypeOf((*MockOVSBridgeClient)(nil).SetPortExternalIDs), arg0, arg1) } + +// UpdateOVSOtherConfig mocks base method +func (m *MockOVSBridgeClient) UpdateOVSOtherConfig(arg0 map[string]interface{}) ovsconfig.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateOVSOtherConfig", arg0) + ret0, _ := ret[0].(ovsconfig.Error) + return ret0 +} + +// UpdateOVSOtherConfig indicates an expected call of UpdateOVSOtherConfig +func (mr *MockOVSBridgeClientMockRecorder) UpdateOVSOtherConfig(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateOVSOtherConfig", reflect.TypeOf((*MockOVSBridgeClient)(nil).UpdateOVSOtherConfig), arg0) +} diff --git a/test/e2e/ipsec_test.go b/test/e2e/ipsec_test.go index 67790def112..054331fe409 100644 --- a/test/e2e/ipsec_test.go +++ b/test/e2e/ipsec_test.go @@ -23,6 +23,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "antrea.io/antrea/pkg/agent/util" + agentconfig "antrea.io/antrea/pkg/config/agent" + controllerconfig "antrea.io/antrea/pkg/config/controller" ) // TestIPSec is the top-level test which contains all subtests for @@ -40,68 +42,119 @@ func TestIPSec(t *testing.T) { } defer teardownTest(t, data) - t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data) }) + t.Logf("Redeploy Antrea with IPsec tunnel enabled") + data.redeployAntrea(t, deployAntreaIPsec) + // Restore normal Antrea deployment with IPsec disabled. + defer data.redeployAntrea(t, deployAntreaDefault) + + t.Run("testIPSecPSKAuth", func(t *testing.T) { + cc := func(config *controllerconfig.ControllerConfig) { + } + ac := func(config *agentconfig.AgentConfig) { + config.IPsec.AuthenticationMode = "psk" + } + if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data, false) }) + }) + + t.Run("testIPSecCertificateAuth", func(t *testing.T) { + // restart the Controller first as Agent needs to get the CSR signed. + cc := func(config *controllerconfig.ControllerConfig) { + config.FeatureGates["IPsecCertAuth"] = true + } + if err := data.mutateAntreaConfigMap(cc, nil, true, false); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + ac := func(config *agentconfig.AgentConfig) { + config.FeatureGates["IPsecCertAuth"] = true + config.IPsec.AuthenticationMode = "cert" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data, true) }) + }) + t.Run("testIPSecDeleteStaleTunnelPorts", func(t *testing.T) { testIPSecDeleteStaleTunnelPorts(t, data) }) } -func (data *TestData) readSecurityAssociationsStatus(nodeName string) (up int, connecting int, err error) { +func (data *TestData) readSecurityAssociationsStatus(nodeName string) (up int, connecting int, isCertAuth bool, err error) { antreaPodName, err := data.getAntreaPodOnNode(nodeName) if err != nil { - return 0, 0, err + return 0, 0, false, err } - cmd := []string{"ipsec", "status"} + cmd := []string{"ipsec", "statusall"} stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-ipsec", cmd) if err != nil { - return 0, 0, fmt.Errorf("error when running 'ipsec status' on '%s': %v - stdout: %s - stderr: %s", nodeName, err, stdout, stderr) + return 0, 0, false, fmt.Errorf("error when running 'ipsec status' on '%s': %v - stdout: %s - stderr: %s", nodeName, err, stdout, stderr) } re := regexp.MustCompile(`Security Associations \((\d+) up, (\d+) connecting\)`) matches := re.FindStringSubmatch(stdout) if len(matches) == 0 { - return 0, 0, fmt.Errorf("unexpected 'ipsec status' output: %s", stdout) + return 0, 0, false, fmt.Errorf("unexpected 'ipsec statusall' output: %s", stdout) } v, err := strconv.ParseUint(matches[1], 10, 32) if err != nil { - return 0, 0, fmt.Errorf("error when retrieving 'up' SAs from 'ipsec status' output: %v", err) + return 0, 0, false, fmt.Errorf("error when retrieving 'up' SAs from 'ipsec statusall' output: %v", err) } up = int(v) v, err = strconv.ParseUint(matches[2], 10, 32) if err != nil { - return 0, 0, fmt.Errorf("error when retrieving 'connecting' SAs from 'ipsec status' output: %v", err) + return 0, 0, false, fmt.Errorf("error when retrieving 'connecting' SAs from 'ipsec statusall' output: %v", err) } connecting = int(v) - return up, connecting, nil + + re = regexp.MustCompile(`uses ([a-z-]+) key authentication`) + match := re.FindStringSubmatch(stdout) + if len(match) == 0 { + return 0, 0, false, fmt.Errorf("failed to determine authentication method from 'ipsec statusall' output: %s", stdout) + } + + if match[1] == "pre-shared" { + isCertAuth = false + } else if match[1] == "public" { + isCertAuth = true + } else { + return 0, 0, false, fmt.Errorf("unknown key authentication mode %q", match[1]) + } + + return up, connecting, isCertAuth, nil } // testIPSecTunnelConnectivity checks that Pod traffic across two Nodes over // the IPsec tunnel, by creating multiple Pods across distinct Nodes and having // them ping each other. -func testIPSecTunnelConnectivity(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) - - data.testPodConnectivityDifferentNodes(t) +func testIPSecTunnelConnectivity(t *testing.T, data *TestData, certAuth bool) { + var tag string + if certAuth { + tag = "ipsec-cert" + } else { + tag = "ipsec-psk" + } + podInfos, deletePods := createPodsOnDifferentNodes(t, data, data.testNamespace, tag) + defer deletePods() + data.runPingMesh(t, podInfos[:2], agnhostContainerName) // We know that testPodConnectivityDifferentNodes always creates a Pod on Node 0 for the // inter-Node ping test. nodeName := nodeName(0) - if up, _, err := data.readSecurityAssociationsStatus(nodeName); err != nil { + if up, _, isCertAuth, err := data.readSecurityAssociationsStatus(nodeName); err != nil { t.Errorf("Error when reading Security Associations: %v", err) } else if up == 0 { t.Errorf("Expected at least one 'up' Security Association, but got %d", up) + } else if isCertAuth != certAuth { + t.Errorf("Expected certificate authentication to be %t, got %t", certAuth, isCertAuth) } else { - t.Logf("Found %d 'up' SecurityAssociation(s) for Node '%s'", up, nodeName) + t.Logf("Found %d 'up' SecurityAssociation(s) for Node '%s', certificate auth: %t", up, nodeName, isCertAuth) } - - // Restore normal Antrea deployment with IPsec disabled. - data.redeployAntrea(t, deployAntreaDefault) } // testIPSecDeleteStaleTunnelPorts checks that when switching from IPsec mode to // non-encrypted mode, the previously created tunnel ports are deleted // correctly. func testIPSecDeleteStaleTunnelPorts(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) nodeName0 := nodeName(0) nodeName1 := nodeName(1) diff --git a/test/integration/ovs/ovs_client_test.go b/test/integration/ovs/ovs_client_test.go index 499ca91cc4c..776bdb621c8 100644 --- a/test/integration/ovs/ovs_client_test.go +++ b/test/integration/ovs/ovs_client_test.go @@ -204,13 +204,36 @@ func TestOVSOtherConfig(t *testing.T) { require.Nil(t, err, "Error when getting OVS other_config") require.Equal(t, map[string]string{"flow-restore-wait": "true", "foo1": "bar1", "foo2": "bar2"}, gotOtherConfigs, "other_config mismatched") + // Expect to modify existing values and insert new values + err = data.br.UpdateOVSOtherConfig(map[string]interface{}{"foo2": "bar3", "foo3": "bar2"}) + require.Nil(t, err, "Error when updating OVS other_config") + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"flow-restore-wait": "true", "foo1": "bar1", "foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + // Expect only the matched config "flow-restore-wait: true" will be deleted. - err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"flow-restore-wait": "true", "foo1": "bar2"}) + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"flow-restore-wait": "true", "foo1": "bar2", "foo2": "bar1"}) + require.Nil(t, err, "Error when deleting OVS other_config") + + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"foo1": "bar1", "foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + + // Expect "foo1" will be deleted + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"foo1": "", "foo2": "bar4"}) + require.Nil(t, err, "Error when deleting OVS other_config") + + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + + // Expect "foo2" will be deleted + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"foo2": "", "foo4": ""}) require.Nil(t, err, "Error when deleting OVS other_config") gotOtherConfigs, err = data.br.GetOVSOtherConfig() require.Nil(t, err, "Error when getting OVS other_config") - require.Equal(t, map[string]string{"foo1": "bar1", "foo2": "bar2"}, gotOtherConfigs, "other_config mismatched") + require.Equal(t, map[string]string{"foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") } func TestTunnelOptionCsum(t *testing.T) { @@ -242,7 +265,7 @@ func TestTunnelOptionCsum(t *testing.T) { defer data.teardown(t) name := "vxlan0" - _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", nil) + _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", "", nil) require.Nil(t, err, "Error when creating tunnel port") options, err := data.br.GetInterfaceOptions(name) require.Nil(t, err, "Error when getting interface options")