diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index d68c2d0faae..0506a97cec2 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -73,6 +73,10 @@ Kubernetes: `>= 1.16.0-0` | flowCollector.idleFlowExportTimeout | string | `"15s"` | timeout after which a flow record is sent to the collector for idle flows. | | hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. | | image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/antrea-ubuntu","tag":"latest"}` | Container image to use for Antrea components. | +| ipsec.authenticationMode | string | `"psk"` | The authentication mode to use for IPsec. Must be one of "psk" or "cert". | +| ipsec.csrSigner | object | `{"autoApprove":true,"selfSignedCA":true}` | CSR signer configuration when the authenticationMode is "cert". | +| ipsec.csrSigner.autoApprove | bool | `true` | - Enable auto approval of Antrea signer for IPsec certificates. | +| ipsec.csrSigner.selfSignedCA | bool | `true` | - Whether or not to use auto-generated self-signed CA. | | ipsec.psk | string | `"changeme"` | Preshared Key (PSK) for IKE authentication. It will be stored in a secret and passed to antrea-agent as an environment variable. | | kubeAPIServerOverride | string | `""` | Address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. | | logVerbosity | int | `0` | | diff --git a/build/charts/antrea/conf/antrea-agent.conf b/build/charts/antrea/conf/antrea-agent.conf index d60c167177e..0527661c0eb 100644 --- a/build/charts/antrea/conf/antrea-agent.conf +++ b/build/charts/antrea/conf/antrea-agent.conf @@ -50,6 +50,9 @@ featureGates: # Enable mirroring or redirecting the traffic Pods send or receive. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "TrafficControl" "default" false) }} +# Enable certificated-based authentication for IPsec. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: {{ .Values.ovs.bridgeName | quote }} @@ -279,3 +282,13 @@ antreaProxy: # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: {{ .proxyLoadBalancerIPs }} {{- end }} + +# IPsec tunnel related configurations. +ipsec: +{{- with .Values.ipsec }} + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: {{ .authenticationMode | quote }} +{{- end }} diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index f54bf5ce864..2ce21423ae7 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -25,6 +25,9 @@ featureGates: # Enable managing external IPs of Services of LoadBalancer type. {{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "ServiceExternalIP" "default" false) }} +# Enable certificated-based authentication for IPsec. +{{- include "featureGate" (dict "featureGates" .Values.featureGates "name" "IPsecCertAuth" "default" false) }} + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -71,3 +74,19 @@ nodeIPAM: # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: {{ .nodeCIDRMaskSizeIPv6 }} {{- end }} + +ipsecCSRSigner: +{{- with .Values.ipsec }} + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: {{ .csrSigner.autoApprove }} + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: {{ .csrSigner.selfSignedCA }} +{{- end }} diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 10e847a669b..4520872afdb 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -180,3 +180,12 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml index 3d24e44b45f..9cee03c60dd 100644 --- a/build/charts/antrea/templates/agent/daemonset.yaml +++ b/build/charts/antrea/templates/agent/daemonset.yaml @@ -205,6 +205,11 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true + {{- end }} # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -305,6 +310,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true {{- end }} volumes: - name: antrea-config @@ -322,6 +330,12 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca + optional: true + {{- end }} - name: host-var-run-antrea hostPath: path: /var/run/antrea diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index 2a5f043af35..0e7bacd56c9 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -80,14 +80,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -128,6 +140,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index f7212eb3289..b4a1b90bf67 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -59,9 +59,17 @@ wireGuard: port: 51820 ipsec: + # -- The authentication mode to use for IPsec. Must be one of "psk" or "cert". + authenticationMode: "psk" # -- Preshared Key (PSK) for IKE authentication. It will be stored in a secret # and passed to antrea-agent as an environment variable. psk: "changeme" + # -- CSR signer configuration when the authenticationMode is "cert". + csrSigner: + # --- Enable auto approval of Antrea signer for IPsec certificates. + autoApprove: true + # --- Whether or not to use auto-generated self-signed CA. + selfSignedCA: true egress: # -- CIDR ranges to which outbound Pod traffic will not be SNAT'd by Egresses. diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index d29d3f36159..0957b19a79c 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 7587cc5d41e..aa2c7efdd6a 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-agent @@ -3738,7 +3811,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: fd449f30e949fff2d22ed79bca0a040535429c5b605b7b93dfdbfd3b359115ae + checksum/config: 215e06b9ae507e0bf11e6da239908ee60b07bc419310825f504208e87815f0eb labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 7294819af5f..47e83a0ec7c 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e0d8c70d728f9f981756d8238d9b19a9c2321206b09a814a1cdb4ac604b190c + checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 7e0d8c70d728f9f981756d8238d9b19a9c2321206b09a814a1cdb4ac604b190c + checksum/config: 9b30c1a8c106bef23da9374bbf18b11a72b5cf96532c2941ca0a11e5af48d2e6 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 2047b52f63d..7ea65444221 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -121,6 +121,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -325,6 +328,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -376,6 +387,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -417,6 +431,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3009,6 +3037,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3170,14 +3207,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3218,6 +3267,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3509,7 +3582,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5038309e5a226d5d860b167b95e0d8ed55af1914526f52e5ef8600e527695e5 + checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -3639,6 +3712,9 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -3727,6 +3803,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true volumes: - name: antrea-config configMap: @@ -3743,6 +3822,10 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca + optional: true - name: host-var-run-antrea hostPath: path: /var/run/antrea @@ -3785,7 +3868,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: d5038309e5a226d5d860b167b95e0d8ed55af1914526f52e5ef8600e527695e5 + checksum/config: 97fb99b7b2d8e9a0a5a6075dc109ea93d55b9ff3b6dc06af72fdfbaabec1d97b labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 2b02024377d..3ec7336f8cc 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -108,6 +108,9 @@ data: # Enable mirroring or redirecting the traffic Pods send or receive. # TrafficControl: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # Name of the OpenVSwitch bridge antrea-agent will create and use. # Make sure it doesn't conflict with your existing OpenVSwitch bridges. ovsBridge: "br-int" @@ -312,6 +315,14 @@ data: # Note that setting ProxyLoadBalancerIPs to false usually only makes sense when ProxyAll is set to true and # kube-proxy is removed from the cluser, otherwise kube-proxy will still load-balance this traffic. proxyLoadBalancerIPs: true + + # IPsec tunnel related configurations. + ipsec: + # The authentication mode of IPsec tunnel. It has the following options: + # - psk (default): Use pre-shared key (PSK) for IKE authentication. + # - cert: Use CA-signed certificates for IKE authentication. This option requires the `IPsecCertAuth` + # feature gate to be enabled. + authenticationMode: "psk" antrea-cni.conflist: | { "cniVersion":"0.3.0", @@ -363,6 +374,9 @@ data: # Enable managing external IPs of Services of LoadBalancer type. # ServiceExternalIP: false + # Enable certificated-based authentication for IPsec. + # IPsecCertAuth: false + # The port for the antrea-controller APIServer to serve on. # Note that if it's set to another value, the `containerPort` of the `api` port of the # `antrea-controller` container must be set to the same value. @@ -404,6 +418,20 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecCSRSigner: + # Determines the auto-approve policy of Antrea CSR signer for IPsec certificates management. + # If enabled, Antrea will auto-approve the CertificateSingingRequest (CSR) if its subject and x509 extensions + # are permitted, and the requestor can be validated. If K8s `BoundServiceAccountTokenVolume` feature is enabled, + # the Pod identity will also be validated to provide maximum security. + # If set to false, Antrea will not auto-approve CertificateSingingRequests and they need to be approved + # manually by `kubectl certificate approve`. + autoApprove: true + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # tls.crt: + # tls.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2996,6 +3024,15 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - create --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3157,14 +3194,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3205,6 +3254,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/antrea-agent-ipsec-tunnel + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3496,7 +3569,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 03ee0481c65f3ec8ff9e879ff10c3cf65991a347a7aec1c9bc866b019ec470f1 + checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 labels: app: antrea component: antrea-agent @@ -3736,7 +3809,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 03ee0481c65f3ec8ff9e879ff10c3cf65991a347a7aec1c9bc866b019ec470f1 + checksum/config: 0cf7fc67ba29593ea1cdb73b19f72f8d53a24ac432b1737a1849591a0aa43e75 labels: app: antrea component: antrea-controller diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 362e3788996..00a162044de 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/cniserver/ipam" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/controller/egress" + "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/controller/networkpolicy" "antrea.io/antrea/pkg/agent/controller/noderoute" "antrea.io/antrea/pkg/agent/controller/serviceexternalip" @@ -148,12 +149,16 @@ func run(o *Options) error { klog.InfoS("enableIPSecTunnel is deprecated, use trafficEncryptionMode instead.") encryptionMode = config.TrafficEncryptionModeIPSec } + _, ipsecAuthenticationMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) networkConfig := &config.NetworkConfig{ TunnelType: ovsconfig.TunnelType(o.config.TunnelType), TrafficEncapMode: encapMode, TrafficEncryptionMode: encryptionMode, TransportIface: o.config.TransportInterface, TransportIfaceCIDRs: o.config.TransportInterfaceCIDRs, + IPsecConfig: config.IPsecConfig{ + AuthenticationMode: ipsecAuthenticationMode, + }, } wireguardConfig := &config.WireGuardConfig{ @@ -228,6 +233,13 @@ func run(o *Options) error { } nodeConfig := agentInitializer.GetNodeConfig() + var ipsecCertController *ipseccertificate.Controller + + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + ipsecCertController = ipseccertificate.NewIPSecCertificateController(k8sClient, ovsBridgeClient, nodeConfig.Name) + } + nodeRouteController := noderoute.NewNodeRouteController( k8sClient, informerFactory, @@ -238,7 +250,9 @@ func run(o *Options) error { networkConfig, nodeConfig, agentInitializer.GetWireGuardClient(), - o.config.AntreaProxy.ProxyAll) + o.config.AntreaProxy.ProxyAll, + ipsecCertController, + ) var groupCounters []proxytypes.GroupCounter groupIDUpdates := make(chan string, 100) @@ -484,6 +498,11 @@ func run(o *Options) error { go antreaClientProvider.Run(ctx) + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + go ipsecCertController.Run(stopCh) + } + go nodeRouteController.Run(stopCh) go networkPolicyController.Run(stopCh) diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index fbc7c7784d0..7f4da6d3d9e 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -124,6 +124,13 @@ func (o *Options) validate(args []string) error { if !ok { return fmt.Errorf("TrafficEncapMode %s is unknown", o.config.TrafficEncapMode) } + ok, ipsecAuthMode := config.GetIPsecAuthenticationModeFromStr(o.config.IPsec.AuthenticationMode) + if !ok { + return fmt.Errorf("IPsec AuthenticationMode %s is unknown", o.config.TrafficEncapMode) + } + if ipsecAuthMode == config.IPsecAuthenticationModeCert && !features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + return fmt.Errorf("IPsec AuthenticationMode %s requires feature gate %s to be enabled", o.config.TrafficEncapMode, features.IPsecCertAuth) + } // Check if the enabled features are supported on the OS. if err := o.checkUnsupportedFeatures(); err != nil { diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 25b75bca3bb..47c0c553514 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -24,21 +24,28 @@ import ( "time" apiextensionclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" genericopenapi "k8s.io/apiserver/pkg/endpoints/openapi" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/informers" + csrinformers "k8s.io/client-go/informers/certificates/v1" clientset "k8s.io/client-go/kubernetes" + csrlisters "k8s.io/client-go/listers/certificates/v1" + "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" netutils "k8s.io/utils/net" + antreaapis "antrea.io/antrea/pkg/apis" "antrea.io/antrea/pkg/apiserver" "antrea.io/antrea/pkg/apiserver/certificate" "antrea.io/antrea/pkg/apiserver/openapi" "antrea.io/antrea/pkg/apiserver/storage" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/clusteridentity" + "antrea.io/antrea/pkg/controller/certificatesigningrequest" "antrea.io/antrea/pkg/controller/egress" egressstore "antrea.io/antrea/pkg/controller/egress/store" "antrea.io/antrea/pkg/controller/externalippool" @@ -176,6 +183,22 @@ func run(o *Options) error { ) } + var csrApprovingController *certificatesigningrequest.CSRApprovingController + var csrSigningController *certificatesigningrequest.IPsecCSRSigningController + var csrInformer cache.SharedIndexInformer + var csrLister csrlisters.CertificateSigningRequestLister + if features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + csrInformer = csrinformers.NewFilteredCertificateSigningRequestInformer(client, 0, nil, func(listOptions *metav1.ListOptions) { + listOptions.FieldSelector = fields.OneTermEqualSelector("spec.signerName", antreaapis.AntreaIPsecCSRSignerName).String() + }) + csrLister = csrlisters.NewCertificateSigningRequestLister(csrInformer.GetIndexer()) + + if *o.config.IPsecCSRSignerConfig.AutoApprove { + csrApprovingController = certificatesigningrequest.NewCSRApprovingController(client, csrInformer, csrLister) + } + csrSigningController = certificatesigningrequest.NewIPsecCSRSigningController(client, csrInformer, csrLister, *o.config.IPsecCSRSignerConfig.SelfSignedCA) + } + if features.DefaultFeatureGate.Enabled(features.Egress) { egressController = egress.NewEgressController(crdClient, groupEntityIndex, egressInformer, externalIPPoolController, egressGroupStore) } @@ -319,6 +342,14 @@ func run(o *Options) error { go antreaIPAMController.Run(stopCh) } + if features.DefaultFeatureGate.Enabled(features.IPsecCertAuth) { + go csrInformer.Run(stopCh) + if *o.config.IPsecCSRSignerConfig.AutoApprove { + go csrApprovingController.Run(stopCh) + } + go csrSigningController.Run(stopCh) + } + <-stopCh klog.Info("Stopping Antrea controller") return nil diff --git a/cmd/antrea-controller/options.go b/cmd/antrea-controller/options.go index f10fa9cada6..cf5034bef6b 100644 --- a/cmd/antrea-controller/options.go +++ b/cmd/antrea-controller/options.go @@ -161,12 +161,10 @@ func (o *Options) setDefaults() { o.config.APIPort = apis.AntreaControllerAPIPort } if o.config.EnablePrometheusMetrics == nil { - o.config.EnablePrometheusMetrics = new(bool) - *o.config.EnablePrometheusMetrics = true + o.config.EnablePrometheusMetrics = ptrBool(true) } if o.config.SelfSignedCert == nil { - o.config.SelfSignedCert = new(bool) - *o.config.SelfSignedCert = true + o.config.SelfSignedCert = ptrBool(true) } if o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 = ipamIPv4MaskDefault @@ -175,4 +173,14 @@ func (o *Options) setDefaults() { if o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 = ipamIPv6MaskDefault } + if o.config.IPsecCSRSignerConfig.SelfSignedCA == nil { + o.config.IPsecCSRSignerConfig.SelfSignedCA = ptrBool(true) + } + if o.config.IPsecCSRSignerConfig.AutoApprove == nil { + o.config.IPsecCSRSignerConfig.AutoApprove = ptrBool(true) + } +} + +func ptrBool(value bool) *bool { + return &value } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index c37fabd3e1b..3914ed2acbd 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -74,6 +74,10 @@ var ( getTransportIPNetDeviceByName = GetTransportIPNetDeviceByName ) +// otherConfigKeysForIPsecCertificates are configurations added to OVS bridge when AuthenticationMode is "cert" and +// need to be deleted when changing to "psk". +var otherConfigKeysForIPsecCertificates = []string{"certificate", "private_key", "ca_cert", "remote_cert", "remote_name"} + // Initializer knows how to setup host networking, OpenVSwitch, and Openflow. type Initializer struct { client clientset.Interface @@ -348,17 +352,56 @@ func (i *Initializer) Initialize() error { } // initializeWireGuard must be executed after setupOVSBridge as it requires gateway addresses on the OVS bridge. - switch i.networkConfig.TrafficEncryptionMode { - case config.TrafficEncryptionModeIPSec: - if err := i.initializeIPSec(); err != nil { + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeWireGuard { + if err := i.initializeWireGuard(); err != nil { return err } - case config.TrafficEncryptionModeWireGuard: - if err := i.initializeWireGuard(); err != nil { + } + // TODO: clean up WireGuard related configurations. + + // Initialize for IPsec PSK mode. + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + i.networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModePSK { + if err := i.waitForIPsecMonitorDaemon(); err != nil { + return err + } + if err := i.readIPSecPSK(); err != nil { return err } } + // Initialize for IPsec Certificate mode. + if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec && + i.networkConfig.IPsecConfig.AuthenticationMode == config.IPsecAuthenticationModeCert { + if err := i.waitForIPsecMonitorDaemon(); err != nil { + return err + } + } else { + configs, err := i.ovsBridgeClient.GetOVSOtherConfig() + if err != nil { + return fmt.Errorf("failed to get OVS other configs: %w", err) + } + // Clean up certificate and private key files. + if configs["certificate"] != "" { + if err := os.Remove(configs["certificate"]); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete unused IPsec certificate", "file", configs["certificate"]) + } + } + if configs["private_key"] != "" { + if err := os.Remove(configs["private_key"]); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete unused IPsec private key", "file", configs["private_key"]) + } + } + toDelete := make(map[string]interface{}) + for _, key := range otherConfigKeysForIPsecCertificates { + toDelete[key] = "" + } + // Clean up stale configs in OVS database. + if err := i.ovsBridgeClient.DeleteOVSOtherConfig(toDelete); err != nil { + return fmt.Errorf("failed to clean up OVS other configs: %w", err) + } + } + wg.Add(1) // routeClient.Initialize() should be after i.setupOVSBridge() which // creates the host gateway interface. @@ -707,7 +750,7 @@ func (i *Initializer) setupDefaultTunnelInterface() error { externalIDs := map[string]interface{}{ interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaTunnel, } - tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", externalIDs) + tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", "", externalIDs) if err != nil { klog.Errorf("Failed to create tunnel port %s type %s on OVS bridge: %v", tunnelPortName, i.networkConfig.TunnelType, err) return err @@ -884,8 +927,8 @@ func (i *Initializer) initNodeLocalConfig() error { return nil } -// initializeIPSec checks if preconditions are met for using IPsec and reads the IPsec PSK value. -func (i *Initializer) initializeIPSec() error { +// waitForIPsecMonitorDaemon checks if preconditions are met for using IPsec. +func (i *Initializer) waitForIPsecMonitorDaemon() error { // At the time the agent is initialized and this code is executed, the // OVS daemons are already running given that we have successfully // connected to OVSDB. Given that the start_ovs script deletes existing @@ -908,10 +951,6 @@ func (i *Initializer) initializeIPSec() error { return fmt.Errorf("IPsec was requested, but the OVS IPsec monitor does not seem to be running") } } - - if err := i.readIPSecPSK(); err != nil { - return err - } return nil } @@ -929,13 +968,13 @@ func (i *Initializer) initializeWireGuard() error { // readIPSecPSK reads the IPsec PSK value from environment variable ANTREA_IPSEC_PSK func (i *Initializer) readIPSecPSK() error { - i.networkConfig.IPSecPSK = os.Getenv(ipsecPSKEnvKey) - if i.networkConfig.IPSecPSK == "" { + i.networkConfig.IPsecConfig.PSK = os.Getenv(ipsecPSKEnvKey) + if i.networkConfig.IPsecConfig.PSK == "" { return fmt.Errorf("IPsec PSK environment variable '%s' is not set or is empty", ipsecPSKEnvKey) } // Usually one does not want to log the secret data. - klog.V(4).Infof("IPsec PSK value: %s", i.networkConfig.IPSecPSK) + klog.V(4).Infof("IPsec PSK value: %s", i.networkConfig.IPsecConfig.PSK) return nil } diff --git a/pkg/agent/config/ipsec_authentication_mode.go b/pkg/agent/config/ipsec_authentication_mode.go new file mode 100644 index 00000000000..6c1ad60cb2c --- /dev/null +++ b/pkg/agent/config/ipsec_authentication_mode.go @@ -0,0 +1,53 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "strings" + +type IPsecAuthenticationMode int + +const ( + IPsecAuthenticationModePSK IPsecAuthenticationMode = iota + IPsecAuthenticationModeCert + IPsecAuthenticationModeInvalid = -1 +) + +var supportedIPsecAuthenticationModeStrs = [...]string{ + "psk", + "cert", +} + +func GetIPsecConfigModes() []IPsecAuthenticationMode { + return []IPsecAuthenticationMode{ + IPsecAuthenticationModePSK, + IPsecAuthenticationModeCert, + } +} + +// String returns value in string. +func (am IPsecAuthenticationMode) String() string { + return supportedIPsecAuthenticationModeStrs[am] +} + +// GetIPsecAuthenticationModeFromStr returns true and IPsecAuthenticationModeType corresponding to input string. +// Otherwise, false and undefined value is returned +func GetIPsecAuthenticationModeFromStr(str string) (bool, IPsecAuthenticationMode) { + for idx, ms := range supportedIPsecAuthenticationModeStrs { + if strings.EqualFold(ms, str) { + return true, IPsecAuthenticationMode(idx) + } + } + return false, IPsecAuthenticationModeInvalid +} diff --git a/pkg/agent/config/node_config.go b/pkg/agent/config/node_config.go index 5b60993e1b7..2de112bb53c 100644 --- a/pkg/agent/config/node_config.go +++ b/pkg/agent/config/node_config.go @@ -146,12 +146,18 @@ func (n *NodeConfig) String() string { n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig) } +// IPsecConfig includes IPsec related configurations. +type IPsecConfig struct { + AuthenticationMode IPsecAuthenticationMode + PSK string +} + // NetworkConfig includes user provided network configuration parameters. type NetworkConfig struct { TrafficEncapMode TrafficEncapModeType TunnelType ovsconfig.TunnelType TrafficEncryptionMode TrafficEncryptionModeType - IPSecPSK string + IPsecConfig IPsecConfig TransportIface string TransportIfaceCIDRs []string IPv4Enabled bool diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go new file mode 100644 index 00000000000..cca760beaac --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go @@ -0,0 +1,482 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync/atomic" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + certutil "k8s.io/client-go/util/cert" + csrutil "k8s.io/client-go/util/certificate/csr" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/ovs/ovsconfig" +) + +const ( + controllerName = "AntreaAgentIPsecCertificateController" + workerItemKey = "key" + minRetryDelay = 5 * time.Second + maxRetryDelay = 60 * time.Second + + // the mount path for CA certificate in antrea-ipsec container. + // StrongSwan will never reads CA certificates from folders other than `/etc/ipsec.d/cacerts`. + // Though StrongSwan will automatically load CA certificates from the folder, we set the ca_path in other_configs + // to the correct path for better consistency. + caCertificatePath = "/etc/ipsec.d/cacerts/ca.crt" + + ovsConfigCACertificateKey = "ca_cert" + ovsConfigPrivateKeyKey = "private_key" + ovsConfigCertificateKey = "certificate" +) + +// certificateWaitTimeout controls the amount of time we wait for certificate +// approval in one iteration. +var certificateWaitTimeout = 15 * time.Minute +var defaultCertificatesPath = "/var/run/openvswitch" + +// Controller is responsible for requesting certificates by CertificateSigningRequest and configure them to OVS +type Controller struct { + kubeClient clientset.Interface + ovsBridgeClient ovsconfig.OVSBridgeClient + nodeName string + queue workqueue.RateLimitingInterface + + rotateCertificate func() (*certificateKeyPair, error) + certificateKeyPair *certificateKeyPair + + // caPath and is initialized with NewIPSecCertificateController and should not + // be changed once Controller starts. + caPath string + // certificateFolderPath is the folder to store private keys and issued certificates. + // defaults to defaultCertificatesPath. + certificateFolderPath string + + syncedOnce uint32 +} + +// Manager is an interface to track the status of the IPsec certificate controller. +type Manager interface { + HasSynced() bool +} + +var _ Manager = (*Controller)(nil) + +func NewIPSecCertificateController( + kubeClient clientset.Interface, + ovsBridgeClient ovsconfig.OVSBridgeClient, + nodeName string, +) *Controller { + controller := &Controller{ + kubeClient: kubeClient, + ovsBridgeClient: ovsBridgeClient, + nodeName: nodeName, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "IPsecCertificateController"), + caPath: filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), + certificateFolderPath: defaultCertificatesPath, + } + controller.rotateCertificate = controller.newCertificateKeyPair + return controller +} + +// worker is a long-running function that will continually call the processNextWorkItem function in +// order to read and process a message on the workqueue. +func (c *Controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *Controller) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.ErrorS(nil, "Unexpected object in work queue", "object", obj) + return true + } else if err := c.syncConfigurations(); err == nil { + c.queue.Forget(key) + } else { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Error syncing IPSec certificates, requeuing") + } + return true +} + +type certificateKeyPair struct { + caCertificate []*x509.Certificate + certificate []*x509.Certificate + privateKey crypto.Signer + certificatePath string + privateKeyPath string + rotationDeadline time.Time +} + +func (pair *certificateKeyPair) validate() error { + if pair == nil { + return fmt.Errorf("certificate and key pair is nil") + } + if len(pair.caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + if len(pair.certificate) == 0 { + return fmt.Errorf("certificate is empty") + } + if pair.privateKey == nil { + return fmt.Errorf("private key is empty") + } + roots := x509.NewCertPool() + for _, r := range pair.caCertificate { + roots.AddCert(r) + } + certificate := pair.certificate[0] + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + if _, err := certificate.Verify(verifyOptions); err != nil { + return err + } + switch pub := certificate.PublicKey.(type) { + //TODO: support key types other than RSA such as *ecdsa.PublicKey. + case *rsa.PublicKey: + priv, ok := pair.privateKey.(*rsa.PrivateKey) + if !ok { + return fmt.Errorf("private key type does not match public key type") + } + if pub.N.Cmp(priv.N) != 0 { + return fmt.Errorf("private key does not match public key") + } + default: + return fmt.Errorf("unrecognized certificate public key type") + } + return nil +} + +// cleanup deletes the files of certificate and private key. +func (pair *certificateKeyPair) cleanup() { + if pair.certificatePath != "" { + // Delete the old certificate file. + if err := os.Remove(pair.certificatePath); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete old certificate", "file", pair.certificatePath) + } + } + if pair.privateKeyPath != "" { + // Delete the old private key file. + if err := os.Remove(pair.privateKeyPath); err != nil && !os.IsNotExist(err) { + klog.ErrorS(err, "Failed to delete old private key", "file", pair.privateKeyPath) + } + } +} + +// jitteryDuration returns a duration in [totalDuration * 0.7, totalDuration * 0.9]. +var jitteryDuration = func(totalDuration time.Duration) time.Duration { + // wait.Jitter returns a duration in [totalDuration, totalDuration * 1.2]. + return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(float64(totalDuration)*0.3) +} + +// nextRotationDeadline returns a value for the threshold at which the +// current certificate should be rotated, 80%+/-10% of the expiration of the +// certificate. The deadline will not change once calculated. +// This function is not thread-safe. +func (pair *certificateKeyPair) nextRotationDeadline() time.Time { + // Return the previous calculated rotation deadline if applicable. + if !pair.rotationDeadline.IsZero() { + return pair.rotationDeadline + } + notAfter := pair.certificate[0].NotAfter + totalDuration := notAfter.Sub(pair.certificate[0].NotBefore) + deadline := pair.certificate[0].NotBefore.Add(jitteryDuration(totalDuration)) + klog.InfoS("Calculated certificate rotation deadline", "expiration", notAfter, "deadline", deadline) + pair.rotationDeadline = deadline + return deadline +} + +func loadCertAndKeyFromFiles(caPath, certPath, keyPath string) (*certificateKeyPair, error) { + ca, err := loadRootCA(caPath) + if err != nil { + return nil, err + } + key, err := loadPrivateKey(keyPath) + if err != nil { + return nil, err + } + cert, err := loadCertificate(certPath) + if err != nil { + return nil, err + } + pair := &certificateKeyPair{ + certificatePath: certPath, + privateKeyPath: keyPath, + caCertificate: ca, + certificate: cert, + privateKey: key, + } + return pair, nil +} + +func (c *Controller) syncConfigurations() error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing IPsec certificate configurations", "duration", d) + }() + + var deadline time.Time + // Validate the existing certificate and key pair. + if err := c.certificateKeyPair.validate(); err != nil { + klog.ErrorS(err, "Verifying current certificate configurations failed") + deadline = time.Now() + } else { + deadline = c.certificateKeyPair.nextRotationDeadline() + } + // Current certificate is about to expire. + if sleepInterval := time.Until(deadline); sleepInterval <= 0 { + klog.InfoS("Start rotating IPsec certificate") + newCertKeyPair, err := c.rotateCertificate() + if err != nil { + return fmt.Errorf("failed to rotate certificate: %w", err) + } + if err := newCertKeyPair.validate(); err != nil { + newCertKeyPair.cleanup() + return fmt.Errorf("failed to validate new certificate: %w", err) + } + // Clean up old certificate and key pair. + if c.certificateKeyPair != nil { + c.certificateKeyPair.cleanup() + } + // Save the known good certificate and key pair. + c.certificateKeyPair = newCertKeyPair + // Calculate the rotation deadline of new certificate. + deadline = c.certificateKeyPair.nextRotationDeadline() + } + // Re-queue after the interval to renew the certificate. + c.queue.AddAfter(workerItemKey, time.Until(deadline)) + // Sync OVS bridge configurations. + if err := c.syncOVSConfigurations(c.certificateKeyPair.certificatePath, + c.certificateKeyPair.privateKeyPath, caCertificatePath); err != nil { + return err + } + atomic.StoreUint32(&c.syncedOnce, 1) + return nil +} + +// SyncedOnce returns true if the controller has configured certificate successfully +// at least once. +func (c *Controller) SyncedOnce() bool { + return atomic.LoadUint32(&c.syncedOnce) == 1 +} + +// HasSynced implements the Manager interface. +func (c *Controller) HasSynced() bool { + if c == nil { + return true + } + return c.SyncedOnce() +} + +func loadRootCA(caPath string) ([]*x509.Certificate, error) { + pemBlock, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, err + } + certs, err := certutil.ParseCertsPEM(pemBlock) + if err != nil { + return nil, fmt.Errorf("error reading root CA %s: %w", caPath, err) + } + return certs, nil +} + +func newRSAPrivateKey() (crypto.Signer, []byte, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate new RSA private key: %v", err) + } + bs, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal private key: %v", err) + } + return key, bs, nil +} + +func loadPrivateKey(privateKeyPath string) (crypto.Signer, error) { + var keyPEMBytes []byte + _, err := os.Stat(privateKeyPath) + if err == nil { + // Load the private key contents from file. + keyPEMBytes, err = ioutil.ReadFile(privateKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to read key file %s: %v", privateKeyPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to stat key file %s: %v", privateKeyPath, err) + } + if len(keyPEMBytes) > 0 { + // Try to parse private key from existing file. + parsed, err := keyutil.ParsePrivateKeyPEM(keyPEMBytes) + privateKey, ok := parsed.(crypto.Signer) + if err != nil || !ok { + klog.ErrorS(err, "Parse key from file error", "file", privateKeyPath) + } else { + return privateKey, nil + } + } + return nil, nil +} + +func loadCertificate(certPath string) ([]*x509.Certificate, error) { + var certPEMBytes []byte + _, err := os.Stat(certPath) + if err == nil { + // Load the certificate from file. + certPEMBytes, err = ioutil.ReadFile(certPath) + if err != nil { + return nil, fmt.Errorf("failed to read certificate file %s: %w", certPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to stat certificate file %s: %w", certPath, err) + } + if len(certPEMBytes) > 0 { + // Try to parse the certificate from the existing file. + certificates, err := certutil.ParseCertsPEM(certPEMBytes) + if err != nil { + klog.ErrorS(err, "Parse certificate from file error", "file", certPath) + } else { + return certificates, nil + } + } + return nil, nil +} + +func (c *Controller) syncOVSConfigurations(certPath, keyPath, caPath string) error { + ovsConfig := map[string]interface{}{ + ovsConfigCertificateKey: certPath, + ovsConfigPrivateKeyKey: keyPath, + ovsConfigCACertificateKey: caPath, + } + klog.InfoS("Updating OVS configurations for IPsec certificates") + return c.ovsBridgeClient.UpdateOVSOtherConfig(ovsConfig) +} + +func newCSR(csrNamePrefix, commonName string, privateKey crypto.Signer) (*certificatesv1.CertificateSigningRequest, error) { + subject := &pkix.Name{ + CommonName: commonName, + Organization: []string{antreaapis.AntreaOrganizationName}, + } + csrBytes, err := certutil.MakeCSR(privateKey, subject, []string{commonName}, nil) + if err != nil { + return nil, err + } + return &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: csrNamePrefix, + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: csrBytes, + SignerName: antreaapis.AntreaIPsecCSRSignerName, + Usages: []certificatesv1.KeyUsage{certificatesv1.UsageIPsecTunnel}, + }, + }, nil +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting " + controllerName) + defer klog.InfoS("Shutting down " + controllerName) + + // Load the previous configured certificate path from OVS database. + config, ovsErr := c.ovsBridgeClient.GetOVSOtherConfig() + if ovsErr != nil { + klog.ErrorS(ovsErr, "Failed to get OVS bridge other configs") + } + + certificatePath := config[ovsConfigCertificateKey] + privateKeyPath := config[ovsConfigPrivateKeyKey] + if certificatePath != "" && privateKeyPath != "" { + pair, err := loadCertAndKeyFromFiles(c.caPath, certificatePath, privateKeyPath) + if err != nil { + klog.ErrorS(err, "Failed to load IPsec certificate and private key from existing files", + "ca", c.caPath, "cert", certificatePath, "key", privateKeyPath) + } else { + c.certificateKeyPair = pair + } + } + + c.queue.Add(workerItemKey) + go wait.Until(c.worker, time.Second, stopCh) + <-stopCh +} + +func (c *Controller) newCertificateKeyPair() (*certificateKeyPair, error) { + key, rawKey, err := newRSAPrivateKey() + if err != nil { + return nil, fmt.Errorf("failed to generate new private key: %w", err) + } + // Always create a new CSR for certificate rotation. The old ones will be GCed automatically. + csrNamePrefix := fmt.Sprintf("%s-", c.nodeName) + csr, err := newCSR(csrNamePrefix, c.nodeName, key) + if err != nil { + return nil, err + } + csr, err = c.kubeClient.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), csr, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), certificateWaitTimeout) + defer cancel() + rawCert, err := csrutil.WaitForCertificate(ctx, c.kubeClient, csr.Name, csr.UID) + if err != nil { + return nil, err + } + // Use the hash of new certificate and key as the filename suffix. + hasher := sha256.New() + hasher.Write(rawCert) + hasher.Write(rawKey) + hash := hasher.Sum(nil) + certPath := filepath.Join(c.certificateFolderPath, fmt.Sprintf("%s-%x.crt", c.nodeName, hash[:5])) + keyPath := filepath.Join(c.certificateFolderPath, fmt.Sprintf("%s-%x.key", c.nodeName, hash[:5])) + if err := certutil.WriteCert(certPath, rawCert); err != nil { + return nil, err + } + if err := keyutil.WriteKey(keyPath, rawKey); err != nil { + return nil, err + } + klog.InfoS("Created new certificate and key for IPSec", "cert", certPath, "key", keyPath) + + return loadCertAndKeyFromFiles(c.caPath, certPath, keyPath) +} diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go new file mode 100644 index 00000000000..2128b838246 --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go @@ -0,0 +1,463 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + certutil "k8s.io/client-go/util/cert" + + ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" +) + +const fakeNodeName = "fake-node-1" + +type fakeController struct { + *Controller + mockController *gomock.Controller + mockBridgeClient *ovsconfigtest.MockOVSBridgeClient + rawCAcert []byte + caCert *x509.Certificate + caKey crypto.Signer +} + +func newFakeController(t *testing.T) *fakeController { + mockController := gomock.NewController(t) + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(mockController) + fakeClient := fake.NewSimpleClientset() + listCSRAction := k8stesting.NewRootListAction(certificatesv1.SchemeGroupVersion.WithResource("certificatesigningrequests"), certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest"), metav1.ListOptions{}) + + // add an reactor to fill the Name and UID in the Create request. + fakeClient.PrependReactor("create", "certificatesigningrequests", k8stesting.ReactionFunc( + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + csr := action.(k8stesting.CreateAction).GetObject().(*certificatesv1.CertificateSigningRequest) + if csr.ObjectMeta.GenerateName != "" { + csr.ObjectMeta.Name = fmt.Sprintf("%s%s", csr.ObjectMeta.GenerateName, utilrand.String(8)) + csr.ObjectMeta.GenerateName = "" + csr.UID = uuid.NewUUID() + csr.CreationTimestamp = metav1.Now() + } + return false, csr, nil + }), + ) + // add an reactor to honor the fieldsSelector in the List request. + fakeClient.PrependReactor("list", "certificatesigningrequests", func(action k8stesting.Action) (bool, runtime.Object, error) { + var csrList *certificatesv1.CertificateSigningRequestList + // list CSRs using the original reactors. + for _, reactor := range fakeClient.Fake.ReactionChain[1:] { + if !reactor.Handles(listCSRAction) { + continue + } + handled, ret, err := reactor.React(listCSRAction) + if !handled { + continue + } + if err != nil { + return false, nil, err + } + csrList = ret.(*certificatesv1.CertificateSigningRequestList) + } + actionList, ok := action.(k8stesting.ListActionImpl) + if !ok { + return true, nil, fmt.Errorf("unexpected action type, expected %T, got %T", k8stesting.ListActionImpl{}, action) + } + listFieldsSelector := actionList.GetListRestrictions().Fields + var filtered []certificatesv1.CertificateSigningRequest + for _, c := range csrList.Items { + csrSpecificFieldsSet := make(fields.Set) + csrSpecificFieldsSet["metadata.name"] = c.Name + if listFieldsSelector.Matches(csrSpecificFieldsSet) { + filtered = append(filtered, c) + } + } + return true, &certificatesv1.CertificateSigningRequestList{ + Items: filtered, + }, nil + }) + + originDefaultPath := defaultCertificatesPath + cfg := certutil.Config{ + CommonName: "antrea-ipsec-ca", + Organization: []string{"antrea.io"}, + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + rootCA, err := certutil.NewSelfSignedCACert(cfg, key) + require.NoError(t, err) + tempDir, err := ioutil.TempDir("", "antrea-ipsec-test") + require.NoError(t, err) + defaultCertificatesPath = tempDir + defer func() { + defaultCertificatesPath = originDefaultPath + }() + caData, err := certutil.EncodeCertificates(rootCA) + require.NoError(t, err) + err = certutil.WriteCert(filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), caData) + require.NoError(t, err) + + c := NewIPSecCertificateController(fakeClient, mockOVSBridgeClient, fakeNodeName) + return &fakeController{ + Controller: c, + mockController: mockController, + mockBridgeClient: mockOVSBridgeClient, + rawCAcert: caData, + caCert: rootCA, + caKey: key, + } +} + +func TestController_syncConfigurations(t *testing.T) { + t.Run("rotate certificate if current certificates are empty", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + return nil, fmt.Errorf("unable to rotate certificate") + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + assert.Nil(t, fakeController.certificateKeyPair) + <-ch + }) + t.Run("should not touch existing certificate if rotate certificate failed", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + fakeController.certificateKeyPair = &certificateKeyPair{ + certificatePath: "cert.crt", + privateKeyPath: "key.key", + } + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + return nil, fmt.Errorf("unable to rotate certificate") + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + assert.NotNil(t, fakeController.certificateKeyPair) + assert.Equal(t, "cert.crt", fakeController.certificateKeyPair.certificatePath) + assert.Equal(t, "key.key", fakeController.certificateKeyPair.privateKeyPath) + <-ch + }) + t.Run("should clean up new certificate if it is not valid", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + certPath := filepath.Join(fakeController.certificateFolderPath, "cert-1.crt") + keyPath := filepath.Join(fakeController.certificateFolderPath, "key-1.key") + ch := make(chan struct{}) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + close(ch) + require.NoError(t, ioutil.WriteFile(certPath, nil, 0600)) + require.NoError(t, ioutil.WriteFile(keyPath, nil, 0400)) + return &certificateKeyPair{ + certificatePath: certPath, + privateKeyPath: keyPath, + }, nil + } + err := fakeController.syncConfigurations() + assert.Error(t, err) + _, err = os.Stat(certPath) + assert.True(t, os.IsNotExist(err)) + _, err = os.Stat(keyPath) + assert.True(t, os.IsNotExist(err)) + <-ch + }) + t.Run("request and configure new certificates", func(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + assert.Equal(t, 0, fakeController.queue.Len()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // start a fake signer in background to sign CSRs. + go func() { + watcher, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for ev := range watcher.ResultChan() { + switch ev.Type { + case watch.Added: + csr, ok := ev.Object.(*certificatesv1.CertificateSigningRequest) + assert.True(t, ok) + signCSR(t, fakeController, csr, time.Hour*24) + } + } + }() + originalRotateCertificate := fakeController.rotateCertificate + newCertDst := filepath.Join(fakeController.certificateFolderPath, "newcert.crt") + newKeyDst := filepath.Join(fakeController.certificateFolderPath, "newkey.key") + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + pair, err := originalRotateCertificate() + assert.NoError(t, err) + os.Link(pair.certificatePath, newCertDst) + os.Link(pair.privateKeyPath, newKeyDst) + pair.certificatePath = newCertDst + pair.privateKeyPath = newKeyDst + return pair, nil + } + // should configure OVS properly in syncConfigurations() + expectedOVSConfig := map[string]interface{}{ + "certificate": newCertDst, + "private_key": newKeyDst, + "ca_cert": caCertificatePath, + } + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(expectedOVSConfig) + // syncConfigurations should not block and get signed certificates from CSR successfully. + err := fakeController.syncConfigurations() + assert.NoError(t, err) + list, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Items, 1) + assert.NotEmpty(t, fakeController.caPath) + + rotationDeadline := fakeController.certificateKeyPair.nextRotationDeadline() + assert.False(t, rotationDeadline.IsZero()) + fakeController.rotateCertificate = func() (*certificateKeyPair, error) { + t.Error("unexpected call rotateCertificate") + return nil, nil + } + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(expectedOVSConfig) + // syncConfigurations again should not request new certificates. + err = fakeController.syncConfigurations() + assert.NoError(t, err) + // rotation deadline should not be changed. + assert.Equal(t, fakeController.certificateKeyPair.nextRotationDeadline(), rotationDeadline) + }) +} + +func TestController_RotateCertificates(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + assert.Equal(t, 0, fakeController.queue.Len()) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + // start a fake signer in background to sign CSRs. + signCh := make(chan struct{}) + go func() { + defer close(signCh) + counter := 0 + watcher, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for ev := range watcher.ResultChan() { + switch ev.Type { + case watch.Added: + csr, ok := ev.Object.(*certificatesv1.CertificateSigningRequest) + assert.True(t, ok) + // issue a certificate with lifetime of 10 seconds. + signCSR(t, fakeController, csr, time.Second*10) + counter++ + if counter == 2 { + return + } + } + } + }() + fakeController.mockBridgeClient.EXPECT().GetOVSOtherConfig().Times(1) + fakeController.mockBridgeClient.EXPECT().UpdateOVSOtherConfig(gomock.Any()).MinTimes(1) + go fakeController.Run(ch) + // wait for the signer to finish signing two CSRs. + <-signCh + list, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().List(context.TODO(), metav1.ListOptions{}) + assert.NoError(t, err) + assert.Len(t, list.Items, 2) + delta := list.Items[0].CreationTimestamp.Sub(list.Items[1].CreationTimestamp.Time) + if delta < 0 { + delta = -delta + } + // the rotation interval should be in [7s, 9s], but it takes time to process the CSR request, + // so add one second to the upper bound. + assert.Less(t, delta, time.Second*10) + assert.LessOrEqual(t, time.Second*7, delta) +} + +func newIPsecCertTemplate(t *testing.T, nodeName string, notBefore, notAfter time.Time) *x509.Certificate { + return &x509.Certificate{ + Subject: pkix.Name{ + CommonName: nodeName, + Organization: []string{"antrea.io"}, + }, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: notBefore, + NotAfter: notAfter, + SerialNumber: big.NewInt(12345), + DNSNames: []string{nodeName}, + BasicConstraintsValid: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } +} + +func createCertificate(t *testing.T, nodeName string, caCert *x509.Certificate, caKey crypto.Signer, + publicKey crypto.PublicKey, notBefore time.Time, expirationDuration time.Duration) []byte { + template := newIPsecCertTemplate(t, nodeName, notBefore, notBefore.Add(expirationDuration)) + derBytes, err := x509.CreateCertificate(rand.Reader, template, caCert, publicKey, caKey) + require.NoError(t, err) + certs, err := x509.ParseCertificates(derBytes) + require.NoError(t, err) + assert.Len(t, certs, 1) + encoded, err := certutil.EncodeCertificates(certs...) + require.NoError(t, err) + return encoded +} + +func signCSR(t *testing.T, controller *fakeController, + csr *certificatesv1.CertificateSigningRequest, expirationDuration time.Duration) { + assert.Empty(t, csr.Status.Certificate) + // use the CreationTimestamp as the NotBefore field of issued certificates for testing. + assert.False(t, csr.CreationTimestamp.IsZero()) + block, remain := pem.Decode(csr.Spec.Request) + assert.Empty(t, remain) + req, err := x509.ParseCertificateRequest(block.Bytes) + assert.NoError(t, err) + newCert := createCertificate(t, req.Subject.CommonName, controller.caCert, + controller.caKey, req.PublicKey, csr.CreationTimestamp.Time, expirationDuration) + toUpdate := csr.DeepCopy() + toUpdate.Status.Conditions = append(toUpdate.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + toUpdate, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateApproval(context.TODO(), csr.Name, toUpdate, metav1.UpdateOptions{}) + assert.NoError(t, err) + + toUpdate = toUpdate.DeepCopy() + toUpdate.Status.Certificate = newCert + _, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}) + assert.NoError(t, err) +} + +func Test_jitteryDuration(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedLowerBound, expectedUpperBound time.Duration + }{ + { + name: "10 seconds", + duration: 10 * time.Second, + expectedLowerBound: 7 * time.Second, + expectedUpperBound: 9 * time.Second, + }, { + name: "10 minutes", + duration: 10 * time.Minute, + expectedLowerBound: 7 * time.Minute, + expectedUpperBound: 9 * time.Minute, + }, + { + name: "10 hours", + duration: 10 * time.Hour, + expectedLowerBound: 7 * time.Hour, + expectedUpperBound: 9 * time.Hour, + }, + { + name: "10 days", + duration: 10 * time.Hour * 24, + expectedLowerBound: 7 * time.Hour * 24, + expectedUpperBound: 9 * time.Hour * 24, + }, + { + name: "100 days", + duration: 100 * time.Hour * 24, + expectedLowerBound: 70 * time.Hour * 24, + expectedUpperBound: 90 * time.Hour * 24, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := jitteryDuration(tt.duration) + assert.LessOrEqual(t, tt.expectedLowerBound, d) + assert.LessOrEqual(t, d, tt.expectedUpperBound) + }) + } +} + +func Test_certificateKeyPair_nextRotationDeadline(t *testing.T) { + tests := []struct { + name string + notBefore, notAfter string + want string + }{ + { + "10 hours certificate should be rotated at notBefore + 8h", + "2022-05-20T00:00:00Z", + "2022-05-20T10:00:00Z", + "2022-05-20T08:00:00Z", + }, + { + "10 days certificate should be rotated at notBefore + 8d", + "2022-05-20T00:00:00Z", + "2022-05-30T00:00:00Z", + "2022-05-28T00:00:00Z", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + before, err := time.Parse(time.RFC3339, tt.notBefore) + require.NoError(t, err) + after, err := time.Parse(time.RFC3339, tt.notAfter) + require.NoError(t, err) + pair := &certificateKeyPair{ + certificate: []*x509.Certificate{{ + NotBefore: before, + NotAfter: after, + }}, + } + originJitterDuration := jitteryDuration + jitteryDuration = func(totalDuration time.Duration) time.Duration { + return time.Duration(float64(totalDuration) * 0.8) + } + defer func() { + jitteryDuration = originJitterDuration + }() + deadline := pair.nextRotationDeadline() + expectedDeadline, err := time.Parse(time.RFC3339, tt.want) + require.NoError(t, err) + assert.Equal(t, expectedDeadline, deadline) + newDeadline := pair.nextRotationDeadline() + assert.Equal(t, deadline, newDeadline, "rotation deadline should not change") + }) + } +} diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index ece61e3c475..d96e49d486c 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -32,6 +32,7 @@ import ( "k8s.io/klog/v2" "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/interfacestore" "antrea.io/antrea/pkg/agent/openflow" "antrea.io/antrea/pkg/agent/route" @@ -79,6 +80,10 @@ type Controller struct { installedNodes cache.Indexer wireGuardClient wireguard.Interface proxyAll bool + // ipsecCertificateManager is useful for determining whether the ipsec certificate has been configured + // or not when IPsec is enabled with "cert" mode. The NodeRouteController must wait for the certificate + // to be configured before installing routes/flows to peer Nodes to prevent unencrypted traffic across Nodes. + ipsecCertificateManager ipseccertificate.Manager } // NewNodeRouteController instantiates a new Controller object which will process Node events @@ -94,25 +99,27 @@ func NewNodeRouteController( nodeConfig *config.NodeConfig, wireguardClient wireguard.Interface, proxyAll bool, + ipsecCertificateManager ipseccertificate.Manager, ) *Controller { nodeInformer := informerFactory.Core().V1().Nodes() svcLister := informerFactory.Core().V1().Services() controller := &Controller{ - kubeClient: kubeClient, - ovsBridgeClient: ovsBridgeClient, - ofClient: client, - routeClient: routeClient, - interfaceStore: interfaceStore, - networkConfig: networkConfig, - nodeConfig: nodeConfig, - nodeInformer: nodeInformer, - nodeLister: nodeInformer.Lister(), - nodeListerSynced: nodeInformer.Informer().HasSynced, - svcLister: svcLister.Lister(), - queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), - installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), - wireGuardClient: wireguardClient, - proxyAll: proxyAll, + kubeClient: kubeClient, + ovsBridgeClient: ovsBridgeClient, + ofClient: client, + routeClient: routeClient, + interfaceStore: interfaceStore, + networkConfig: networkConfig, + nodeConfig: nodeConfig, + nodeInformer: nodeInformer, + nodeLister: nodeInformer.Lister(), + nodeListerSynced: nodeInformer.Informer().HasSynced, + svcLister: svcLister.Lister(), + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), + installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), + wireGuardClient: wireguardClient, + proxyAll: proxyAll, + ipsecCertificateManager: ipsecCertificateManager, } nodeInformer.Informer().AddEventHandlerWithResyncPeriod( cache.ResourceEventHandlerFuncs{ @@ -252,10 +259,17 @@ func (c *Controller) removeStaleTunnelPorts() error { klog.Errorf("Failed to retrieve IP address of Node %s: %v", node.Name, err) continue } - + var remoteName, psk string + // remote_name and psk are mutually exclusive. + switch c.networkConfig.IPsecConfig.AuthenticationMode { + case config.IPsecAuthenticationModeCert: + remoteName = node.Name + case config.IPsecAuthenticationModePSK: + psk = c.networkConfig.IPsecConfig.PSK + } ifaceID := util.GenerateNodeTunnelInterfaceKey(node.Name) ifaceName := util.GenerateNodeTunnelInterfaceName(node.Name) - if c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv4, ifaceName) || c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv6, ifaceName) { + if c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv4, psk, remoteName, ifaceName) || c.compareInterfaceConfig(interfaceConfig, peerNodeIPs.IPv6, psk, remoteName, ifaceName) { desiredInterfaces[ifaceID] = true } } @@ -289,9 +303,10 @@ func (c *Controller) removeStaleTunnelPorts() error { } func (c *Controller) compareInterfaceConfig(interfaceConfig *interfacestore.InterfaceConfig, - peerNodeIP net.IP, interfaceName string) bool { + peerNodeIP net.IP, psk, remoteName, interfaceName string) bool { return interfaceConfig.InterfaceName == interfaceName && - interfaceConfig.PSK == c.networkConfig.IPSecPSK && + interfaceConfig.PSK == psk && + interfaceConfig.RemoteName == remoteName && interfaceConfig.RemoteIP.Equal(peerNodeIP) && interfaceConfig.TunnelInterfaceConfig.Type == c.networkConfig.TunnelType } @@ -347,7 +362,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) { klog.Infof("Starting %s", controllerName) defer klog.Infof("Shutting down %s", controllerName) - if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.nodeListerSynced) { + if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.nodeListerSynced, c.ipsecCertificateManager.HasSynced) { return } @@ -626,11 +641,20 @@ func getPodCIDRsOnNode(node *corev1.Node) []string { func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int32, error) { portName := util.GenerateNodeTunnelInterfaceName(nodeName) interfaceConfig, exists := c.interfaceStore.GetNodeTunnelInterface(nodeName) - // check if Node IP, PSK, or tunnel type changes. This can + + var remoteName, psk string + // remote_name and psk are mutually exclusive. + switch c.networkConfig.IPsecConfig.AuthenticationMode { + case config.IPsecAuthenticationModeCert: + remoteName = nodeName + case config.IPsecAuthenticationModePSK: + psk = c.networkConfig.IPsecConfig.PSK + } + // check if Node IP, PSK, remote name, or tunnel type changes. This can // happen if removeStaleTunnelPorts fails to remove a "stale" // tunnel port for which the configuration has changed, return error to requeue the Node. if exists { - if !c.compareInterfaceConfig(interfaceConfig, nodeIP, portName) { + if !c.compareInterfaceConfig(interfaceConfig, nodeIP, psk, remoteName, portName) { klog.InfoS("IPsec tunnel interface config doesn't match the cached one, deleting the stale IPsec tunnel port", "node", nodeName, "interface", interfaceConfig.InterfaceName) if err := c.ovsBridgeClient.DeletePort(interfaceConfig.PortUUID); err != nil { return 0, fmt.Errorf("fail to delete the stale IPsec tunnel port %s: %v", interfaceConfig.InterfaceName, err) @@ -653,7 +677,8 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 false, "", nodeIP.String(), - c.networkConfig.IPSecPSK, + remoteName, + psk, ovsExternalIDs) if err != nil { return 0, fmt.Errorf("failed to create IPsec tunnel port for Node %s", nodeName) @@ -666,7 +691,9 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 c.networkConfig.TunnelType, nodeName, nodeIP, - c.networkConfig.IPSecPSK) + psk, + remoteName, + ) interfaceConfig.OVSPortConfig = ovsPortConfig c.interfaceStore.AddInterface(interfaceConfig) } @@ -693,20 +720,22 @@ func ParseTunnelInterfaceConfig( klog.V(2).Infof("OVS port %s has no options", portData.Name) return nil } - remoteIP, localIP, psk, csum := ovsconfig.ParseTunnelInterfaceOptions(portData) + remoteIP, localIP, psk, remoteName, csum := ovsconfig.ParseTunnelInterfaceOptions(portData) var interfaceConfig *interfacestore.InterfaceConfig var nodeName string if portData.ExternalIDs != nil { nodeName = portData.ExternalIDs[ovsExternalIDNodeName] } - if psk != "" { + if psk != "" || remoteName != "" { interfaceConfig = interfacestore.NewIPSecTunnelInterface( portData.Name, ovsconfig.TunnelType(portData.IFType), nodeName, remoteIP, - psk) + psk, + remoteName, + ) } else { interfaceConfig = interfacestore.NewTunnelInterface(portData.Name, ovsconfig.TunnelType(portData.IFType), localIP, csum) } diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index 7932c66b05e..8009d61e572 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -60,6 +60,12 @@ type fakeController struct { interfaceStore interfacestore.InterfaceStore } +type fakeIPsecCertificateManager struct{} + +func (f *fakeIPsecCertificateManager) HasSynced() bool { + return true +} + func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeController, func()) { clientset := fake.NewSimpleClientset() informerFactory := informers.NewSharedInformerFactory(clientset, 12*time.Hour) @@ -68,11 +74,11 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont ovsClient := ovsconfigtest.NewMockOVSBridgeClient(ctrl) routeClient := routetest.NewMockInterface(ctrl) interfaceStore := interfacestore.NewInterfaceStore() - + ipsecCertificateManager := &fakeIPsecCertificateManager{} c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ IPv4: nil, MAC: gatewayMAC, - }}, nil, false) + }}, nil, false, ipsecCertificateManager) return &fakeController{ Controller: c, clientset: clientset, @@ -229,12 +235,15 @@ func TestIPInPodSubnets(t *testing.T) { assert.Equal(t, false, c.Controller.IPInPodSubnets(net.ParseIP("8.8.8.8"))) } -func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig) (*fakeController, func()) { +func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig, authenticationMode config.IPsecAuthenticationMode) (*fakeController, func()) { c, closeFn := newController(t, &config.NetworkConfig{ TrafficEncapMode: 0, TunnelType: ovsconfig.TunnelType("vxlan"), TrafficEncryptionMode: config.TrafficEncryptionModeIPSec, - IPSecPSK: "changeme", + IPsecConfig: config.IPsecConfig{ + PSK: "changeme", + AuthenticationMode: authenticationMode, + }, }) for _, i := range ifaces { c.interfaceStore.AddInterface(i) @@ -257,7 +266,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { PortUUID: "123", }, }, - }) + }, config.IPsecAuthenticationModePSK) defer closeFn() defer c.queue.ShutDown() @@ -290,7 +299,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { assert.NoError(t, err) } -func TestCreateIPSecTunnelPort(t *testing.T) { +func TestCreateIPSecTunnelPortPSK(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { Type: interfacestore.TunnelInterface, @@ -319,7 +328,7 @@ func TestCreateIPSecTunnelPort(t *testing.T) { OFPort: int32(5), }, }, - }) + }, config.IPsecAuthenticationModePSK) defer closeFn() defer c.queue.ShutDown() @@ -332,11 +341,11 @@ func TestCreateIPSecTunnelPort(t *testing.T) { node2PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-2") c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP1.String(), "changeme", + false, "", nodeIP1.String(), "", "changeme", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) c.ovsClient.EXPECT().CreateTunnelPortExt( node2PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP2.String(), "changeme", + false, "", nodeIP2.String(), "", "changeme", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2"}).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) c.ovsClient.EXPECT().GetOFPort(node2PortName, false).Return(int32(2), nil) @@ -381,3 +390,46 @@ func TestCreateIPSecTunnelPort(t *testing.T) { }) } } + +func TestCreateIPSecTunnelPortCert(t *testing.T) { + c, closeFn := setup(t, nil, config.IPsecAuthenticationModeCert) + + defer closeFn() + defer c.queue.ShutDown() + stopCh := make(chan struct{}) + defer close(stopCh) + c.informerFactory.Start(stopCh) + c.informerFactory.WaitForCacheSync(stopCh) + + node1PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1") + c.ovsClient.EXPECT().CreateTunnelPortExt( + node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), + false, "", nodeIP1.String(), "xyz-k8s-0-1", "", + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + + tests := []struct { + name string + nodeName string + peerNodeIP net.IP + wantErr bool + want int32 + }{ + { + name: "create new port", + nodeName: "xyz-k8s-0-1", + peerNodeIP: nodeIP1, + wantErr: false, + want: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := c.createIPSecTunnelPort(tt.nodeName, tt.peerNodeIP) + hasErr := err != nil + assert.Equal(t, tt.wantErr, hasErr) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index a443dd6dd6d..dd4131398ba 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -68,7 +68,10 @@ type TunnelInterfaceConfig struct { LocalIP net.IP // IP address of the remote Node. RemoteIP net.IP - PSK string + // CommonName of the remote Name for certificate based authentication. + RemoteName string + // Pre-shard key for authentication. + PSK string // Whether options:csum is set for this tunnel interface. // If true, encapsulation header UDP checksums will be computed on outgoing packets. Csum bool @@ -144,8 +147,8 @@ func NewTunnelInterface(tunnelName string, tunnelType ovsconfig.TunnelType, loca // NewIPSecTunnelInterface creates InterfaceConfig for the IPsec tunnel to the // Node. -func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk string) *InterfaceConfig { - tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk} +func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk, remoteName string) *InterfaceConfig { + tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk, RemoteName: remoteName} return &InterfaceConfig{InterfaceName: interfaceName, Type: TunnelInterface, TunnelInterfaceConfig: tunnelConfig} } diff --git a/pkg/apis/certificates.go b/pkg/apis/certificates.go new file mode 100644 index 00000000000..ea699db1a8c --- /dev/null +++ b/pkg/apis/certificates.go @@ -0,0 +1,22 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package apis + +const ( + // AntreaOrganizationName is the organization name of Antrea managed certificates. + AntreaOrganizationName = "antrea.io" + // AntreaIPsecCSRSignerName is the signer name for signing IPsec certificates for antrea-agents. + AntreaIPsecCSRSignerName = "antrea.io/antrea-agent-ipsec-tunnel" +) diff --git a/pkg/config/agent/config.go b/pkg/config/agent/config.go index 0c913d191f0..4604434c930 100644 --- a/pkg/config/agent/config.go +++ b/pkg/config/agent/config.go @@ -192,6 +192,8 @@ type AgentConfig struct { AntreaProxy AntreaProxyConfig `yaml:"antreaProxy,omitempty"` // Egress related configurations. Egress EgressConfig `yaml:"egress"` + // IPsec related configurations. + IPsec IPsecConfig `yaml:"ipsec"` } type AntreaProxyConfig struct { @@ -246,3 +248,10 @@ type MulticastConfig struct { type EgressConfig struct { ExceptCIDRs []string `yaml:"exceptCIDRs,omitempty"` } + +type IPsecConfig struct { + // The authentication mode of IPsec tunnel. It has the following options: + // - psk (default): Use pre-shared key (PSK) for IKE authentication. + // - cert: Use CA-signed certificates for IKE authentication. + AuthenticationMode string `yaml:"authenticationMode,omitempty"` +} diff --git a/pkg/config/controller/config.go b/pkg/config/controller/config.go index 643d7e476cb..b0004b7e5a8 100644 --- a/pkg/config/controller/config.go +++ b/pkg/config/controller/config.go @@ -65,4 +65,18 @@ type ControllerConfig struct { LegacyCRDMirroring *bool `yaml:"legacyCRDMirroring,omitempty"` // NodeIPAM Configuration NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"` + // IPsec CSR signer configuration + IPsecCSRSignerConfig IPsecCSRSignerConfig `yaml:"ipsecCSRSigner"` +} + +type IPsecCSRSignerConfig struct { + // Indicates whether to use auto-generated self-signed CA certificate. + // If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + // tls.crt: + // tls.key: + // Defaults to true. + SelfSignedCA *bool `yaml:"selfSignedCA,omitempty"` + // Antrea signer auto approve policy. + // Defaults to true. + AutoApprove *bool `yaml:"autoApprove,omitempty"` } diff --git a/pkg/controller/certificatesigningrequest/approving_controller.go b/pkg/controller/certificatesigningrequest/approving_controller.go new file mode 100644 index 00000000000..1f26a26a371 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approving_controller.go @@ -0,0 +1,169 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "fmt" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + csrlisters "k8s.io/client-go/listers/certificates/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" +) + +const ( + approvingControllerName = "CertificateSigningRequestApprovingController" +) + +type approver interface { + recognize(csr *certificatesv1.CertificateSigningRequest) bool + verify(csr *certificatesv1.CertificateSigningRequest) (bool, error) + name() string +} + +// CSRApprovingController is responsible for approving CertificateSigningRequests. +type CSRApprovingController struct { + client clientset.Interface + csrInformer cache.SharedIndexInformer + csrLister csrlisters.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + queue workqueue.RateLimitingInterface + approvers []approver +} + +// NewCSRApprovingController returns a new *CSRApprovingController. +func NewCSRApprovingController(client clientset.Interface, csrInformer cache.SharedIndexInformer, csrLister csrlisters.CertificateSigningRequestLister) *CSRApprovingController { + c := &CSRApprovingController{ + client: client, + csrInformer: csrInformer, + csrLister: csrLister, + csrListerSynced: csrInformer.HasSynced, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + approvers: []approver{ + &ipsecCSRApprover{ + client: client, + }, + }, + } + csrInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + }, + resyncPeriod, + ) + return c +} + +// Run begins watching and syncing of the CSRApprovingController. +func (c *CSRApprovingController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.InfoS("Starting " + approvingControllerName) + defer klog.InfoS("Shutting down " + approvingControllerName) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced} + if !cache.WaitForNamedCacheSync(approvingControllerName, stopCh, cacheSyncs...) { + return + } + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +func (c *CSRApprovingController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *CSRApprovingController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + return + } + c.queue.Add(csr.Name) +} + +func (c *CSRApprovingController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing CertificateSigningRequest", "name", key, "duration", d) + }() + + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + // The spec will not be updated by antrea-agent once it is approved or denied. + if approved, denied := getCertApprovalCondition(&csr.Status); approved || denied { + return nil + } + for _, a := range c.approvers { + if a.recognize(csr) { + approved, err := a.verify(csr) + if err != nil { + return err + } + if approved { + toUpdate := csr.DeepCopy() + appendApprovalCondition(toUpdate, fmt.Sprintf("Automatically approved by %s", a.name())) + _, err = c.client.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.Background(), toUpdate.Name, toUpdate, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("error updating approval for csr: %w", err) + } + return nil + } + } + } + return nil +} + +func appendApprovalCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: message, + }) +} + +func (c *CSRApprovingController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} diff --git a/pkg/controller/certificatesigningrequest/approving_controller_test.go b/pkg/controller/certificatesigningrequest/approving_controller_test.go new file mode 100644 index 00000000000..714a9712b6b --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approving_controller_test.go @@ -0,0 +1,195 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +type fakeApprover struct { + approverName string + recognized, verified bool +} + +var _ approver = (*fakeApprover)(nil) + +func (f *fakeApprover) recognize(_ *certificatesv1.CertificateSigningRequest) bool { + return f.recognized +} + +func (f *fakeApprover) verify(_ *certificatesv1.CertificateSigningRequest) (bool, error) { + return f.verified, nil +} + +func (f *fakeApprover) name() string { + return f.approverName +} + +func TestCSRApprovingController_syncCSR(t *testing.T) { + tests := []struct { + name string + approvers []approver + expectErr bool + expectApproved, expectedDenied bool + csrToSync *certificatesv1.CertificateSigningRequest + }{ + { + name: "recognized and approved", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "not approved by any approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: false, + verified: false, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: false, + verified: false, + }, + }, + expectErr: false, + expectApproved: false, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "recognized by both approver and approved by the second approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: false, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "recognized by and approved by the second approver", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: false, + verified: true, + }, + &fakeApprover{ + approverName: "FakeApprover2", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: true, + expectedDenied: false, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + }, + }, + { + name: "do not approve denied CSR", + approvers: []approver{ + &fakeApprover{ + approverName: "FakeApprover", + recognized: true, + verified: true, + }, + }, + expectErr: false, + expectApproved: false, + expectedDenied: true, + csrToSync: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "csr-1", + }, + Status: certificatesv1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1.CertificateSigningRequestCondition{ + { + Type: certificatesv1.CertificateDenied, + Status: corev1.ConditionTrue, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tt.csrToSync) + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + stopCh := make(chan struct{}) + defer close(stopCh) + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + controller := NewCSRApprovingController(clientset, csrInformer.Informer(), csrInformer.Lister()) + controller.csrInformer = csrInformer.Informer() + controller.csrLister = csrInformer.Lister() + controller.approvers = tt.approvers + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + + err := controller.syncCSR(tt.csrToSync.Name) + assert.Equal(t, tt.expectErr, err != nil) + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csrToSync.Name, metav1.GetOptions{}) + require.NoError(t, err) + approved, denied := getCertApprovalCondition(&csr.Status) + assert.Equal(t, tt.expectApproved, approved) + assert.Equal(t, tt.expectedDenied, denied) + }) + } +} diff --git a/pkg/controller/certificatesigningrequest/common.go b/pkg/controller/certificatesigningrequest/common.go new file mode 100644 index 00000000000..9909863cb94 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/common.go @@ -0,0 +1,158 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "sort" + "time" + + certificates "k8s.io/api/certificates/v1" + sautil "k8s.io/apiserver/pkg/authentication/serviceaccount" + certutil "k8s.io/client-go/util/cert" + + antreaapis "antrea.io/antrea/pkg/apis" +) + +const ( + // Set resyncPeriod to 0 to disable resyncing. + resyncPeriod time.Duration = 0 + // How long to wait before retrying the processing of a CertificateSigningRequest change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing a CertificateSigningRequest change. + defaultWorkers = 2 +) + +var ( + errOrganizationNotAntrea = fmt.Errorf("subject organization is not %s", antreaapis.AntreaOrganizationName) + errDNSSANNotMatchCommonName = fmt.Errorf("DNS subjectAltNames do not match subject common name") + errEmailSANNotAllowed = fmt.Errorf("email subjectAltNames are not allowed") + errIPSANNotAllowed = fmt.Errorf("IP subjectAltNames are not allowed") + errURISANNotAllowed = fmt.Errorf("URI subjectAltNames are not allowed") + errCommonNameRequired = fmt.Errorf("subject common name is required") + errExtraFieldsRequired = fmt.Errorf("extra values must contain %q and %q", sautil.PodNameKey, sautil.PodUIDKey) + errPodUIDMismatch = fmt.Errorf("Pod UID does not match") + errPodNotOnNode = fmt.Errorf("Pod is not on requested Node") + errUserUnauthorized = fmt.Errorf("Unrecognized username") +) + +// isCertificateRequestApproved returns true if a certificate request has the +// "Approved" condition and no "Denied" conditions; false otherwise. +func isCertificateRequestApproved(csr *certificates.CertificateSigningRequest) bool { + approved, denied := getCertApprovalCondition(&csr.Status) + return approved && !denied +} + +func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != certutil.CertificateRequestBlockType { + err := fmt.Errorf("PEM block type must be %s", certutil.CertificateRequestBlockType) + return nil, err + } + return x509.ParseCertificateRequest(block.Bytes) +} + +type transientError struct { + error +} + +func getCertApprovalCondition(status *certificates.CertificateSigningRequestStatus) (bool, bool) { + var approved, denied bool + for _, c := range status.Conditions { + if c.Type == certificates.CertificateApproved { + approved = true + } + if c.Type == certificates.CertificateDenied { + denied = true + } + } + return approved, denied +} + +var keyUsageDict = map[certificates.KeyUsage]x509.KeyUsage{ + certificates.UsageSigning: x509.KeyUsageDigitalSignature, + certificates.UsageDigitalSignature: x509.KeyUsageDigitalSignature, + certificates.UsageContentCommitment: x509.KeyUsageContentCommitment, + certificates.UsageKeyEncipherment: x509.KeyUsageKeyEncipherment, + certificates.UsageKeyAgreement: x509.KeyUsageKeyAgreement, + certificates.UsageDataEncipherment: x509.KeyUsageDataEncipherment, + certificates.UsageCertSign: x509.KeyUsageCertSign, + certificates.UsageCRLSign: x509.KeyUsageCRLSign, + certificates.UsageEncipherOnly: x509.KeyUsageEncipherOnly, + certificates.UsageDecipherOnly: x509.KeyUsageDecipherOnly, +} + +var extKeyUsageDict = map[certificates.KeyUsage]x509.ExtKeyUsage{ + certificates.UsageAny: x509.ExtKeyUsageAny, + certificates.UsageServerAuth: x509.ExtKeyUsageServerAuth, + certificates.UsageClientAuth: x509.ExtKeyUsageClientAuth, + certificates.UsageCodeSigning: x509.ExtKeyUsageCodeSigning, + certificates.UsageEmailProtection: x509.ExtKeyUsageEmailProtection, + certificates.UsageSMIME: x509.ExtKeyUsageEmailProtection, + certificates.UsageIPsecEndSystem: x509.ExtKeyUsageIPSECEndSystem, + certificates.UsageIPsecTunnel: x509.ExtKeyUsageIPSECTunnel, + certificates.UsageIPsecUser: x509.ExtKeyUsageIPSECUser, + certificates.UsageTimestamping: x509.ExtKeyUsageTimeStamping, + certificates.UsageOCSPSigning: x509.ExtKeyUsageOCSPSigning, + certificates.UsageMicrosoftSGC: x509.ExtKeyUsageMicrosoftServerGatedCrypto, + certificates.UsageNetscapeSGC: x509.ExtKeyUsageNetscapeServerGatedCrypto, +} + +// keyUsagesFromStrings will translate a slice of usage strings from the +// certificates API ("pkg/apis/certificates".KeyUsage) to x509.KeyUsage and +// x509.ExtKeyUsage types. +func keyUsagesFromStrings(usages []certificates.KeyUsage) (x509.KeyUsage, []x509.ExtKeyUsage, error) { + var keyUsage x509.KeyUsage + var unrecognized []certificates.KeyUsage + extKeyUsages := make(map[x509.ExtKeyUsage]struct{}) + for _, usage := range usages { + if val, ok := keyUsageDict[usage]; ok { + keyUsage |= val + } else if val, ok := extKeyUsageDict[usage]; ok { + extKeyUsages[val] = struct{}{} + } else { + unrecognized = append(unrecognized, usage) + } + } + + var sorted sortedExtKeyUsage + for eku := range extKeyUsages { + sorted = append(sorted, eku) + } + sort.Sort(sorted) + + if len(unrecognized) > 0 { + return 0, nil, fmt.Errorf("unrecognized usage values: %q", unrecognized) + } + + return keyUsage, sorted, nil +} + +type sortedExtKeyUsage []x509.ExtKeyUsage + +func (s sortedExtKeyUsage) Len() int { + return len(s) +} + +func (s sortedExtKeyUsage) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedExtKeyUsage) Less(i, j int) bool { + return s[i] < s[j] +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go b/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go new file mode 100644 index 00000000000..0e909ff0c21 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_approver.go @@ -0,0 +1,155 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "fmt" + "reflect" + "strings" + + certificatesv1 "k8s.io/api/certificates/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + sautil "k8s.io/apiserver/pkg/authentication/serviceaccount" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/util/env" +) + +const ( + ipsecCSRApproverName = "AntreaIPsecCSRApprover" +) + +var ( + antreaAgentServiceAccountName = strings.Join([]string{ + "system", "serviceaccount", env.GetAntreaNamespace(), "antrea-agent", + }, ":") +) + +type ipsecCSRApprover struct { + client clientset.Interface +} + +var ipsecTunnelUsages = sets.NewString( + string(certificatesv1.UsageIPsecTunnel), +) + +var _ approver = (*ipsecCSRApprover)(nil) + +func (ic *ipsecCSRApprover) recognize(csr *certificatesv1.CertificateSigningRequest) bool { + return csr.Spec.SignerName == antreaapis.AntreaIPsecCSRSignerName +} + +func (ic *ipsecCSRApprover) verify(csr *certificatesv1.CertificateSigningRequest) (bool, error) { + var failedReasons []string + cr, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + return false, err + } + if err := ic.verifyCertificateRequest(cr, csr.Spec.Usages); err != nil { + if _, ok := err.(*transientError); ok { + return false, err + } else { + failedReasons = append(failedReasons, err.Error()) + } + } + if err := ic.verifyIdentity(cr.Subject.CommonName, csr); err != nil { + if _, ok := err.(*transientError); ok { + return false, err + } + failedReasons = append(failedReasons, err.Error()) + } + + if len(failedReasons) > 0 { + klog.InfoS("Verifing CertificateSigningRequest for IPsec failed", "reasons", failedReasons, "CSR", csr.Name) + return false, nil + } + return true, nil +} + +func (ic *ipsecCSRApprover) name() string { + return ipsecCSRApproverName +} + +func (ic *ipsecCSRApprover) verifyCertificateRequest(req *x509.CertificateRequest, usages []certificatesv1.KeyUsage) error { + if !reflect.DeepEqual(req.Subject.Organization, []string{antreaapis.AntreaOrganizationName}) { + return errOrganizationNotAntrea + } + if req.Subject.CommonName == "" { + return errCommonNameRequired + } + if len(req.URIs) > 0 { + return errURISANNotAllowed + } + if len(req.IPAddresses) > 0 { + return errIPSANNotAllowed + } + if len(req.EmailAddresses) > 0 { + return errEmailSANNotAllowed + } + if !reflect.DeepEqual([]string{req.Subject.CommonName}, req.DNSNames) { + return errDNSSANNotMatchCommonName + } + for _, u := range usages { + if !ipsecTunnelUsages.Has(string(u)) { + return fmt.Errorf("unsupported key usage: %v", u) + } + } + _, err := ic.client.CoreV1().Nodes().Get(context.TODO(), req.Subject.CommonName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("requested Node %s not found", req.Subject.CommonName) + } else if err != nil { + return &transientError{err} + } + return nil +} + +func (ic *ipsecCSRApprover) verifyIdentity(nodeName string, csr *certificatesv1.CertificateSigningRequest) error { + if csr.Spec.Username != antreaAgentServiceAccountName { + return errUserUnauthorized + } + podNameValues, podUIDValues := csr.Spec.Extra[sautil.PodNameKey], csr.Spec.Extra[sautil.PodUIDKey] + if len(podNameValues) == 0 && len(podUIDValues) == 0 { + klog.Warning("Could not determine Pod identity from CertificateSigningRequest.", + " Enable K8s BoundServiceAccountTokenVolume feature gate to provide maximum security.") + return nil + } + if len(podNameValues) == 0 || len(podUIDValues) == 0 { + return errExtraFieldsRequired + } + podName, podUID := podNameValues[0], podUIDValues[0] + if podName == "" || podUID == "" { + return errExtraFieldsRequired + } + pod, err := ic.client.CoreV1().Pods(env.GetAntreaNamespace()).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("Pod %s not found", podName) + } else if err != nil { + return &transientError{err} + } + if pod.ObjectMeta.UID != types.UID(podUID) { + return errPodUIDMismatch + } + if pod.Spec.NodeName != nodeName { + return errPodNotOnNode + } + return nil +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go b/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go new file mode 100644 index 00000000000..f4aaec4c854 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_approver_test.go @@ -0,0 +1,604 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "net" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func Test_validIPSecCSR(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + cr *x509.CertificateRequest + keyUsages []certificatesv1.KeyUsage + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: nil, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Organization missing", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errOrganizationNotAntrea, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "requested Node not found", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("requested Node worker-node-1 not found"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "DNS SAN not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + }, + expectedErr: errDNSSANNotMatchCommonName, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "key usages not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("unsupported key usage: client auth"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageClientAuth, + }, + }, + { + name: "IP SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + IPAddresses: []net.IP{net.ParseIP("1.2.3.4")}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errIPSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "URI SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + URIs: []*url.URL{ + {Host: "antrea.io"}, + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errURISANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Email SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + EmailAddresses: []string{"user@antrea.io"}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errEmailSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + err := ic.verifyCertificateRequest(tt.cr, tt.keyUsages) + if tt.expectedErr == nil { + assert.NoError(t, err, "validIPSecCSR should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "validIPSecCSR should return an error") + } + }) + } +} + +func Test_verifyIdentity(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + nodeName string + csr *certificatesv1.CertificateSigningRequest + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: nil, + }, + { + name: "invalid username", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:my-sa", + }, + }, + nodeName: "worker-node-1", + expectedErr: errUserUnauthorized, + }, + { + name: "Pod UID mismatch", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"7afec259-ba03-441d-adeb-be163da2da2c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodUIDMismatch, + }, + { + name: "extra fields missing", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{}, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: nil, + }, + { + name: "Pod is not on requested Node", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-2", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodNotOnNode, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + err := ic.verifyIdentity(tt.nodeName, tt.csr) + if tt.expectedErr == nil { + assert.NoError(t, err, "verifyPodOnNode should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "verifyPodOnNode should return an error") + } + }) + } +} + +func Test_ipsecCertificateApprover_recognize(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedResult bool + }{ + { + name: "valid IPsec CSR", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/antrea-agent-ipsec-tunnel", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + expectedResult: true, + }, + { + name: "Unknown signer name", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "k8s.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + expectedResult: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCSRApprover{ + client: client, + } + recognized := ic.recognize(tt.csr) + assert.Equal(t, tt.expectedResult, recognized) + }) + } +} + +func x509CRtoPEM(t *testing.T, cr *x509.CertificateRequest) (crypto.PrivateKey, []byte) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + crDer, err := x509.CreateCertificateRequest(rand.Reader, cr, privateKey) + require.NoError(t, err) + csrPemBlock := &pem.Block{ + Type: certutil.CertificateRequestBlockType, + Bytes: crDer, + } + crBytes := pem.EncodeToMemory(csrPemBlock) + assert.NotEmpty(t, crBytes) + return privateKey, crBytes +} + +func Test_ipsecCertificateApprover_verify(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedError error + expectedApproved bool + }{ + { + name: "valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + }, + { + name: "IPsec CSR with unknown username", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "my-pod-1", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ipsec-unknown-user", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"my-pod-1"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:user-1", + }, + }, + expectedApproved: false, + }, + { + name: "CSR missing ExtraValue", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/ipsec", + Extra: map[string]certificatesv1.ExtraValue{}, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objs := append(tt.objects, tt.csr) + client := fake.NewSimpleClientset(objs...) + ic := &ipsecCSRApprover{ + client: client, + } + approved, err := ic.verify(tt.csr) + if tt.expectedError != nil { + assert.EqualError(t, err, tt.expectedError.Error()) + } else { + assert.Equal(t, tt.expectedApproved, approved) + } + }) + } +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go new file mode 100644 index 00000000000..e6fc51bc254 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller.go @@ -0,0 +1,470 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "reflect" + "sync/atomic" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + corev1informers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + csrlister "k8s.io/client-go/listers/certificates/v1" + csrlisters "k8s.io/client-go/listers/certificates/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + antreaapis "antrea.io/antrea/pkg/apis" + "antrea.io/antrea/pkg/util/env" +) + +const ( + ipsecRootCAName = "antrea-ipsec-ca" + ipsecCSRSigningControllerName = "IPsecCertificateSigningRequestSigningController" + workerItemKey = "key" + rootCACertKey = "ca.crt" + + duration365d = time.Hour * 24 * 365 + duration10y = duration365d * 10 +) + +// IPsecCSRSigningController is responsible for signing CertificateSigningRequests. +type IPsecCSRSigningController struct { + client clientset.Interface + csrInformer cache.SharedIndexInformer + csrLister csrlister.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + + configMapInformer cache.SharedIndexInformer + configMapLister corev1listers.ConfigMapLister + configMapListerSynced cache.InformerSynced + + selfSignedCA bool + + // saved CertificateAuthority + certificateAuthority atomic.Value + + queue workqueue.RateLimitingInterface + fixturesQueue workqueue.RateLimitingInterface +} + +// certificateAuthority implements a certificate authority and used by the signing controller. +type certificateAuthority struct { + // RawCert is an optional field to determine if signing cert/key pairs have changed + RawCert []byte + // RawKey is an optional field to determine if signing cert/key pairs have changed + RawKey []byte + + Certificate *x509.Certificate + PrivateKey crypto.Signer +} + +func (c *certificateAuthority) signCSR(template *x509.Certificate, requestKey crypto.PublicKey) (*x509.Certificate, error) { + if len(c.RawCert) == 0 || len(c.RawKey) == 0 { + return nil, fmt.Errorf("certificate authority is not valid") + } + derBytes, err := x509.CreateCertificate(rand.Reader, template, c.Certificate, requestKey, c.PrivateKey) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(derBytes) + if err != nil { + return nil, err + } + if len(certs) != 1 { + return nil, fmt.Errorf("expect a single certificate, got %d", len(certs)) + } + return certs[0], nil +} + +// NewIPsecCSRSigningController returns a new *IPsecCSRSigningController. +func NewIPsecCSRSigningController(client clientset.Interface, csrInformer cache.SharedIndexInformer, csrLister csrlisters.CertificateSigningRequestLister, selfSignedCA bool) *IPsecCSRSigningController { + + caConfigMapInformer := corev1informers.NewFilteredConfigMapInformer(client, env.GetAntreaNamespace(), resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *metav1.ListOptions) { + listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", ipsecRootCAName).String() + }) + + configMapLister := corev1listers.NewConfigMapLister(caConfigMapInformer.GetIndexer()) + + c := &IPsecCSRSigningController{ + client: client, + csrInformer: csrInformer, + csrLister: csrLister, + csrListerSynced: csrInformer.HasSynced, + configMapInformer: caConfigMapInformer, + configMapLister: configMapLister, + configMapListerSynced: caConfigMapInformer.HasSynced, + selfSignedCA: selfSignedCA, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + fixturesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + } + + csrInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + UpdateFunc: func(old, cur interface{}) { + c.enqueueCertificateSigningRequest(cur) + }, + }, + resyncPeriod, + ) + + caConfigMapInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + UpdateFunc: func(old, cur interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + DeleteFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + }, + resyncPeriod, + ) + + return c +} + +// Run begins watching and syncing of the IPsecCSRSigningController. +func (c *IPsecCSRSigningController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.Infof("Starting %s", ipsecCSRSigningControllerName) + defer klog.Infof("Shutting down %s", ipsecCSRSigningControllerName) + + go c.configMapInformer.Run(stopCh) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced, c.configMapListerSynced} + if !cache.WaitForNamedCacheSync(ipsecCSRSigningControllerName, stopCh, cacheSyncs...) { + return + } + c.fixturesQueue.Add(workerItemKey) + + go wait.Until(c.fixturesWorker, time.Second, stopCh) + + go wait.NonSlidingUntil(func() { + if err := c.watchSecretChanges(stopCh); err != nil { + klog.ErrorS(err, "Watch Secret error", "secret", ipsecRootCAName) + } + }, time.Second*10, stopCh) + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.csrWorker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *IPsecCSRSigningController) syncRootCertificateAndKey() error { + var caBytes, caKeyBytes []byte + caSecret, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Get(context.TODO(), ipsecRootCAName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + if !c.selfSignedCA { + klog.InfoS("Self-signed CA is disabled. Ensure CA Secret exists", "name", ipsecRootCAName, "namespace", env.GetAntreaNamespace()) + return nil + } + caBytes, caKeyBytes, err = generateSelfSignedRootCertificate(ipsecRootCAName) + if err != nil { + return err + } + caSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: ipsecRootCAName, + Namespace: env.GetAntreaNamespace(), + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{ + corev1.TLSCertKey: caBytes, + corev1.TLSPrivateKeyKey: caKeyBytes, + }, + } + caSecret, err = c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Create(context.TODO(), caSecret, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.Info("Created Secret for self-signed IPsec root CA") + } + caCertificate, err := certutil.ParseCertsPEM(caSecret.Data[corev1.TLSCertKey]) + if err != nil { + return err + } + if len(caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + privateKey, err := keyutil.ParsePrivateKeyPEM(caSecret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return err + } + priv, ok := privateKey.(crypto.Signer) + if !ok { + return fmt.Errorf("error reading CA: key did not implement crypto.Signer") + } + ca := &certificateAuthority{ + RawCert: caSecret.Data[corev1.TLSCertKey], + RawKey: caSecret.Data[corev1.TLSPrivateKeyKey], + Certificate: caCertificate[0], + PrivateKey: priv, + } + c.certificateAuthority.Store(ca) + desiredConfigMapData := map[string]string{ + rootCACertKey: string(caSecret.Data[corev1.TLSCertKey]), + } + caConfigMap, err := c.configMapLister.ConfigMaps(env.GetAntreaNamespace()).Get(ipsecRootCAName) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + caConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: ipsecRootCAName, + Namespace: env.GetAntreaNamespace(), + }, + Data: desiredConfigMapData, + } + caConfigMap, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Create(context.TODO(), caConfigMap, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.InfoS("Created ConfigMap for self-signed IPsec root CA") + } + if !reflect.DeepEqual(desiredConfigMapData, caConfigMap.Data) { + toUpdate := caConfigMap.DeepCopy() + toUpdate.Data = desiredConfigMapData + _, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Update(context.TODO(), toUpdate, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (c *IPsecCSRSigningController) csrWorker() { + for c.processNextWorkItem() { + } +} + +// watchSecretChanges uses watch API directly to watch for Secret changes. +// Antrea Controller should not have List permission for Secrets. +func (c *IPsecCSRSigningController) watchSecretChanges(endCh <-chan struct{}) error { + watcher, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Watch(context.TODO(), metav1.SingleObject(metav1.ObjectMeta{ + Namespace: env.GetAntreaNamespace(), + Name: ipsecRootCAName, + })) + if err != nil { + return fmt.Errorf("failed to create Secret watcher: %v", err) + } + // re-queue in case of missing events before watcher starts. + c.fixturesQueue.Add(workerItemKey) + ch := watcher.ResultChan() + defer watcher.Stop() + for { + select { + case _, ok := <-ch: + if !ok { + return nil + } + // we do not care the actual Event. + c.fixturesQueue.Add(workerItemKey) + case <-endCh: + return nil + } + } +} + +func (c *IPsecCSRSigningController) fixturesWorker() { + for c.processNextFixtureWorkItem() { + } +} + +func (c *IPsecCSRSigningController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + return + } + c.queue.Add(csr.Name) +} + +func (c *IPsecCSRSigningController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).InfoS("Finished syncing CertificateSigningRequest", "name", key, "duration", d) + }() + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if csr.Spec.SignerName != antreaapis.AntreaIPsecCSRSignerName { + return nil + } + if len(csr.Status.Certificate) != 0 { + klog.V(2).InfoS("CertificateSigningRequest is already signed", "CertificateSigningRequest", csr.Name) + return nil + } + if !isCertificateRequestApproved(csr) { + klog.V(2).InfoS("CertificateSigningRequest is not approved", "CertificateSigningRequest", csr.Name) + return nil + } + req, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + klog.ErrorS(err, "Failed to decode CertificateSigningRequest", "CertificateSigningRequest", csr.Name) + return nil + } + template, err := newCertificateTemplate(req, csr.Spec.Usages) + if err != nil { + return err + } + currCA, ok := c.certificateAuthority.Load().(*certificateAuthority) + if !ok || currCA == nil { + return fmt.Errorf("certificate authority is not initialized") + } + signed, err := currCA.signCSR(template, req.PublicKey) + if err != nil { + return err + } + bs, err := certutil.EncodeCertificates(signed) + if err != nil { + return err + } + toUpdate := csr.DeepCopy() + toUpdate.Status.Certificate = bs + _, err = c.client.CertificatesV1().CertificateSigningRequests().UpdateStatus(context.TODO(), toUpdate, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil +} + +func newCertificateTemplate(certReq *x509.CertificateRequest, usage []certificatesv1.KeyUsage) (*x509.Certificate, error) { + var sn big.Int + snBytes := make([]byte, 18) + _, err := rand.Read(snBytes) + if err != nil { + return nil, err + } + sn.SetBytes(snBytes) + keyUsage, extKeyUsage, err := keyUsagesFromStrings(usage) + if err != nil { + return nil, err + } + template := &x509.Certificate{ + Subject: certReq.Subject, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(duration365d), // defaults to 1 year + SerialNumber: &sn, + DNSNames: certReq.DNSNames, + BasicConstraintsValid: true, + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + } + return template, nil +} + +func (c *IPsecCSRSigningController) processNextFixtureWorkItem() bool { + key, quit := c.fixturesQueue.Get() + if quit { + return false + } + defer c.fixturesQueue.Done(key) + err := c.syncRootCertificateAndKey() + if err != nil { + c.fixturesQueue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync root CA and private key") + return true + } + c.fixturesQueue.Forget(key) + return true +} + +func (c *IPsecCSRSigningController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} + +// generateSelfSignedRootCertificate creates self-signed CA certificates and returns the PEM encoded +// certificates and private key. +func generateSelfSignedRootCertificate(commonName string) ([]byte, []byte, error) { + validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + caTemplate := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: commonName, + Organization: []string{antreaapis.AntreaOrganizationName}, + }, + NotBefore: validFrom, + NotAfter: validFrom.Add(duration10y), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + + caDERBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{Type: certutil.CertificateBlockType, Bytes: caDERBytes}); err != nil { + return nil, nil, err + } + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(caKey)}); err != nil { + return nil, nil, err + } + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} diff --git a/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go new file mode 100644 index 00000000000..5100f5bc63a --- /dev/null +++ b/pkg/controller/certificatesigningrequest/ipsec_csr_signing_controller_test.go @@ -0,0 +1,139 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func TestIPsecCertificateApproverAndSigner(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + expectedError error + expectedApproved, expectedDenied bool + }{ + { + name: "verify and sign valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/antrea-agent-ipsec-tunnel", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + Username: "system:serviceaccount:kube-system:antrea-agent", + }, + }, + expectedApproved: true, + expectedDenied: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tt.objects...) + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + stopCh := make(chan struct{}) + defer close(stopCh) + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + + approvingController := NewCSRApprovingController(clientset, csrInformer.Informer(), csrInformer.Lister()) + signingController := NewIPsecCSRSigningController(clientset, csrInformer.Informer(), csrInformer.Lister(), true) + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + + go approvingController.Run(stopCh) + go signingController.Run(stopCh) + + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), tt.csr, metav1.CreateOptions{}) + require.NoError(t, err) + err = wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csr.Name, metav1.GetOptions{}) + require.NoError(t, err) + if !isCertificateRequestApproved(csr) { + return false, nil + } + if len(csr.Status.Certificate) == 0 { + return false, nil + } + return true, nil + }) + require.NoError(t, err) + issued := csr.Status.Certificate + parsed, err := certutil.ParseCertsPEM(issued) + assert.NoError(t, err) + require.Len(t, parsed, 1) + roots := x509.NewCertPool() + roots.AddCert(signingController.certificateAuthority.Load().(*certificateAuthority).Certificate) + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + _, err = parsed[0].Verify(verifyOptions) + assert.NoError(t, err) + }) + } +} diff --git a/pkg/features/antrea_features.go b/pkg/features/antrea_features.go index 73a7d15f461..aa3dc1fe1f8 100644 --- a/pkg/features/antrea_features.go +++ b/pkg/features/antrea_features.go @@ -95,6 +95,10 @@ const ( // alpha: v1.7 // Enable mirroring or redirecting the traffic Pods send or receive. TrafficControl featuregate.Feature = "TrafficControl" + + // alpha: v1.7 + // Enable certificated-based authentication for IPsec. + IPsecCertAuth featuregate.Feature = "IPsecCertAuth" ) var ( @@ -123,6 +127,7 @@ var ( SecondaryNetwork: {Default: false, PreRelease: featuregate.Alpha}, ServiceExternalIP: {Default: false, PreRelease: featuregate.Alpha}, TrafficControl: {Default: false, PreRelease: featuregate.Alpha}, + IPsecCertAuth: {Default: false, PreRelease: featuregate.Alpha}, } // UnsupportedFeaturesOnWindows records the features not supported on diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index f4b2bd07115..0f6907f5061 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -40,7 +40,7 @@ type OVSBridgeClient interface { CreateAccessPort(name, ifDev string, externalIDs map[string]interface{}, vlanID uint16) (string, Error) CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) - CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, psk string, externalIDs map[string]interface{}) (string, Error) + CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) DeletePort(portUUID string) Error DeletePorts(portUUIDList []string) Error @@ -51,6 +51,7 @@ type OVSBridgeClient interface { GetOVSVersion() (string, Error) AddOVSOtherConfig(configs map[string]interface{}) Error GetOVSOtherConfig() (map[string]string, Error) + UpdateOVSOtherConfig(configs map[string]interface{}) Error DeleteOVSOtherConfig(configs map[string]interface{}) Error AddBridgeOtherConfig(configs map[string]interface{}) Error SetBridgeMcastSnooping(enabled bool) Error diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index 0d0f01ac3b0..03e09027ad9 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -373,7 +373,7 @@ func (br *OVSBridge) CreateInternalPort(name string, ofPortRequest int32, extern // the bridge. // If ofPortRequest is not zero, it will be passed to the OVS port creation. func (br *OVSBridge) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) { - return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", nil) + return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", "", nil) } // CreateTunnelPortExt creates a tunnel port with the specified name and type @@ -393,12 +393,16 @@ func (br *OVSBridge) CreateTunnelPortExt( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { if psk != "" && remoteIP == "" { return "", newInvalidArgumentsError("IPsec tunnel can not be flow based. remoteIP must be set") } - return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, psk, externalIDs) + if psk != "" && remoteName != "" { + return "", newInvalidArgumentsError("Cannot set psk and remoteName together") + } + return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, remoteName, psk, externalIDs) } func (br *OVSBridge) createTunnelPort( @@ -408,6 +412,7 @@ func (br *OVSBridge) createTunnelPort( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { @@ -429,7 +434,9 @@ func (br *OVSBridge) createTunnelPort( if localIP != "" { options["local_ip"] = localIP } - + if remoteName != "" { + options["remote_name"] = remoteName + } if psk != "" { options["psk"] = psk } @@ -481,13 +488,13 @@ func (br *OVSBridge) SetInterfaceOptions(name string, options map[string]interfa // ParseTunnelInterfaceOptions reads remote IP, local IP, IPsec PSK, and csum // from the tunnel interface options and returns them. -func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, bool) { +func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, string, bool) { if portData.Options == nil { - return nil, nil, "", false + return nil, nil, "", "", false } var ok bool - var remoteIPStr, localIPStr, psk string + var remoteIPStr, localIPStr, psk, remoteName string var remoteIP, localIP net.IP var csum bool @@ -504,7 +511,8 @@ func ParseTunnelInterfaceOptions(portData *OVSPortData) (net.IP, net.IP, string, if csumStr, ok := portData.Options["csum"]; ok { csum, _ = strconv.ParseBool(csumStr) } - return remoteIP, localIP, psk, csum + remoteName = portData.Options["remote_name"] + return remoteIP, localIP, psk, remoteName, csum } // CreateUplinkPort creates uplink port. @@ -860,17 +868,57 @@ func (br *OVSBridge) GetOVSOtherConfig() (map[string]string, Error) { return buildMapFromOVSDBMap(otherConfigs), nil } +// UpdateOVSOtherConfig updates the given configs to the "other_config" column of +// the single record in the "Open_vSwitch" table. +// For each config, it will be updated if the existing value does not match the given one, +// and it will be added if its key does not exist. +// It the configs are already up to date, this function will be a no-op. +func (br *OVSBridge) UpdateOVSOtherConfig(configs map[string]interface{}) Error { + tx := br.ovsdb.Transaction(openvSwitchSchema) + list := make([]string, 0, len(configs)) + for k := range configs { + list = append(list, k) + } + deleteSet := makeOVSDBSetFromList(list) + insertSet := helpers.MakeOVSDBMap(configs) + tx.Mutate(dbtransaction.Mutate{ + Table: "Open_vSwitch", + Mutations: [][]interface{}{ + {"other_config", "delete", deleteSet}, + {"other_config", "insert", insertSet}, + }}) + _, err, temporary := tx.Commit() + if err != nil { + klog.Error("Transaction failed: ", err) + return NewTransactionError(err, temporary) + } + return nil +} + // DeleteOVSOtherConfig deletes the given configs from the "other_config" column of // the single record in the "Open_vSwitch" table. -// For each config, it will only be deleted if its key exists and its value matches the stored one. -// No error is returned if configs don't exist or don't match. +// For each config, it will be deleted if its key exists and the given value is empty string or +// its value matches the given one. No error is returned if configs don't exist or don't match. func (br *OVSBridge) DeleteOVSOtherConfig(configs map[string]interface{}) Error { tx := br.ovsdb.Transaction(openvSwitchSchema) - mutateSet := helpers.MakeOVSDBMap(configs) + mapToDelete := make(map[string]interface{}) + listToDelete := []string{} + for k, v := range configs { + if v.(string) != "" { + mapToDelete[k] = v + } else { + listToDelete = append(listToDelete, k) + } + } + mutateMap := helpers.MakeOVSDBMap(mapToDelete) + mutateSet := makeOVSDBSetFromList(listToDelete) tx.Mutate(dbtransaction.Mutate{ - Table: "Open_vSwitch", - Mutations: [][]interface{}{{"other_config", "delete", mutateSet}}, + Table: "Open_vSwitch", + Mutations: [][]interface{}{ + {"other_config", "delete", mutateSet}, + {"other_config", "delete", mutateMap}, + }, }) _, err, temporary := tx.Commit() diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index 882fb72bcf8..35b6dee874f 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -151,18 +151,18 @@ func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPort(arg0, arg1, arg2 int } // CreateTunnelPortExt mocks base method -func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6 string, arg7 map[string]interface{}) (string, ovsconfig.Error) { +func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6, arg7 string, arg8 map[string]interface{}) (string, ovsconfig.Error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) ret0, _ := ret[0].(string) ret1, _ := ret[1].(ovsconfig.Error) return ret0, ret1 } // CreateTunnelPortExt indicates an expected call of CreateTunnelPortExt -func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { +func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } // CreateUplinkPort mocks base method @@ -480,3 +480,17 @@ func (mr *MockOVSBridgeClientMockRecorder) SetPortExternalIDs(arg0, arg1 interfa mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetPortExternalIDs", reflect.TypeOf((*MockOVSBridgeClient)(nil).SetPortExternalIDs), arg0, arg1) } + +// UpdateOVSOtherConfig mocks base method +func (m *MockOVSBridgeClient) UpdateOVSOtherConfig(arg0 map[string]interface{}) ovsconfig.Error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateOVSOtherConfig", arg0) + ret0, _ := ret[0].(ovsconfig.Error) + return ret0 +} + +// UpdateOVSOtherConfig indicates an expected call of UpdateOVSOtherConfig +func (mr *MockOVSBridgeClientMockRecorder) UpdateOVSOtherConfig(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateOVSOtherConfig", reflect.TypeOf((*MockOVSBridgeClient)(nil).UpdateOVSOtherConfig), arg0) +} diff --git a/test/e2e/ipsec_test.go b/test/e2e/ipsec_test.go index 67790def112..054331fe409 100644 --- a/test/e2e/ipsec_test.go +++ b/test/e2e/ipsec_test.go @@ -23,6 +23,8 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "antrea.io/antrea/pkg/agent/util" + agentconfig "antrea.io/antrea/pkg/config/agent" + controllerconfig "antrea.io/antrea/pkg/config/controller" ) // TestIPSec is the top-level test which contains all subtests for @@ -40,68 +42,119 @@ func TestIPSec(t *testing.T) { } defer teardownTest(t, data) - t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data) }) + t.Logf("Redeploy Antrea with IPsec tunnel enabled") + data.redeployAntrea(t, deployAntreaIPsec) + // Restore normal Antrea deployment with IPsec disabled. + defer data.redeployAntrea(t, deployAntreaDefault) + + t.Run("testIPSecPSKAuth", func(t *testing.T) { + cc := func(config *controllerconfig.ControllerConfig) { + } + ac := func(config *agentconfig.AgentConfig) { + config.IPsec.AuthenticationMode = "psk" + } + if err := data.mutateAntreaConfigMap(cc, ac, true, true); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data, false) }) + }) + + t.Run("testIPSecCertificateAuth", func(t *testing.T) { + // restart the Controller first as Agent needs to get the CSR signed. + cc := func(config *controllerconfig.ControllerConfig) { + config.FeatureGates["IPsecCertAuth"] = true + } + if err := data.mutateAntreaConfigMap(cc, nil, true, false); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + ac := func(config *agentconfig.AgentConfig) { + config.FeatureGates["IPsecCertAuth"] = true + config.IPsec.AuthenticationMode = "cert" + } + if err := data.mutateAntreaConfigMap(nil, ac, false, true); err != nil { + t.Fatalf("Failed to enable IPsecCertAuth feature: %v", err) + } + t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data, true) }) + }) + t.Run("testIPSecDeleteStaleTunnelPorts", func(t *testing.T) { testIPSecDeleteStaleTunnelPorts(t, data) }) } -func (data *TestData) readSecurityAssociationsStatus(nodeName string) (up int, connecting int, err error) { +func (data *TestData) readSecurityAssociationsStatus(nodeName string) (up int, connecting int, isCertAuth bool, err error) { antreaPodName, err := data.getAntreaPodOnNode(nodeName) if err != nil { - return 0, 0, err + return 0, 0, false, err } - cmd := []string{"ipsec", "status"} + cmd := []string{"ipsec", "statusall"} stdout, stderr, err := data.RunCommandFromPod(antreaNamespace, antreaPodName, "antrea-ipsec", cmd) if err != nil { - return 0, 0, fmt.Errorf("error when running 'ipsec status' on '%s': %v - stdout: %s - stderr: %s", nodeName, err, stdout, stderr) + return 0, 0, false, fmt.Errorf("error when running 'ipsec status' on '%s': %v - stdout: %s - stderr: %s", nodeName, err, stdout, stderr) } re := regexp.MustCompile(`Security Associations \((\d+) up, (\d+) connecting\)`) matches := re.FindStringSubmatch(stdout) if len(matches) == 0 { - return 0, 0, fmt.Errorf("unexpected 'ipsec status' output: %s", stdout) + return 0, 0, false, fmt.Errorf("unexpected 'ipsec statusall' output: %s", stdout) } v, err := strconv.ParseUint(matches[1], 10, 32) if err != nil { - return 0, 0, fmt.Errorf("error when retrieving 'up' SAs from 'ipsec status' output: %v", err) + return 0, 0, false, fmt.Errorf("error when retrieving 'up' SAs from 'ipsec statusall' output: %v", err) } up = int(v) v, err = strconv.ParseUint(matches[2], 10, 32) if err != nil { - return 0, 0, fmt.Errorf("error when retrieving 'connecting' SAs from 'ipsec status' output: %v", err) + return 0, 0, false, fmt.Errorf("error when retrieving 'connecting' SAs from 'ipsec statusall' output: %v", err) } connecting = int(v) - return up, connecting, nil + + re = regexp.MustCompile(`uses ([a-z-]+) key authentication`) + match := re.FindStringSubmatch(stdout) + if len(match) == 0 { + return 0, 0, false, fmt.Errorf("failed to determine authentication method from 'ipsec statusall' output: %s", stdout) + } + + if match[1] == "pre-shared" { + isCertAuth = false + } else if match[1] == "public" { + isCertAuth = true + } else { + return 0, 0, false, fmt.Errorf("unknown key authentication mode %q", match[1]) + } + + return up, connecting, isCertAuth, nil } // testIPSecTunnelConnectivity checks that Pod traffic across two Nodes over // the IPsec tunnel, by creating multiple Pods across distinct Nodes and having // them ping each other. -func testIPSecTunnelConnectivity(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) - - data.testPodConnectivityDifferentNodes(t) +func testIPSecTunnelConnectivity(t *testing.T, data *TestData, certAuth bool) { + var tag string + if certAuth { + tag = "ipsec-cert" + } else { + tag = "ipsec-psk" + } + podInfos, deletePods := createPodsOnDifferentNodes(t, data, data.testNamespace, tag) + defer deletePods() + data.runPingMesh(t, podInfos[:2], agnhostContainerName) // We know that testPodConnectivityDifferentNodes always creates a Pod on Node 0 for the // inter-Node ping test. nodeName := nodeName(0) - if up, _, err := data.readSecurityAssociationsStatus(nodeName); err != nil { + if up, _, isCertAuth, err := data.readSecurityAssociationsStatus(nodeName); err != nil { t.Errorf("Error when reading Security Associations: %v", err) } else if up == 0 { t.Errorf("Expected at least one 'up' Security Association, but got %d", up) + } else if isCertAuth != certAuth { + t.Errorf("Expected certificate authentication to be %t, got %t", certAuth, isCertAuth) } else { - t.Logf("Found %d 'up' SecurityAssociation(s) for Node '%s'", up, nodeName) + t.Logf("Found %d 'up' SecurityAssociation(s) for Node '%s', certificate auth: %t", up, nodeName, isCertAuth) } - - // Restore normal Antrea deployment with IPsec disabled. - data.redeployAntrea(t, deployAntreaDefault) } // testIPSecDeleteStaleTunnelPorts checks that when switching from IPsec mode to // non-encrypted mode, the previously created tunnel ports are deleted // correctly. func testIPSecDeleteStaleTunnelPorts(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) nodeName0 := nodeName(0) nodeName1 := nodeName(1) diff --git a/test/integration/ovs/ovs_client_test.go b/test/integration/ovs/ovs_client_test.go index 499ca91cc4c..776bdb621c8 100644 --- a/test/integration/ovs/ovs_client_test.go +++ b/test/integration/ovs/ovs_client_test.go @@ -204,13 +204,36 @@ func TestOVSOtherConfig(t *testing.T) { require.Nil(t, err, "Error when getting OVS other_config") require.Equal(t, map[string]string{"flow-restore-wait": "true", "foo1": "bar1", "foo2": "bar2"}, gotOtherConfigs, "other_config mismatched") + // Expect to modify existing values and insert new values + err = data.br.UpdateOVSOtherConfig(map[string]interface{}{"foo2": "bar3", "foo3": "bar2"}) + require.Nil(t, err, "Error when updating OVS other_config") + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"flow-restore-wait": "true", "foo1": "bar1", "foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + // Expect only the matched config "flow-restore-wait: true" will be deleted. - err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"flow-restore-wait": "true", "foo1": "bar2"}) + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"flow-restore-wait": "true", "foo1": "bar2", "foo2": "bar1"}) + require.Nil(t, err, "Error when deleting OVS other_config") + + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"foo1": "bar1", "foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + + // Expect "foo1" will be deleted + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"foo1": "", "foo2": "bar4"}) + require.Nil(t, err, "Error when deleting OVS other_config") + + gotOtherConfigs, err = data.br.GetOVSOtherConfig() + require.Nil(t, err, "Error when getting OVS other_config") + require.Equal(t, map[string]string{"foo2": "bar3", "foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") + + // Expect "foo2" will be deleted + err = data.br.DeleteOVSOtherConfig(map[string]interface{}{"foo2": "", "foo4": ""}) require.Nil(t, err, "Error when deleting OVS other_config") gotOtherConfigs, err = data.br.GetOVSOtherConfig() require.Nil(t, err, "Error when getting OVS other_config") - require.Equal(t, map[string]string{"foo1": "bar1", "foo2": "bar2"}, gotOtherConfigs, "other_config mismatched") + require.Equal(t, map[string]string{"foo3": "bar2"}, gotOtherConfigs, "other_config mismatched") } func TestTunnelOptionCsum(t *testing.T) { @@ -242,7 +265,7 @@ func TestTunnelOptionCsum(t *testing.T) { defer data.teardown(t) name := "vxlan0" - _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", nil) + _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", "", nil) require.Nil(t, err, "Error when creating tunnel port") options, err := data.br.GetInterfaceOptions(name) require.Nil(t, err, "Error when getting interface options")