From 4745ceb9449c2344eae1993fac998faef55a2695 Mon Sep 17 00:00:00 2001 From: Xu Liu Date: Fri, 13 May 2022 02:45:35 +0000 Subject: [PATCH] Support IPsec Certificate-based Authentication Introduce antrea-signer to sign CertificateSigningRequests requested by Antrea Agents for IPsec authentication. Signed-off-by: Xu Liu --- build/charts/antrea/README.md | 3 + .../charts/antrea/conf/antrea-controller.conf | 22 + .../antrea/templates/agent/clusterrole.yaml | 14 + .../antrea/templates/agent/daemonset.yaml | 13 + .../templates/controller/clusterrole.yaml | 44 ++ build/charts/antrea/values.yaml | 6 + build/yamls/antrea-aks.yml | 24 +- build/yamls/antrea-eks.yml | 24 +- build/yamls/antrea-gke.yml | 24 +- build/yamls/antrea-ipsec.yml | 81 ++- build/yamls/antrea.yml | 24 +- build/yamls/chart-values/antrea-ipsec.yml | 2 + cmd/antrea-agent/agent.go | 10 + cmd/antrea-controller/controller.go | 19 + cmd/antrea-controller/options.go | 23 +- hack/generate-manifest.sh | 2 +- pkg/agent/agent.go | 2 +- .../ipsec_certificate_controller.go | 520 +++++++++++++++ .../ipsec_certificate_controller_test.go | 268 ++++++++ .../noderoute/node_route_controller.go | 3 +- .../noderoute/node_route_controller_test.go | 4 +- pkg/config/controller/auto_approve_policy.go | 57 ++ pkg/config/controller/config.go | 16 + .../approver_controller.go | 335 ++++++++++ .../approver_controller_test.go | 599 ++++++++++++++++++ .../certificatesigningrequest/common.go | 131 ++++ .../signer_controller.go | 474 ++++++++++++++ .../signer_controller_test.go | 142 +++++ pkg/ovs/ovsconfig/interfaces.go | 2 +- pkg/ovs/ovsconfig/ovs_client.go | 13 +- pkg/ovs/ovsconfig/testing/mock_ovsconfig.go | 8 +- test/e2e/ipsec_test.go | 13 +- test/integration/ovs/ovs_client_test.go | 2 +- 33 files changed, 2888 insertions(+), 36 deletions(-) create mode 100644 pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go create mode 100644 pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go create mode 100644 pkg/config/controller/auto_approve_policy.go create mode 100644 pkg/controller/certificatesigningrequest/approver_controller.go create mode 100644 pkg/controller/certificatesigningrequest/approver_controller_test.go create mode 100644 pkg/controller/certificatesigningrequest/common.go create mode 100644 pkg/controller/certificatesigningrequest/signer_controller.go create mode 100644 pkg/controller/certificatesigningrequest/signer_controller_test.go diff --git a/build/charts/antrea/README.md b/build/charts/antrea/README.md index 2b14782d05f..f699f22983b 100644 --- a/build/charts/antrea/README.md +++ b/build/charts/antrea/README.md @@ -73,6 +73,9 @@ Kubernetes: `>= 1.16.0-0` | hostGateway | string | `"antrea-gw0"` | Name of the interface antrea-agent will create and use for host <-> Pod communication. | | image | object | `{"pullPolicy":"IfNotPresent","repository":"projects.registry.vmware.com/antrea/antrea-ubuntu","tag":"latest"}` | Container image to use for Antrea components. | | ipsec.psk | string | `"changeme"` | Preshared Key (PSK) for IKE authentication. It will be stored in a secret and passed to antrea-agent as an environment variable. | +| ipsecSigner.autoApprovePolicy | string | `"validateAgentPodName"` | Auto approve policy of Antrea signer | +| ipsecSigner.enable | bool | `false` | | +| ipsecSigner.selfSignedCA | bool | `true` | | | kubeAPIServerOverride | string | `""` | Address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig. | | logVerbosity | int | `0` | | | multicastInterfaces | list | `[]` | Names of the interfaces on Nodes that are used to forward multicast traffic. | diff --git a/build/charts/antrea/conf/antrea-controller.conf b/build/charts/antrea/conf/antrea-controller.conf index f54bf5ce864..83fdc85325d 100644 --- a/build/charts/antrea/conf/antrea-controller.conf +++ b/build/charts/antrea/conf/antrea-controller.conf @@ -71,3 +71,25 @@ nodeIPAM: # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: {{ .nodeCIDRMaskSizeIPv6 }} {{- end }} + +ipsecSigner: +{{- with .Values.ipsecSigner }} + # Enable the signer controller within the Antrea controller. + enable: {{ .enable }} + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: {{ .autoApprovePolicy | quote }} + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: {{ .selfSignedCA }} +{{- end }} diff --git a/build/charts/antrea/templates/agent/clusterrole.yaml b/build/charts/antrea/templates/agent/clusterrole.yaml index 10e847a669b..90fab25685b 100644 --- a/build/charts/antrea/templates/agent/clusterrole.yaml +++ b/build/charts/antrea/templates/agent/clusterrole.yaml @@ -180,3 +180,17 @@ rules: - get - list - watch + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - update + - patch + - create + - delete + {{- end }} diff --git a/build/charts/antrea/templates/agent/daemonset.yaml b/build/charts/antrea/templates/agent/daemonset.yaml index 3d24e44b45f..bc849805afb 100644 --- a/build/charts/antrea/templates/agent/daemonset.yaml +++ b/build/charts/antrea/templates/agent/daemonset.yaml @@ -205,6 +205,11 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true + {{- end }} # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -305,6 +310,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true {{- end }} volumes: - name: antrea-config @@ -322,6 +330,11 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca + {{- end }} - name: host-var-run-antrea hostPath: path: /var/run/antrea diff --git a/build/charts/antrea/templates/controller/clusterrole.yaml b/build/charts/antrea/templates/controller/clusterrole.yaml index 2a5f043af35..da742a86a72 100644 --- a/build/charts/antrea/templates/controller/clusterrole.yaml +++ b/build/charts/antrea/templates/controller/clusterrole.yaml @@ -80,14 +80,32 @@ rules: - configmaps resourceNames: - antrea-ca + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - antrea-ipsec-ca + {{- end }} - antrea-cluster-identity verbs: - get - update + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch + {{- end }} - apiGroups: - "" resources: - configmaps + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - secrets + {{- end }} verbs: - create - apiGroups: @@ -128,6 +146,32 @@ rules: verbs: - get - update + {{- if eq .Values.trafficEncryptionMode "ipsec" }} + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/signer + verbs: + - approve + - sign + {{- end }} - apiGroups: - crd.antrea.io resources: diff --git a/build/charts/antrea/values.yaml b/build/charts/antrea/values.yaml index 45a1297ba26..970f9d6ce32 100644 --- a/build/charts/antrea/values.yaml +++ b/build/charts/antrea/values.yaml @@ -89,6 +89,12 @@ nodeIPAM: # -- Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. nodeCIDRMaskSizeIPv6: 64 +ipsecSigner: + # -- Auto approve policy of Antrea signer + autoApprovePolicy: "validateAgentPodName" + enable: false + selfSignedCA: true + # -- Address of Kubernetes apiserver, to override any value provided in # kubeconfig or InClusterConfig. kubeAPIServerOverride: "" diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 72a8a5db275..ff821c40800 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -394,6 +394,26 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecSigner: + # Enable the signer controller within the Antrea controller. + enable: false + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: "validateAgentPodName" + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3486,7 +3506,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4554a36b927c6e64fdbc53b4d4c64673d48c9c829ec444e3be6e699ade8481b6 + checksum/config: dd6b34730b68c2f4159c567a8f97011dade5d9b566537a8120c1ddb7f845af98 labels: app: antrea component: antrea-agent @@ -3726,7 +3746,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4554a36b927c6e64fdbc53b4d4c64673d48c9c829ec444e3be6e699ade8481b6 + checksum/config: dd6b34730b68c2f4159c567a8f97011dade5d9b566537a8120c1ddb7f845af98 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index c274ef7b119..c4ce3195363 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -394,6 +394,26 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecSigner: + # Enable the signer controller within the Antrea controller. + enable: false + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: "validateAgentPodName" + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3486,7 +3506,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4554a36b927c6e64fdbc53b4d4c64673d48c9c829ec444e3be6e699ade8481b6 + checksum/config: dd6b34730b68c2f4159c567a8f97011dade5d9b566537a8120c1ddb7f845af98 labels: app: antrea component: antrea-agent @@ -3728,7 +3748,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 4554a36b927c6e64fdbc53b4d4c64673d48c9c829ec444e3be6e699ade8481b6 + checksum/config: dd6b34730b68c2f4159c567a8f97011dade5d9b566537a8120c1ddb7f845af98 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 6c63773e17f..4dcbbeab008 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -394,6 +394,26 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecSigner: + # Enable the signer controller within the Antrea controller. + enable: false + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: "validateAgentPodName" + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3486,7 +3506,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: edef4c00e4f28a10dc1e077086ef68641a9a3b53d0fe7d47ff3dafc2ce5d5c9b + checksum/config: beae75e2e51b9f980bafdb2974b763444c4ce48eafabffd91a363166d9bf6024 labels: app: antrea component: antrea-agent @@ -3726,7 +3746,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: edef4c00e4f28a10dc1e077086ef68641a9a3b53d0fe7d47ff3dafc2ce5d5c9b + checksum/config: beae75e2e51b9f980bafdb2974b763444c4ce48eafabffd91a363166d9bf6024 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index 2c8f8b76521..a4c2291603a 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -407,6 +407,26 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecSigner: + # Enable the signer controller within the Antrea controller. + enable: true + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: "validateAgentPodName" + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -2999,6 +3019,18 @@ rules: - get - list - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - watch + - list + - update + - patch + - create + - delete --- # Source: antrea/templates/antctl/clusterrole.yaml kind: ClusterRole @@ -3160,14 +3192,26 @@ rules: - configmaps resourceNames: - antrea-ca + - antrea-ipsec-ca - antrea-cluster-identity verbs: - get - update + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - antrea-ipsec-ca + verbs: + - get + - update + - watch - apiGroups: - "" resources: - configmaps + - secrets verbs: - create - apiGroups: @@ -3208,6 +3252,30 @@ rules: verbs: - get - update + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + - certificatesigningrequests/status + verbs: + - update + - apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - antrea.io/signer + verbs: + - approve + - sign - apiGroups: - crd.antrea.io resources: @@ -3499,7 +3567,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 50cc962db93c0354f5eaa088e51d690e779692979cbafac0c3e27a88fc2c0c7c + checksum/config: 2aec4c8e83e7dd79fcbb05b387541c5d16d9a143a3410c213e440642117b1734 checksum/ipsec-secret: d0eb9c52d0cd4311b6d252a951126bf9bea27ec05590bed8a394f0f792dcb2a4 labels: app: antrea @@ -3629,6 +3697,9 @@ spec: - name: host-var-run-antrea mountPath: /var/run/openvswitch subPath: openvswitch + - name: antrea-ipsec-ca + mountPath: /var/run/openvswitch/ca + readOnly: true # host-local IPAM stores allocated IP addresses as files in /var/lib/cni/networks/$NETWORK_NAME. # Mount a sub-directory of host-var-run-antrea to it for persistence of IP allocation. - name: host-var-run-antrea @@ -3717,6 +3788,9 @@ spec: - name: host-var-log-antrea mountPath: /var/log/strongswan subPath: strongswan + - mountPath: /etc/ipsec.d/cacerts + name: antrea-ipsec-ca + readOnly: true volumes: - name: antrea-config configMap: @@ -3733,6 +3807,9 @@ spec: - name: host-var-run-netns hostPath: path: /var/run/netns + - name: antrea-ipsec-ca + configMap: + name: antrea-ipsec-ca - name: host-var-run-antrea hostPath: path: /var/run/antrea @@ -3775,7 +3852,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: 50cc962db93c0354f5eaa088e51d690e779692979cbafac0c3e27a88fc2c0c7c + checksum/config: 2aec4c8e83e7dd79fcbb05b387541c5d16d9a143a3410c213e440642117b1734 labels: app: antrea component: antrea-controller diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 8fb2b759fa3..1e0e7358b19 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -394,6 +394,26 @@ data: # Mask size for IPv6 Node CIDR in IPv6 or dual-stack cluster. Value ignored when enableNodeIPAM is false # or when IPv6 Pod CIDR is not configured. Valid range is 64 to 126. nodeCIDRMaskSizeIPv6: 64 + + ipsecSigner: + # Enable the signer controller within the Antrea controller. + enable: false + # Determines the auto approval policy of Antrea signer for IPsec certificates management. + # It has the following options: + # never: Controller will never auto-approve CertificateSingingRequests and they need + # to be approved manually by `kubectl certificate approve` + # validateAgentPodName (default): Controller will auto-approve the CertificateSingingRequest if the creator can + # be validated. This ensures that each Agent can only request certificates for + # its own Node. This requires `BoundServiceAccountTokenVolume` feature to + # be enabled. + # always: Controller will auto-approve the CertificateSingingRequest without checking + # the identity of the creator. + autoApprovePolicy: "validateAgentPodName" + # Indicates whether to use auto-generated self-signed CA certificate. + # If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + # ca.crt: + # ca.key: + selfSignedCA: true --- # Source: antrea/templates/crds/antreaagentinfo.yaml apiVersion: apiextensions.k8s.io/v1 @@ -3486,7 +3506,7 @@ spec: kubectl.kubernetes.io/default-container: antrea-agent # Automatically restart Pods with a RollingUpdate if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f7414e9171ab246b09dc380bc7934ebac81af7a5ef7bd3f73d661b6301040768 + checksum/config: 5250e9447575a3358e4d9999ff5db22ba8670d530128619bf69bbe2483070bee labels: app: antrea component: antrea-agent @@ -3726,7 +3746,7 @@ spec: annotations: # Automatically restart Pod if the ConfigMap changes # See https://helm.sh/docs/howto/charts_tips_and_tricks/#automatically-roll-deployments - checksum/config: f7414e9171ab246b09dc380bc7934ebac81af7a5ef7bd3f73d661b6301040768 + checksum/config: 5250e9447575a3358e4d9999ff5db22ba8670d530128619bf69bbe2483070bee labels: app: antrea component: antrea-controller diff --git a/build/yamls/chart-values/antrea-ipsec.yml b/build/yamls/chart-values/antrea-ipsec.yml index d6770ea44a7..779c6df1854 100644 --- a/build/yamls/chart-values/antrea-ipsec.yml +++ b/build/yamls/chart-values/antrea-ipsec.yml @@ -1,3 +1,5 @@ trafficEncryptionMode: "ipsec" # change the tunnel type to GRE which works better with IPsec encryption than other types. tunnelType: "gre" +ipsecSigner: + enable: true \ No newline at end of file diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index eb838a97a66..149560ab48a 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -34,6 +34,7 @@ import ( "antrea.io/antrea/pkg/agent/cniserver/ipam" "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/controller/egress" + "antrea.io/antrea/pkg/agent/controller/ipseccertificate" "antrea.io/antrea/pkg/agent/controller/networkpolicy" "antrea.io/antrea/pkg/agent/controller/noderoute" "antrea.io/antrea/pkg/agent/controller/serviceexternalip" @@ -401,6 +402,11 @@ func run(o *Options) error { serviceCIDRNet) } + var ipsecCertController *ipseccertificate.Controller + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec { + ipsecCertController = ipseccertificate.NewIPSecCertificateController(k8sClient, ovsBridgeClient, nodeConfig.Name) + } + // TODO: we should call this after installing flows for initial node routes // and initial NetworkPolicies so that no packets will be mishandled. if err := agentInitializer.FlowRestoreComplete(); err != nil { @@ -644,6 +650,10 @@ func run(o *Options) error { go flowExporter.Run(stopCh) } + if networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec { + go ipsecCertController.Run(stopCh) + } + <-stopCh klog.Info("Stopping Antrea agent") return nil diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go index 121c44b8e01..3726abfa566 100644 --- a/cmd/antrea-controller/controller.go +++ b/cmd/antrea-controller/controller.go @@ -38,6 +38,8 @@ import ( "antrea.io/antrea/pkg/apiserver/storage" crdinformers "antrea.io/antrea/pkg/client/informers/externalversions" "antrea.io/antrea/pkg/clusteridentity" + config "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/controller/certificatesigningrequest" "antrea.io/antrea/pkg/controller/egress" egressstore "antrea.io/antrea/pkg/controller/egress/store" "antrea.io/antrea/pkg/controller/externalippool" @@ -115,6 +117,7 @@ func run(o *Options) error { serviceInformer := informerFactory.Core().V1().Services() networkPolicyInformer := informerFactory.Networking().V1().NetworkPolicies() nodeInformer := informerFactory.Core().V1().Nodes() + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() cnpInformer := crdInformerFactory.Crd().V1alpha1().ClusterNetworkPolicies() eeInformer := crdInformerFactory.Crd().V1alpha2().ExternalEntities() anpInformer := crdInformerFactory.Crd().V1alpha1().NetworkPolicies() @@ -175,6 +178,17 @@ func run(o *Options) error { ) } + var csrApproverController *certificatesigningrequest.CSRApproverController + var csrSignerController *certificatesigningrequest.CSRSignerController + if *o.config.IPsecSignerConfig.Enable { + _, autoApprovePolicy := config.GetAutoApprovePolicyFromStr(o.config.IPsecSignerConfig.AutoApprovePolicy) + csrApproverController = certificatesigningrequest.NewCSRApproverController(client, csrInformer, autoApprovePolicy) + csrSignerController, err = certificatesigningrequest.NewCSRSignerController(client, csrInformer, *o.config.IPsecSignerConfig.SelfSignedCertCA) + if err != nil { + return fmt.Errorf("error creating CSRSignerController: %v", err) + } + } + if features.DefaultFeatureGate.Enabled(features.Egress) { egressController = egress.NewEgressController(crdClient, groupEntityIndex, egressInformer, externalIPPoolController, egressGroupStore) } @@ -313,6 +327,11 @@ func run(o *Options) error { go antreaIPAMController.Run(stopCh) } + if *o.config.IPsecSignerConfig.Enable { + go csrApproverController.Run(stopCh) + go csrSignerController.Run(stopCh) + } + <-stopCh klog.Info("Stopping Antrea controller") return nil diff --git a/cmd/antrea-controller/options.go b/cmd/antrea-controller/options.go index f10fa9cada6..95efff45314 100644 --- a/cmd/antrea-controller/options.go +++ b/cmd/antrea-controller/options.go @@ -81,6 +81,13 @@ func (o *Options) validate(args []string) error { } } + if o.config.IPsecSignerConfig.AutoApprovePolicy != "" { + if ok, _ := controllerconfig.GetAutoApprovePolicyFromStr(o.config.IPsecSignerConfig.AutoApprovePolicy); !ok { + return fmt.Errorf("invalid field for AutoApprovePolicy %s. Supported value: %v", + o.config.IPsecSignerConfig.AutoApprovePolicy, controllerconfig.GetAutoApprovePolicies()) + } + } + if o.config.LegacyCRDMirroring != nil { klog.InfoS("The legacyCRDMirroring config option is deprecated and will be ignored (no CRD mirroring)") } @@ -161,12 +168,10 @@ func (o *Options) setDefaults() { o.config.APIPort = apis.AntreaControllerAPIPort } if o.config.EnablePrometheusMetrics == nil { - o.config.EnablePrometheusMetrics = new(bool) - *o.config.EnablePrometheusMetrics = true + o.config.EnablePrometheusMetrics = ptrBool(true) } if o.config.SelfSignedCert == nil { - o.config.SelfSignedCert = new(bool) - *o.config.SelfSignedCert = true + o.config.SelfSignedCert = ptrBool(true) } if o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv4 = ipamIPv4MaskDefault @@ -175,4 +180,14 @@ func (o *Options) setDefaults() { if o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 == 0 { o.config.NodeIPAM.NodeCIDRMaskSizeIPv6 = ipamIPv6MaskDefault } + if o.config.IPsecSignerConfig.Enable == nil { + o.config.IPsecSignerConfig.Enable = ptrBool(false) + } + if o.config.IPsecSignerConfig.SelfSignedCertCA == nil { + o.config.IPsecSignerConfig.SelfSignedCertCA = ptrBool(true) + } +} + +func ptrBool(value bool) *bool { + return &value } diff --git a/hack/generate-manifest.sh b/hack/generate-manifest.sh index 521a9663dc9..5fdcd635bfc 100755 --- a/hack/generate-manifest.sh +++ b/hack/generate-manifest.sh @@ -299,7 +299,7 @@ TMP_DIR=$(mktemp -d $THIS_DIR/../build/yamls/chart-values.XXXXXXXX) HELM_VALUES=() if $IPSEC; then - HELM_VALUES+=("trafficEncryptionMode=ipsec" "tunnelType=gre") + HELM_VALUES+=("trafficEncryptionMode=ipsec" "tunnelType=gre" "ipsecSigner.enable=true") fi if $FLEXIBLE_IPAM; then diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index c37fabd3e1b..a806adb9b9e 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -707,7 +707,7 @@ func (i *Initializer) setupDefaultTunnelInterface() error { externalIDs := map[string]interface{}{ interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaTunnel, } - tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", externalIDs) + tunnelPortUUID, err := i.ovsBridgeClient.CreateTunnelPortExt(tunnelPortName, i.networkConfig.TunnelType, config.DefaultTunOFPort, shouldEnableCsum, localIPStr, "", "", "", externalIDs) if err != nil { klog.Errorf("Failed to create tunnel port %s type %s on OVS bridge: %v", tunnelPortName, i.networkConfig.TunnelType, err) return err diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go new file mode 100644 index 00000000000..3cfa6d56865 --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller.go @@ -0,0 +1,520 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync/atomic" + "time" + + "github.com/fsnotify/fsnotify" + csrv1 "k8s.io/api/certificates/v1" + v1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + certutil "k8s.io/client-go/util/cert" + csrutil "k8s.io/client-go/util/certificate/csr" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/ovs/ovsconfig" +) + +const ( + workItemKey = "key" + controllerName = "AntreaAgentIPsecCertificateController" + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + signerName = "antrea.io/signer" +) + +// certificateWaitTimeout controls the amount of time we wait for certificate +// approval in one iteration. +var certificateWaitTimeout = 15 * time.Minute +var defaultCertificatesPath = "/var/run/openvswitch" + +// Controller is responsible for requesting certificates by CertificateSigningRequest and configure them to OVS +type Controller struct { + kubeClient clientset.Interface + ovsBridgeClient ovsconfig.OVSBridgeClient + nodeName string + queue workqueue.RateLimitingInterface + renewEventChan chan struct{} + certificateKeyPair atomic.Value + + caPath, certPath, privateKeyPath string +} + +func NewIPSecCertificateController( + kubeClient clientset.Interface, + ovsBridgeClient ovsconfig.OVSBridgeClient, + nodeName string, +) *Controller { + controller := &Controller{ + kubeClient: kubeClient, + ovsBridgeClient: ovsBridgeClient, + renewEventChan: make(chan struct{}, 1), // do not block for the first configuration sync before rotation goroutine starts. + nodeName: nodeName, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "IPsecCertificateController"), + caPath: filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), + privateKeyPath: filepath.Join(defaultCertificatesPath, fmt.Sprintf("%s.key", nodeName)), + certPath: filepath.Join(defaultCertificatesPath, fmt.Sprintf("%s.crt", nodeName)), + } + return controller +} + +// worker is a long-running function that will continually call the processNextWorkItem function in +// order to read and process a message on the workqueue. +func (c *Controller) worker() { + for c.processNextWorkItem() { + } +} + +func (c *Controller) processNextWorkItem() bool { + obj, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(obj) + if key, ok := obj.(string); !ok { + c.queue.Forget(obj) + klog.Errorf("Expected string in work queue but got %#v", obj) + return true + } else if err := c.syncConfigurations(); err == nil { + c.queue.Forget(key) + } else { + c.queue.AddRateLimited(key) + klog.Errorf("Error syncing IPSec certificates, requeuing. Error: %v", err) + } + return true +} + +type certificateKeyPair struct { + // rawCert is an optional field to determine if signing cert/key pairs have changed + rawCert []byte + // rawKey is an optional field to determine if signing cert/key pairs have changed + rawKey []byte + // rawCert is an optional field to determine if root CA have changed + rawCACert []byte + + caCertificate []*x509.Certificate + certificate []*x509.Certificate + privateKey crypto.Signer +} + +func (pair *certificateKeyPair) equals(target *certificateKeyPair) bool { + return bytes.Equal(pair.rawCACert, target.rawCACert) && + bytes.Equal(pair.rawKey, target.rawKey) && + bytes.Equal(pair.rawCert, target.rawCert) +} + +func (pair *certificateKeyPair) validate() error { + if len(pair.caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + if len(pair.certificate) == 0 { + return fmt.Errorf("certificate is empty") + } + if pair.privateKey == nil { + return fmt.Errorf("private key is empty") + } + roots := x509.NewCertPool() + for _, r := range pair.caCertificate { + roots.AddCert(r) + } + certificate := pair.certificate[0] + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + if _, err := certificate.Verify(verifyOptions); err != nil { + return err + } + switch pub := certificate.PublicKey.(type) { + case *rsa.PublicKey: + priv, ok := pair.privateKey.(*rsa.PrivateKey) + if !ok { + return fmt.Errorf("private key type does not match public key type") + } + if pub.N.Cmp(priv.N) != 0 { + return fmt.Errorf("private key does not match public key") + } + default: + return fmt.Errorf("unrecognized certificate public key type") + } + return nil +} + +var jitteryDuration = func(totalDuration float64) time.Duration { + return wait.Jitter(time.Duration(totalDuration), 0.2) - time.Duration(totalDuration*0.3) +} + +// nextRotationDeadline returns a value for the threshold at which the +// current certificate should be rotated, 80%+/-10% of the expiration of the +// certificate. +func (pair *certificateKeyPair) nextRotationDeadline() time.Time { + if err := pair.validate(); err != nil { + klog.ErrorS(err, "Verify certificate configurations failed") + return time.Now() + } + notAfter := pair.certificate[0].NotAfter + totalDuration := float64(notAfter.Sub(pair.certificate[0].NotBefore)) + deadline := pair.certificate[0].NotBefore.Add(jitteryDuration(totalDuration)) + klog.Infof("Certificate expiration is %v, rotation deadline is %v", notAfter, deadline) + return deadline +} + +func (c *Controller) syncConfigurations() error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).Infof("Finished syncing certificate configurations (%v)", d) + }() + caData, ca, err := loadRootCA(c.caPath) + if err != nil { + return err + } + keyData, key, err := loadOrCreatePrivateKey(c.privateKeyPath) + if err != nil { + return err + } + certData, cert, err := loadCertificate(c.certPath) + if err != nil { + return err + } + if _, ok := key.(crypto.Signer); !ok { + return fmt.Errorf("error reading key: key did not implement crypto.Signer") + } + newPair := &certificateKeyPair{ + rawCert: certData, + rawKey: keyData, + rawCACert: caData, + caCertificate: ca, + certificate: cert, + privateKey: key.(crypto.Signer), + } + c.certificateKeyPair.Store(newPair) + err = newPair.validate() + if err != nil { + c.renewEventChan <- struct{}{} + klog.ErrorS(err, "Current certificate configurations are not valid") + return nil + } + return c.configureOVS() +} + +func loadRootCA(caPath string) ([]byte, []*x509.Certificate, error) { + pemBlock, err := ioutil.ReadFile(caPath) + if err != nil { + return nil, nil, err + } + certs, err := certutil.ParseCertsPEM(pemBlock) + if err != nil { + return nil, nil, fmt.Errorf("error reading %s: %s", caPath, err) + } + return pemBlock, certs, nil +} + +func loadOrCreatePrivateKey(privateKeyPath string) ([]byte, crypto.PrivateKey, error) { + var keyPEMBytes []byte + _, err := os.Stat(privateKeyPath) + if err == nil { + // load the private key contents from file + keyPEMBytes, err = ioutil.ReadFile(privateKeyPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read key file %s: %v", privateKeyPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, nil, fmt.Errorf("failed to stat key file %s: %v", privateKeyPath, err) + } + if len(keyPEMBytes) > 0 { + // try to parse private key from existing file + privateKey, err := keyutil.ParsePrivateKeyPEM(keyPEMBytes) + if err != nil { + klog.ErrorS(err, "Parse key from file error", "file", privateKeyPath) + } else { + return keyPEMBytes, privateKey, nil + } + } + // The key file does not exist or has been deleted. Generate a new key. + klog.Info("Generating new private key for IPSec") + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate new RSA private key: %v", err) + } + desiredKeyBytes, err := keyutil.MarshalPrivateKeyToPEM(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal privatekey: %v", err) + } + if bytes.Equal(desiredKeyBytes, keyPEMBytes) { + return desiredKeyBytes, key, nil + } + klog.InfoS("Updating private key to file", "file", privateKeyPath) + if err = keyutil.WriteKey(privateKeyPath, desiredKeyBytes); err != nil { + return nil, nil, err + } + return desiredKeyBytes, key, nil +} + +func loadCertificate(certPath string) ([]byte, []*x509.Certificate, error) { + var certPEMBytes []byte + _, err := os.Stat(certPath) + if err == nil { + // load the certificate from file + certPEMBytes, err = ioutil.ReadFile(certPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read certificate file %s: %v", certPath, err) + } + } else if err != nil && !os.IsNotExist(err) { + return nil, nil, fmt.Errorf("failed to stat certificate file %s: %v", certPath, err) + } + if len(certPEMBytes) > 0 { + // try to parse the certificate from the existing file + certificates, err := certutil.ParseCertsPEM(certPEMBytes) + if err != nil { + klog.ErrorS(err, "Parse certificate from file error", "file", certPath) + } else { + return certPEMBytes, certificates, nil + } + } + return nil, nil, nil +} + +func (c *Controller) configureOVS() error { + // calculate the hash of current configurations and save it to OVS DB to trigger updates. + hasher := sha256.New() + existing := c.certificateKeyPair.Load().(*certificateKeyPair) + hasher.Write(existing.rawCACert) + hasher.Write(existing.rawCert) + hasher.Write(existing.rawKey) + hash := hex.EncodeToString(hasher.Sum(nil)) + + ovsConfig := map[string]interface{}{ + "certificate": c.certPath, + "private_key": c.privateKeyPath, + "ca_cert": c.caPath, + "certificate_hash": hash, + } + oldConfig, err := c.ovsBridgeClient.GetOVSOtherConfig() + if err != nil { + return err + } + // delete the old config if needed + toDelete := make(map[string]interface{}) + toAdd := make(map[string]interface{}) + for k, v := range ovsConfig { + oldValue, ok := oldConfig[k] + if ok && oldValue != v { + toDelete[k] = oldConfig[k] + } + if oldValue == v { + continue + } + toAdd[k] = v + } + if len(toDelete) > 0 { + if err := c.ovsBridgeClient.DeleteOVSOtherConfig(toDelete); err != nil { + return err + } + } + if len(toAdd) == 0 { + return nil + } + klog.InfoS("Updating OVS configurations for IPsec certificates") + return c.ovsBridgeClient.AddOVSOtherConfig(ovsConfig) +} + +func newCSR(csrName, commonName string, privateKey crypto.PrivateKey) (*csrv1.CertificateSigningRequest, error) { + subject := &pkix.Name{ + CommonName: commonName, + Organization: []string{"antrea.io"}, + } + csrBytes, err := certutil.MakeCSR(privateKey, subject, []string{commonName}, nil) + if err != nil { + return nil, err + } + return &csrv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: csrName, + }, + Spec: csrv1.CertificateSigningRequestSpec{ + Request: csrBytes, + SignerName: signerName, + Usages: []v1.KeyUsage{v1.UsageIPsecTunnel}, + }, + }, nil +} + +func (c *Controller) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.Infof("Starting %s", controllerName) + defer klog.Infof("Shutting down %s", controllerName) + + // load the existing certificates from files before starting the rotation goroutine. + if err := c.syncConfigurations(); err != nil { + klog.ErrorS(err, "Sync configurations error") + } + + go wait.Until(c.worker, time.Second, stopCh) + + go wait.Until(func() { + if err := c.watchFileChanges(stopCh); err != nil { + klog.ErrorS(err, "Failed to watch cert and key file, will retry later") + } + }, time.Minute, stopCh) + + go wait.Until(func() { + if err := c.rotateCertificates(stopCh); err != nil { + klog.ErrorS(err, "Failed to rotate certificates") + } + }, time.Minute, stopCh) + <-stopCh +} + +func (c *Controller) rotateCertificates(endCh <-chan struct{}) error { + for { + currentCert := c.certificateKeyPair.Load().(*certificateKeyPair) + deadline := currentCert.nextRotationDeadline() + if sleepInterval := time.Until(deadline); sleepInterval > 0 { + timer := time.NewTimer(sleepInterval) + defer timer.Stop() + select { + case <-endCh: + return nil + case <-c.renewEventChan: + // Received update event. Check the next rotation deadline again. + timer.Stop() + continue + case <-timer.C: + } + } + klog.InfoS("Starting rotating certificates") + csrName := fmt.Sprintf("%s-ipsec", c.nodeName) + err := c.kubeClient.CertificatesV1().CertificateSigningRequests().Delete(context.TODO(), csrName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete previous CSR: %v", csrName) + } + csr, err := newCSR(csrName, c.nodeName, currentCert.privateKey) + if err != nil { + return err + } + csr, err = c.kubeClient.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), csr, metav1.CreateOptions{}) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), certificateWaitTimeout) + defer cancel() + issuedCertificate, err := csrutil.WaitForCertificate(ctx, c.kubeClient, csr.Name, csr.UID) + if err != nil { + return err + } + err = c.updateCertificates(issuedCertificate) + if err != nil { + klog.ErrorS(err, "Failed to update new certificates") + return err + } + klog.InfoS("Created new certificates for IPSec") + } +} + +func (c *Controller) updateCertificates(data []byte) error { + cert, err := certutil.ParseCertsPEM(data) + if err != nil { + return fmt.Errorf("failed to parse certificate: %w", err) + } + if err := certutil.WriteCert(c.certPath, data); err != nil { + return err + } + existing := c.certificateKeyPair.Load().(*certificateKeyPair) + newCertPair := &certificateKeyPair{ + rawCert: data, + certificate: cert, + rawKey: existing.rawKey, + privateKey: existing.privateKey, + rawCACert: existing.rawCACert, + caCertificate: existing.caCertificate, + } + c.certificateKeyPair.Store(newCertPair) + c.queue.Add(workItemKey) + return nil +} + +func (c *Controller) watchFileChanges(stopCh <-chan struct{}) error { + w, err := fsnotify.NewWatcher() + if err != nil { + return fmt.Errorf("error creating fsnotify watcher: %v", err) + } + defer w.Close() + + c.queue.Add(workItemKey) + + if err := w.Add(c.caPath); err != nil { + return fmt.Errorf("error adding watch for file %s: %v", c.caPath, err) + } + if err := w.Add(c.privateKeyPath); err != nil { + return fmt.Errorf("error adding watch for file %s: %v", c.privateKeyPath, err) + } + if err := w.Add(c.certPath); err != nil { + return fmt.Errorf("error adding watch for file %s: %v", c.certPath, err) + } + // Trigger a check in case the file is updated before the watch starts. + c.queue.Add(workItemKey) + + for { + select { + case e := <-w.Events: + if err := c.handleWatchEvent(e, w); err != nil { + return err + } + case err := <-w.Errors: + return fmt.Errorf("received fsnotify error: %v", err) + case <-stopCh: + return nil + } + } +} + +func (c *Controller) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error { + // This should be executed after restarting the watch (if applicable) to ensure no file event will be missing. + defer c.queue.Add(workItemKey) + if e.Op&(fsnotify.Remove|fsnotify.Rename) == 0 { + return nil + } + if err := w.Remove(e.Name); err != nil { + klog.InfoS("Failed to remove file watch, it may have been deleted", "file", e.Name, "err", err) + } + if err := w.Add(e.Name); err != nil { + return fmt.Errorf("error adding watch for file %s: %v", e.Name, err) + } + return nil +} diff --git a/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go new file mode 100644 index 00000000000..dcec54b08c2 --- /dev/null +++ b/pkg/agent/controller/ipseccertificate/ipsec_certificate_controller_test.go @@ -0,0 +1,268 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ipseccertificate + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "fmt" + "io/ioutil" + "math/big" + "os" + "path/filepath" + "testing" + "time" + + mock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + + ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" +) + +const fakeNodeName = "fake-node-1" + +type fakeController struct { + *Controller + mockController *mock.Controller + mockBridgeClient *ovsconfigtest.MockOVSBridgeClient + rawCAcert []byte + caCert *x509.Certificate + caKey crypto.Signer +} + +func newFakeController(t *testing.T) *fakeController { + mockController := mock.NewController(t) + mockOVSBridgeClient := ovsconfigtest.NewMockOVSBridgeClient(mockController) + fakeClient := fake.NewSimpleClientset() + originDefaultPath := defaultCertificatesPath + cfg := certutil.Config{ + CommonName: "antrea-ipsec-ca", + Organization: []string{"antrea.io"}, + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + rootCA, err := certutil.NewSelfSignedCACert(cfg, key) + require.NoError(t, err) + tempDir, err := ioutil.TempDir("", "antrea-ipsec-test") + require.NoError(t, err) + defaultCertificatesPath = tempDir + defer func() { + defaultCertificatesPath = originDefaultPath + }() + caData, err := certutil.EncodeCertificates(rootCA) + require.NoError(t, err) + err = certutil.WriteCert(filepath.Join(defaultCertificatesPath, "ca", "ca.crt"), caData) + require.NoError(t, err) + + c := NewIPSecCertificateController(fakeClient, mockOVSBridgeClient, fakeNodeName) + return &fakeController{ + Controller: c, + mockController: mockController, + mockBridgeClient: mockOVSBridgeClient, + rawCAcert: caData, + caCert: rootCA, + caKey: key, + } +} + +func TestController_syncConfigurations(t *testing.T) { + t.Run("should create necessary files on fresh start", func(t *testing.T) { + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + err := fakeController.syncConfigurations() + require.NoError(t, err) + // should notify certificate renew goroutine + <-fakeController.renewEventChan + // should create private key on fresh start + keyBytes, err := ioutil.ReadFile(fakeController.privateKeyPath) + require.NoError(t, err) + key, err := keyutil.ParsePrivateKeyPEM(keyBytes) + require.NoError(t, err) + pKey, ok := key.(crypto.Signer) + assert.True(t, ok) + // should not create any certificate files + _, err = os.Stat(fakeController.certPath) + assert.True(t, os.IsNotExist(err)) + + pair, ok := fakeController.certificateKeyPair.Load().(*certificateKeyPair) + assert.True(t, ok) + assert.Len(t, pair.caCertificate, 1) + assert.Equal(t, pair.rawCACert, fakeController.rawCAcert) + assert.NotNil(t, pair.privateKey) + assert.Equal(t, pair.rawKey, keyBytes) + assert.Empty(t, pair.certificate) + assert.Empty(t, pair.rawCert) + + template := newIPsecCertTemplate(t, fakeNodeName) + derBytes, err := x509.CreateCertificate(rand.Reader, template, fakeController.caCert, pKey.Public(), fakeController.caKey) + assert.NoError(t, err) + certs, err := x509.ParseCertificates(derBytes) + assert.NoError(t, err) + assert.Len(t, certs, 1) + encoded, err := certutil.EncodeCertificates(certs...) + assert.NoError(t, err) + err = fakeController.updateCertificates(encoded) + assert.NoError(t, err) + + hasher := sha256.New() + hasher.Write(fakeController.rawCAcert) + hasher.Write(encoded) + hasher.Write(keyBytes) + hash := hex.EncodeToString(hasher.Sum(nil)) + + expectedOVSConfig := map[string]interface{}{ + "certificate": fakeController.certPath, + "private_key": fakeController.privateKeyPath, + "ca_cert": fakeController.caPath, + "certificate_hash": hash, + } + + fakeController.mockBridgeClient.EXPECT().GetOVSOtherConfig().Return(map[string]string{}, nil).Times(1) + fakeController.mockBridgeClient.EXPECT().AddOVSOtherConfig(expectedOVSConfig).Times(1) + err = fakeController.syncConfigurations() + assert.NoError(t, err) + + returnedOVSConfig := map[string]string{} + for k, v := range expectedOVSConfig { + returnedOVSConfig[k] = v.(string) + } + // sync configuration again should not change any OVS configurations. + fakeController.mockBridgeClient.EXPECT().GetOVSOtherConfig().Return(returnedOVSConfig, nil).Times(1) + err = fakeController.syncConfigurations() + assert.NoError(t, err) + }) +} + +func newIPsecCertTemplate(t *testing.T, nodeName string) *x509.Certificate { + return &x509.Certificate{ + Subject: pkix.Name{ + CommonName: nodeName, + Organization: []string{"antrea.io"}, + }, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(365 * 24 * time.Hour), // 1 year + SerialNumber: big.NewInt(12345), + DNSNames: []string{nodeName}, + BasicConstraintsValid: true, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } +} + +func TestController_rotateCertificates(t *testing.T) { + t.Run("should rotate certificates on fresh start", func(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + err := fakeController.syncConfigurations() + assert.NoError(t, err) + go fakeController.rotateCertificates(ch) + err = wait.PollImmediate(200*time.Millisecond, 3*time.Second, func() (bool, error) { + _, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), + fmt.Sprintf("%s-ipsec", fakeNodeName), metav1.GetOptions{}) + if err != nil { + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + }) + t.Run("should update signed CSR", func(t *testing.T) { + ch := make(chan struct{}) + defer close(ch) + fakeController := newFakeController(t) + defer fakeController.mockController.Finish() + err := fakeController.syncConfigurations() + assert.NoError(t, err) + go fakeController.rotateCertificates(ch) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + watcher, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for ev := range watcher.ResultChan() { + switch ev.Type { + case watch.Added: + csr, ok := ev.Object.(*v1.CertificateSigningRequest) + assert.True(t, ok) + signCSR(t, fakeController, csr) + } + } + }() + err = wait.PollImmediate(200*time.Millisecond, 3*time.Second, func() (bool, error) { + _, err := fakeController.kubeClient.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), + fmt.Sprintf("%s-ipsec", fakeNodeName), metav1.GetOptions{}) + if err != nil { + return false, nil + } + if fakeController.queue.Len() == 0 { + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + updated := fakeController.certificateKeyPair.Load().(*certificateKeyPair) + assert.NotEmpty(t, updated.certificate) + assert.NotEmpty(t, updated.rawCert) + }) +} + +func signCSR(t *testing.T, controller *fakeController, csr *v1.CertificateSigningRequest) { + csr = csr.DeepCopy() + assert.Empty(t, csr.Status.Certificate) + block, remain := pem.Decode(csr.Spec.Request) + assert.Empty(t, remain) + req, err := x509.ParseCertificateRequest(block.Bytes) + assert.NoError(t, err) + template := newIPsecCertTemplate(t, req.Subject.CommonName) + derBytes, err := x509.CreateCertificate(rand.Reader, template, controller.caCert, + req.PublicKey, controller.caKey) + assert.NoError(t, err) + certs, err := x509.ParseCertificates(derBytes) + assert.NoError(t, err) + assert.Len(t, certs, 1) + encoded, err := certutil.EncodeCertificates(certs...) + assert.NoError(t, err) + csr.Status.Conditions = append(csr.Status.Conditions, v1.CertificateSigningRequestCondition{ + Type: v1.CertificateApproved, + Status: corev1.ConditionTrue, + }) + csr, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{}) + assert.NoError(t, err) + csr.Status.Certificate = encoded + _, err = controller.kubeClient.CertificatesV1().CertificateSigningRequests(). + UpdateStatus(context.TODO(), csr, metav1.UpdateOptions{}) + assert.NoError(t, err) +} diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index ece61e3c475..14ee82a23dc 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -653,7 +653,8 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 false, "", nodeIP.String(), - c.networkConfig.IPSecPSK, + nodeName, + "", ovsExternalIDs) if err != nil { return 0, fmt.Errorf("failed to create IPsec tunnel port for Node %s", nodeName) diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index 7932c66b05e..ed77d0c6118 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -332,11 +332,11 @@ func TestCreateIPSecTunnelPort(t *testing.T) { node2PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-2") c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP1.String(), "changeme", + false, "", nodeIP1.String(), "xyz-k8s-0-1", "", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) c.ovsClient.EXPECT().CreateTunnelPortExt( node2PortName, ovsconfig.TunnelType("vxlan"), int32(0), - false, "", nodeIP2.String(), "changeme", + false, "", nodeIP2.String(), "xyz-k8s-0-2", "", map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2"}).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) c.ovsClient.EXPECT().GetOFPort(node2PortName, false).Return(int32(2), nil) diff --git a/pkg/config/controller/auto_approve_policy.go b/pkg/config/controller/auto_approve_policy.go new file mode 100644 index 00000000000..d5a083f8f50 --- /dev/null +++ b/pkg/config/controller/auto_approve_policy.go @@ -0,0 +1,57 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package config + +import "strings" + +type AutoApprovePolicy int + +const ( + AutoApprovePolicyNever AutoApprovePolicy = iota + AutoApprovePolicyValidateAgentPodName + AutoApprovePolicyAlways + AutoApprovePolicyInvalid = -1 +) + +var ( + approvePolicyModeStrs = [...]string{ + "never", + "validateAgentPodName", + "always", + } +) + +// GetAutoApprovePolicyFromStr returns true and AutoApprovePolicy corresponding to input string. +// Otherwise, false and undefined value is returned +func GetAutoApprovePolicyFromStr(str string) (bool, AutoApprovePolicy) { + for idx, ms := range approvePolicyModeStrs { + if strings.EqualFold(ms, str) { + return true, AutoApprovePolicy(idx) + } + } + return false, AutoApprovePolicyInvalid +} + +// String returns value in string. +func (m AutoApprovePolicy) String() string { + return approvePolicyModeStrs[m] +} + +func GetAutoApprovePolicies() []AutoApprovePolicy { + return []AutoApprovePolicy{ + AutoApprovePolicyNever, + AutoApprovePolicyValidateAgentPodName, + AutoApprovePolicyAlways, + } +} diff --git a/pkg/config/controller/config.go b/pkg/config/controller/config.go index 643d7e476cb..34ab3ebeb00 100644 --- a/pkg/config/controller/config.go +++ b/pkg/config/controller/config.go @@ -65,4 +65,20 @@ type ControllerConfig struct { LegacyCRDMirroring *bool `yaml:"legacyCRDMirroring,omitempty"` // NodeIPAM Configuration NodeIPAM NodeIPAMConfig `yaml:"nodeIPAM"` + // IPsec certificate signer configuration + IPsecSignerConfig IPsecSignerConfig `yaml:"ipsecSigner"` +} + +type IPsecSignerConfig struct { + // Enable the signer controller within the Antrea controller. + // Defaults to false. + Enable *bool `yaml:"enable,omitempty"` + // Indicates whether to use auto-generated self-signed CA certificate. + // If false, a Secret named "antrea-ipsec-ca" must be provided with the following keys: + // ca.crt: + // ca.key: + // Defaults to true. + SelfSignedCertCA *bool `yaml:"selfSignedCA,omitempty"` + // Antrea signer auto approval policy. + AutoApprovePolicy string `yaml:"autoApprovePolicy"` } diff --git a/pkg/controller/certificatesigningrequest/approver_controller.go b/pkg/controller/certificatesigningrequest/approver_controller.go new file mode 100644 index 00000000000..a200fd65371 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approver_controller.go @@ -0,0 +1,335 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "fmt" + "reflect" + "strings" + + "encoding/pem" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + sautil "k8s.io/apiserver/pkg/authentication/serviceaccount" + csrinformer "k8s.io/client-go/informers/certificates/v1" + clientset "k8s.io/client-go/kubernetes" + csrlister "k8s.io/client-go/listers/certificates/v1" + "k8s.io/client-go/tools/cache" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + config "antrea.io/antrea/pkg/config/controller" + "antrea.io/antrea/pkg/util/env" +) + +const ( + approverControllerName = "CertificateSigningRequestApproverController" +) + +var ( + errOrganizationNotAntrea = fmt.Errorf("subject organization is not %s", antreaOrganization) + errDNSSANNotMatchCommonName = fmt.Errorf("DNS subjectAltNames do not match subject common name") + errEmailSANNotAllowed = fmt.Errorf("email subjectAltNames are not allowed") + errIPSANNotAllowed = fmt.Errorf("IP subjectAltNames are not allowed") + errURISANNotAllowed = fmt.Errorf("URI subjectAltNames are not allowed") + errCommonNameRequired = fmt.Errorf("subject common name is required") + errExtraFieldsRequired = fmt.Errorf("extra values must contain %q and %q", sautil.PodNameKey, sautil.PodUIDKey) + errPodUIDMismatch = fmt.Errorf("Pod UID does not match") + errPodNotOnNode = fmt.Errorf("Pod is not on requested Node") +) + +type approver interface { + recognize(csr *certificatesv1.CertificateSigningRequest) bool + verify(csr *certificatesv1.CertificateSigningRequest) error +} + +// CSRApproverController is responsible for synchronizing the CertificateSigningRequest resources. +type CSRApproverController struct { + client clientset.Interface + csrLister csrlister.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + autoApprovePolicy config.AutoApprovePolicy + queue workqueue.RateLimitingInterface + approvers []approver +} + +// NewCSRApproverController returns a new *CSRApproverController. +func NewCSRApproverController(client clientset.Interface, csrInformer csrinformer.CertificateSigningRequestInformer, approvePolicy config.AutoApprovePolicy) *CSRApproverController { + c := &CSRApproverController{ + client: client, + csrLister: csrInformer.Lister(), + csrListerSynced: csrInformer.Informer().HasSynced, + autoApprovePolicy: approvePolicy, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + approvers: []approver{ + &ipsecCertificateApprover{ + client: client, + approvePolicy: approvePolicy, + }, + }, + } + csrInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + UpdateFunc: func(old, cur interface{}) { + c.enqueueCertificateSigningRequest(cur) + }, + DeleteFunc: c.enqueueCertificateSigningRequest, + }, + resyncPeriod, + ) + return c +} + +// Run begins watching and syncing of the CSRApproverController. +func (c *CSRApproverController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.Infof("Starting %s", approverControllerName) + defer klog.Infof("Shutting down %s", approverControllerName) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced} + if !cache.WaitForNamedCacheSync(approverControllerName, stopCh, cacheSyncs...) { + return + } + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.worker, time.Second, stopCh) + } + <-stopCh +} + +func (c *CSRApproverController) worker() { + for c.processNextWorkItem() { + } +} + +func (c *CSRApproverController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + deletedState, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Received unexpected object: %v", obj) + return + } + csr, ok = deletedState.Obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + klog.Errorf("DeletedFinalStateUnknown contains non-CertificateSigningRequest object: %v", deletedState.Obj) + return + } + } + c.queue.Add(csr.Name) +} + +func (c *CSRApproverController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).Infof("Finished syncing CertificateSigningRequest %s. (%v)", key, d) + }() + + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + if approved, denied := getCertApprovalCondition(&csr.Status); approved || denied { + return nil + } + for _, a := range c.approvers { + if a.recognize(csr) { + return a.verify(csr) + } + } + klog.Warningf("unrecognized CertificateSigningRequest %s", key) + return nil +} + +func appendApprovalCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: corev1.ConditionTrue, + Reason: "AutoApproved", + Message: message, + }) +} + +func appendDeniedCondition(csr *certificatesv1.CertificateSigningRequest, message string) { + csr.Status.Conditions = append(csr.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateDenied, + Status: corev1.ConditionTrue, + Reason: "ValidationFailed", + Message: message, + }) +} + +func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) { + block, _ := pem.Decode(pemBytes) + if block == nil || block.Type != certutil.CertificateRequestBlockType { + err := fmt.Errorf("PEM block type must be %s", certutil.CertificateRequestBlockType) + return nil, err + } + return x509.ParseCertificateRequest(block.Bytes) +} + +func (c *CSRApproverController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + // Put the item back in the workqueue to handle any transient errors. + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} + +type ipsecCertificateApprover struct { + client clientset.Interface + approvePolicy config.AutoApprovePolicy +} + +var _ approver = (*ipsecCertificateApprover)(nil) + +func (ic *ipsecCertificateApprover) recognize(csr *certificatesv1.CertificateSigningRequest) bool { + if csr.Spec.SignerName != antreaSignerName { + return false + } + for _, usage := range csr.Spec.Usages { + if !ipsecTunnelUsages.Has(string(usage)) { + return false + } + } + return true +} + +func (ic *ipsecCertificateApprover) verify(csr *certificatesv1.CertificateSigningRequest) error { + // Do not approve or deny the requests if AutoApprovePolicy is set to Never. + // User should approve or deny the requests manually to get the CSR signed. + if ic.approvePolicy == config.AutoApprovePolicyNever { + return nil + } + var deniedReasons []string + cr, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + return err + } + if err := verifyIPSecCSR(ic.client, cr, csr.Spec.Usages); err != nil { + if _, ok := err.(*transientError); ok { + return err + } else { + deniedReasons = append(deniedReasons, err.Error()) + } + } + if err := verifyPodOnNode(ic.client, cr.Subject.CommonName, csr); err != nil { + if _, ok := err.(*transientError); ok { + return err + } + if ic.approvePolicy == config.AutoApprovePolicyValidateAgentPodName { + deniedReasons = append(deniedReasons, err.Error()) + } else { + klog.Warningf("Verify CSR %s failed: %v. Ignore the error due to ApprovePolicy", csr.Name, err) + } + } + if len(deniedReasons) > 0 { + appendDeniedCondition(csr, strings.Join(deniedReasons, "; ")) + } else { + appendApprovalCondition(csr, fmt.Sprintf("Automatically approved by %s", antreaSignerName)) + } + _, err = ic.client.CertificatesV1().CertificateSigningRequests().UpdateApproval(context.TODO(), csr.Name, csr, metav1.UpdateOptions{}) + return err +} + +type transientError struct { + error +} + +var ipsecTunnelUsages = sets.NewString( + string(certificatesv1.UsageIPsecTunnel), +) + +func verifyIPSecCSR(client clientset.Interface, req *x509.CertificateRequest, usages []certificatesv1.KeyUsage) error { + if !reflect.DeepEqual(req.Subject.Organization, []string{antreaOrganization}) { + return errOrganizationNotAntrea + } + if req.Subject.CommonName == "" { + return errCommonNameRequired + } + if len(req.URIs) > 0 { + return errURISANNotAllowed + } + if len(req.IPAddresses) > 0 { + return errIPSANNotAllowed + } + if len(req.EmailAddresses) > 0 { + return errEmailSANNotAllowed + } + if !reflect.DeepEqual([]string{req.Subject.CommonName}, req.DNSNames) { + return errDNSSANNotMatchCommonName + } + for _, u := range usages { + if !ipsecTunnelUsages.Has(string(u)) { + return fmt.Errorf("unsupported key usage: %v", u) + } + } + _, err := client.CoreV1().Nodes().Get(context.TODO(), req.Subject.CommonName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("requested Node %s not found", req.Subject.CommonName) + } else if err != nil { + return &transientError{err} + } + return nil +} + +func verifyPodOnNode(client clientset.Interface, nodeName string, csr *certificatesv1.CertificateSigningRequest) error { + podNameValues, podUIDValues := csr.Spec.Extra[sautil.PodNameKey], csr.Spec.Extra[sautil.PodUIDKey] + if len(podNameValues) == 0 || len(podUIDValues) == 0 { + return errExtraFieldsRequired + } + podName, podUID := podNameValues[0], podUIDValues[0] + if podName == "" || podUID == "" { + return errExtraFieldsRequired + } + pod, err := client.CoreV1().Pods(env.GetAntreaNamespace()).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil && apierrors.IsNotFound(err) { + return fmt.Errorf("Pod %s not found", podName) + } else if err != nil { + return &transientError{err} + } + if pod.ObjectMeta.UID != types.UID(podUID) { + return errPodUIDMismatch + } + if pod.Spec.NodeName != nodeName { + return errPodNotOnNode + } + return nil +} diff --git a/pkg/controller/certificatesigningrequest/approver_controller_test.go b/pkg/controller/certificatesigningrequest/approver_controller_test.go new file mode 100644 index 00000000000..2aa7cbe1761 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/approver_controller_test.go @@ -0,0 +1,599 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "net" + "net/url" + "testing" + + config "antrea.io/antrea/pkg/config/controller" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func Test_validIPSecCSR(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + cr *x509.CertificateRequest + keyUsages []certificatesv1.KeyUsage + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: nil, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Organization missing", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errOrganizationNotAntrea, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "requested Node not found", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("requested Node worker-node-1 not found"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "DNS SAN not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + }, + expectedErr: errDNSSANNotMatchCommonName, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "key usages not match", + objects: []runtime.Object{}, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errors.New("unsupported key usage: client auth"), + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageClientAuth, + }, + }, + { + name: "IP SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + IPAddresses: []net.IP{net.ParseIP("1.2.3.4")}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errIPSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "URI SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + URIs: []*url.URL{ + {Host: "antrea.io"}, + }, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errURISANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + { + name: "Email SAN should not be permitted", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + }, + cr: &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + EmailAddresses: []string{"user@antrea.io"}, + DNSNames: []string{"worker-node-1"}, + }, + expectedErr: errEmailSANNotAllowed, + keyUsages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + err := verifyIPSecCSR(client, tt.cr, tt.keyUsages) + if tt.expectedErr == nil { + assert.NoError(t, err, "validIPSecCSR should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "validIPSecCSR should return an error") + } + }) + } +} + +func Test_verifyPodOnNode(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + nodeName string + csr *certificatesv1.CertificateSigningRequest + expectedErr error + }{ + { + name: "valid CSR", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + }, + }, + nodeName: "worker-node-1", + expectedErr: nil, + }, + { + name: "Pod UID mismatch", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"7afec259-ba03-441d-adeb-be163da2da2c"}, + }, + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodUIDMismatch, + }, + { + name: "extra fields missing", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{}, + }, + }, + nodeName: "worker-node-1", + expectedErr: errExtraFieldsRequired, + }, + { + name: "Pod is not on requested Node", + objects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-2", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + }, + }, + nodeName: "worker-node-1", + expectedErr: errPodNotOnNode, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + err := verifyPodOnNode(client, tt.nodeName, tt.csr) + if tt.expectedErr == nil { + assert.NoError(t, err, "verifyPodOnNode should not return an error") + } else { + assert.EqualError(t, err, tt.expectedErr.Error(), "verifyPodOnNode should return an error") + } + }) + } +} + +func Test_ipsecCertificateApprover_recognize(t *testing.T) { + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + approvePolicy config.AutoApprovePolicy + expectedResult bool + }{ + { + name: "valid IPsec CSR", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedResult: true, + }, + { + name: "Unknown key usages", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + certificatesv1.UsageDigitalSignature, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedResult: false, + }, + { + name: "Unknown signer name", + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + SignerName: "k8s.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedResult: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client := fake.NewSimpleClientset(tt.objects...) + ic := &ipsecCertificateApprover{ + client: client, + approvePolicy: tt.approvePolicy, + } + recognized := ic.recognize(tt.csr) + assert.Equal(t, tt.expectedResult, recognized) + }) + } +} + +func x509CRtoPEM(t *testing.T, cr *x509.CertificateRequest) (crypto.PrivateKey, []byte) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + crDer, err := x509.CreateCertificateRequest(rand.Reader, cr, privateKey) + require.NoError(t, err) + csrPemBlock := &pem.Block{ + Type: certutil.CertificateRequestBlockType, + Bytes: crDer, + } + crBytes := pem.EncodeToMemory(csrPemBlock) + assert.NotEmpty(t, crBytes) + return privateKey, crBytes +} + +func Test_ipsecCertificateApprover_verify(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + approvePolicy config.AutoApprovePolicy + expectedError error + expectedApproved, expectedDenied bool + }{ + { + name: "valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedApproved: true, + expectedDenied: false, + }, + { + name: "CSR missing ExtraValue", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{}, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedApproved: false, + expectedDenied: true, + }, + { + name: "CSR missing ExtraValue and config.AutoApprovePolicy setting to always", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{}, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyAlways, + expectedApproved: true, + expectedDenied: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objs := append(tt.objects, tt.csr) + client := fake.NewSimpleClientset(objs...) + ic := &ipsecCertificateApprover{ + client: client, + approvePolicy: tt.approvePolicy, + } + err := ic.verify(tt.csr) + if tt.expectedError != nil { + assert.EqualError(t, err, tt.expectedError.Error()) + assert.Equal(t, nil, tt.csr.Status) + } else { + assert.NoError(t, err) + updated, err := client.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csr.Name, metav1.GetOptions{}) + require.NoError(t, err) + approved, denied := getCertApprovalCondition(&updated.Status) + assert.Equal(t, tt.expectedApproved, approved) + assert.Equal(t, tt.expectedDenied, denied) + } + }) + } +} diff --git a/pkg/controller/certificatesigningrequest/common.go b/pkg/controller/certificatesigningrequest/common.go new file mode 100644 index 00000000000..2a37acf61c5 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/common.go @@ -0,0 +1,131 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "crypto/x509" + "fmt" + "sort" + "time" + + certificates "k8s.io/api/certificates/v1" +) + +const ( + // Set resyncPeriod to 0 to disable resyncing. + resyncPeriod time.Duration = 0 + // How long to wait before retrying the processing of a CertificateSigningRequest change. + minRetryDelay = 5 * time.Second + maxRetryDelay = 300 * time.Second + // Default number of workers processing a CertificateSigningRequest change. + defaultWorkers = 4 + // CertificateBlockType is a possible value for pem.Block.Type. + certificateBlockType = "CERTIFICATE" + antreaSignerName = "antrea.io/signer" + antreaOrganization = "antrea.io" +) + +// isCertificateRequestApproved returns true if a certificate request has the +// "Approved" condition and no "Denied" conditions; false otherwise. +func isCertificateRequestApproved(csr *certificates.CertificateSigningRequest) bool { + approved, denied := getCertApprovalCondition(&csr.Status) + return approved && !denied +} + +func getCertApprovalCondition(status *certificates.CertificateSigningRequestStatus) (bool, bool) { + var approved, denied bool + for _, c := range status.Conditions { + if c.Type == certificates.CertificateApproved { + approved = true + } + if c.Type == certificates.CertificateDenied { + denied = true + } + } + return approved, denied +} + +var keyUsageDict = map[certificates.KeyUsage]x509.KeyUsage{ + certificates.UsageSigning: x509.KeyUsageDigitalSignature, + certificates.UsageDigitalSignature: x509.KeyUsageDigitalSignature, + certificates.UsageContentCommitment: x509.KeyUsageContentCommitment, + certificates.UsageKeyEncipherment: x509.KeyUsageKeyEncipherment, + certificates.UsageKeyAgreement: x509.KeyUsageKeyAgreement, + certificates.UsageDataEncipherment: x509.KeyUsageDataEncipherment, + certificates.UsageCertSign: x509.KeyUsageCertSign, + certificates.UsageCRLSign: x509.KeyUsageCRLSign, + certificates.UsageEncipherOnly: x509.KeyUsageEncipherOnly, + certificates.UsageDecipherOnly: x509.KeyUsageDecipherOnly, +} + +var extKeyUsageDict = map[certificates.KeyUsage]x509.ExtKeyUsage{ + certificates.UsageAny: x509.ExtKeyUsageAny, + certificates.UsageServerAuth: x509.ExtKeyUsageServerAuth, + certificates.UsageClientAuth: x509.ExtKeyUsageClientAuth, + certificates.UsageCodeSigning: x509.ExtKeyUsageCodeSigning, + certificates.UsageEmailProtection: x509.ExtKeyUsageEmailProtection, + certificates.UsageSMIME: x509.ExtKeyUsageEmailProtection, + certificates.UsageIPsecEndSystem: x509.ExtKeyUsageIPSECEndSystem, + certificates.UsageIPsecTunnel: x509.ExtKeyUsageIPSECTunnel, + certificates.UsageIPsecUser: x509.ExtKeyUsageIPSECUser, + certificates.UsageTimestamping: x509.ExtKeyUsageTimeStamping, + certificates.UsageOCSPSigning: x509.ExtKeyUsageOCSPSigning, + certificates.UsageMicrosoftSGC: x509.ExtKeyUsageMicrosoftServerGatedCrypto, + certificates.UsageNetscapeSGC: x509.ExtKeyUsageNetscapeServerGatedCrypto, +} + +// keyUsagesFromStrings will translate a slice of usage strings from the +// certificates API ("pkg/apis/certificates".KeyUsage) to x509.KeyUsage and +// x509.ExtKeyUsage types. +func keyUsagesFromStrings(usages []certificates.KeyUsage) (x509.KeyUsage, []x509.ExtKeyUsage, error) { + var keyUsage x509.KeyUsage + var unrecognized []certificates.KeyUsage + extKeyUsages := make(map[x509.ExtKeyUsage]struct{}) + for _, usage := range usages { + if val, ok := keyUsageDict[usage]; ok { + keyUsage |= val + } else if val, ok := extKeyUsageDict[usage]; ok { + extKeyUsages[val] = struct{}{} + } else { + unrecognized = append(unrecognized, usage) + } + } + + var sorted sortedExtKeyUsage + for eku := range extKeyUsages { + sorted = append(sorted, eku) + } + sort.Sort(sorted) + + if len(unrecognized) > 0 { + return 0, nil, fmt.Errorf("unrecognized usage values: %q", unrecognized) + } + + return keyUsage, sorted, nil +} + +type sortedExtKeyUsage []x509.ExtKeyUsage + +func (s sortedExtKeyUsage) Len() int { + return len(s) +} + +func (s sortedExtKeyUsage) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s sortedExtKeyUsage) Less(i, j int) bool { + return s[i] < s[j] +} diff --git a/pkg/controller/certificatesigningrequest/signer_controller.go b/pkg/controller/certificatesigningrequest/signer_controller.go new file mode 100644 index 00000000000..61e4577dada --- /dev/null +++ b/pkg/controller/certificatesigningrequest/signer_controller.go @@ -0,0 +1,474 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "reflect" + "sync/atomic" + "time" + + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/util/wait" + csrinformer "k8s.io/client-go/informers/certificates/v1" + corev1informers "k8s.io/client-go/informers/core/v1" + clientset "k8s.io/client-go/kubernetes" + csrlister "k8s.io/client-go/listers/certificates/v1" + corev1listers "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + certutil "k8s.io/client-go/util/cert" + "k8s.io/client-go/util/keyutil" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + + "antrea.io/antrea/pkg/util/env" +) + +const ( + signerRootCASecertName = "antrea-ipsec-ca" + signerControllerName = "CertificateSigningRequestSignerController" + workerItemKey = "key" +) + +// CSRSignerController is responsible for synchronizing the CertificateSigningRequest resources. +type CSRSignerController struct { + client clientset.Interface + csrLister csrlister.CertificateSigningRequestLister + csrListerSynced cache.InformerSynced + + configMapInformer cache.SharedIndexInformer + configMapLister corev1listers.ConfigMapLister + configMapListerSynced cache.InformerSynced + + selfSignedCA bool + + // saved CertificateAuthority + certificateAuthority atomic.Value + + queue workqueue.RateLimitingInterface + fixturesQueue workqueue.RateLimitingInterface +} + +// certificateAuthority implements a certificate authority that supports policy +// based signing. It's used by the signing controller. +type certificateAuthority struct { + // RawCert is an optional field to determine if signing cert/key pairs have changed + RawCert []byte + // RawKey is an optional field to determine if signing cert/key pairs have changed + RawKey []byte + + Certificate *x509.Certificate + PrivateKey crypto.Signer +} + +// NewCSRSignerController returns a new *CSRSignerController. +func NewCSRSignerController(client clientset.Interface, + csrInformer csrinformer.CertificateSigningRequestInformer, + selfSignedCA bool, +) (*CSRSignerController, error) { + + uncastConfigmapInformer := corev1informers.NewFilteredConfigMapInformer(client, env.GetAntreaNamespace(), 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *v1.ListOptions) { + listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", signerRootCASecertName).String() + }) + + configmapLister := corev1listers.NewConfigMapLister(uncastConfigmapInformer.GetIndexer()) + + c := &CSRSignerController{ + client: client, + csrLister: csrInformer.Lister(), + csrListerSynced: csrInformer.Informer().HasSynced, + configMapInformer: uncastConfigmapInformer, + configMapLister: configmapLister, + configMapListerSynced: uncastConfigmapInformer.HasSynced, + selfSignedCA: selfSignedCA, + queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + fixturesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "certificateSigningRequest"), + } + + csrInformer.Informer().AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: c.enqueueCertificateSigningRequest, + UpdateFunc: func(old, cur interface{}) { + c.enqueueCertificateSigningRequest(cur) + }, + DeleteFunc: c.enqueueCertificateSigningRequest, + }, + resyncPeriod, + ) + + uncastConfigmapInformer.AddEventHandlerWithResyncPeriod( + cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + UpdateFunc: func(old, cur interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + DeleteFunc: func(obj interface{}) { + c.fixturesQueue.Add(workerItemKey) + }, + }, + resyncPeriod, + ) + + if err := c.syncRootCertificateAndKey(); err != nil { + return nil, err + } + return c, nil +} + +// Run begins watching and syncing of the CSRSignerController. +func (c *CSRSignerController) Run(stopCh <-chan struct{}) { + defer c.queue.ShutDown() + + klog.Infof("Starting %s", signerControllerName) + defer klog.Infof("Shutting down %s", signerControllerName) + + go c.configMapInformer.Run(stopCh) + + cacheSyncs := []cache.InformerSynced{c.csrListerSynced, c.configMapListerSynced} + if !cache.WaitForNamedCacheSync(signerControllerName, stopCh, cacheSyncs...) { + return + } + + go wait.Until(c.fixturesWorker, time.Second, stopCh) + + go wait.Until(func() { + if err := c.watchSecretChanges(stopCh); err != nil { + klog.ErrorS(err, "watch Secret error", "secret", signerRootCASecertName) + } + }, time.Second, stopCh) + + for i := 0; i < defaultWorkers; i++ { + go wait.Until(c.csrWorker, time.Second, stopCh) + } + + <-stopCh +} + +func (c *CSRSignerController) syncRootCertificateAndKey() error { + var caBytes, caKeyBytes []byte + caSecret, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Get(context.TODO(), signerRootCASecertName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + if !c.selfSignedCA { + klog.Warningf("Self-signed CA disabled. Ensure Secret %q exists in namespace %q", signerRootCASecertName, env.GetAntreaNamespace()) + return nil + } + caBytes, caKeyBytes, err = generateSelfSignedRootCertificate() + if err != nil { + return err + } + caSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: signerRootCASecertName, + Namespace: env.GetAntreaNamespace(), + }, + Data: map[string][]byte{ + "ca.crt": caBytes, + "ca.key": caKeyBytes, + }, + } + caSecret, err = c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Create(context.TODO(), caSecret, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.Info("Created Secret for self-signed root CA") + } + caCertificate, err := certutil.ParseCertsPEM(caSecret.Data["ca.crt"]) + if err != nil { + return err + } + if len(caCertificate) == 0 { + return fmt.Errorf("CA certificate is empty") + } + privateKey, err := keyutil.ParsePrivateKeyPEM(caSecret.Data["ca.key"]) + if err != nil { + return err + } + priv, ok := privateKey.(crypto.Signer) + if !ok { + return fmt.Errorf("error reading CA: key did not implement crypto.Signer") + } + ca := &certificateAuthority{ + RawCert: caSecret.Data["ca.crt"], + RawKey: caSecret.Data["ca.key"], + Certificate: caCertificate[0], + PrivateKey: priv, + } + c.certificateAuthority.Store(ca) + desiredConfigMapData := map[string]string{ + "ca.crt": string(caSecret.Data["ca.crt"]), + } + caConfigMap, err := c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Get(context.TODO(), signerRootCASecertName, metav1.GetOptions{}) + if err != nil { + if !apierrors.IsNotFound(err) { + return err + } + caConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: signerRootCASecertName, + Namespace: env.GetAntreaNamespace(), + }, + Data: desiredConfigMapData, + } + caConfigMap, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Create(context.TODO(), caConfigMap, metav1.CreateOptions{}) + if err != nil { + return err + } + klog.Info("Created Configmap for self-signed root CA") + } + if !reflect.DeepEqual(desiredConfigMapData, caConfigMap.Data) { + caConfigMap.Data = desiredConfigMapData + _, err = c.client.CoreV1().ConfigMaps(env.GetAntreaNamespace()).Update(context.TODO(), caConfigMap, metav1.UpdateOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (c *CSRSignerController) csrWorker() { + for c.processNextWorkItem() { + } +} + +func (c *CSRSignerController) watchSecretChanges(endCh <-chan struct{}) error { + c.fixturesQueue.Add(workerItemKey) + watcher, err := c.client.CoreV1().Secrets(env.GetAntreaNamespace()).Watch(context.TODO(), metav1.SingleObject(metav1.ObjectMeta{ + Namespace: env.GetAntreaNamespace(), + Name: signerRootCASecertName, + })) + if err != nil { + return fmt.Errorf("failed to create watcher: %v", err) + } + ch := watcher.ResultChan() + defer watcher.Stop() + for { + select { + case _, ok := <-ch: + if !ok { + return fmt.Errorf("event channel closed") + } + c.fixturesQueue.Add(workerItemKey) + case <-endCh: + return nil + } + } +} + +func (c *CSRSignerController) fixturesWorker() { + for c.processNextFixtureWorkItem() { + } +} + +func (c *CSRSignerController) enqueueCertificateSigningRequest(obj interface{}) { + csr, ok := obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + deletedState, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + klog.Errorf("Received unexpected object: %v", obj) + return + } + csr, ok = deletedState.Obj.(*certificatesv1.CertificateSigningRequest) + if !ok { + klog.Errorf("DeletedFinalStateUnknown contains non-CertificateSigningRequest object: %v", deletedState.Obj) + return + } + } + c.queue.Add(csr.Name) +} + +func (c *CSRSignerController) syncCSR(key string) error { + startTime := time.Now() + defer func() { + d := time.Since(startTime) + klog.V(2).Infof("Finished syncing CSR %s. (%v)", key, d) + }() + csr, err := c.csrLister.Get(key) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + if csr.Spec.SignerName != antreaSignerName { + return nil + } + if len(csr.Status.Certificate) != 0 { + klog.InfoS("CertificateSigningRequest is already signed", "CertificateSigningRequest", csr.Name) + return nil + } + if !isCertificateRequestApproved(csr) { + klog.InfoS("CertificateSigningRequest is not approved", "CertificateSigningRequest", csr.Name) + return nil + } + req, err := decodeCertificateRequest(csr.Spec.Request) + if err != nil { + klog.ErrorS(err, "failed to decode CertificateSigningRequest", "CertificateSigningRequest", csr.Name) + return nil + } + template, err := newCertificateTemplate(req, csr.Spec.Usages) + if err != nil { + return err + } + currCA, ok := c.certificateAuthority.Load().(*certificateAuthority) + if !ok || currCA == nil { + return fmt.Errorf("certificate authority is not initialized") + } + signed, err := signCSR(currCA, template, req.PublicKey) + if err != nil { + return err + } + bs, err := certutil.EncodeCertificates(signed) + if err != nil { + return err + } + csr.Status.Certificate = bs + _, err = c.client.CertificatesV1().CertificateSigningRequests().UpdateStatus(context.TODO(), csr, metav1.UpdateOptions{}) + if err != nil { + return err + } + return nil +} + +func newCertificateTemplate(certReq *x509.CertificateRequest, usage []certificatesv1.KeyUsage) (*x509.Certificate, error) { + var sn big.Int + snBytes := make([]byte, 18) + _, err := rand.Read(snBytes) + if err != nil { + return nil, err + } + sn.SetBytes(snBytes) + keyUsage, extKeyUsage, err := keyUsagesFromStrings(usage) + if err != nil { + return nil, err + } + template := &x509.Certificate{ + Subject: certReq.Subject, + SignatureAlgorithm: x509.SHA512WithRSA, + NotBefore: time.Now().Add(-5 * time.Minute), + NotAfter: time.Now().Add(365 * 24 * time.Hour), // 1 year + SerialNumber: &sn, + DNSNames: certReq.DNSNames, + BasicConstraintsValid: true, + KeyUsage: keyUsage, + ExtKeyUsage: extKeyUsage, + } + return template, nil +} + +func signCSR(ca *certificateAuthority, template *x509.Certificate, requestKey crypto.PublicKey) (*x509.Certificate, error) { + if len(ca.RawCert) == 0 || len(ca.RawKey) == 0 { + return nil, fmt.Errorf("certificate authority is not valid") + } + derBytes, err := x509.CreateCertificate(rand.Reader, template, ca.Certificate, requestKey, ca.PrivateKey) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(derBytes) + if err != nil { + return nil, err + } + if len(certs) != 1 { + return nil, fmt.Errorf("expect a single certificate, got %d", len(certs)) + } + return certs[0], nil +} + +func (c *CSRSignerController) processNextFixtureWorkItem() bool { + key, quit := c.fixturesQueue.Get() + if quit { + return false + } + defer c.fixturesQueue.Done(key) + err := c.syncRootCertificateAndKey() + if err != nil { + c.fixturesQueue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync root CA and private key") + return true + } + c.fixturesQueue.Forget(key) + return true +} + +func (c *CSRSignerController) processNextWorkItem() bool { + key, quit := c.queue.Get() + if quit { + return false + } + defer c.queue.Done(key) + err := c.syncCSR(key.(string)) + if err != nil { + c.queue.AddRateLimited(key) + klog.ErrorS(err, "Failed to sync CertificateSigningRequest", "CertificateSigningRequest", key) + return true + } + c.queue.Forget(key) + return true +} + +func generateSelfSignedRootCertificate() ([]byte, []byte, error) { + validFrom := time.Now().Add(-time.Hour) // valid an hour earlier to avoid flakes due to clock skew + maxAge := time.Hour * 24 * 365 * 10 // 10 years self-signed certs + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + caTemplate := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "antrea-ipsec-ca", + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: validFrom.Add(maxAge), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + + caDERBytes, err := x509.CreateCertificate(rand.Reader, &caTemplate, &caTemplate, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, err + } + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{Type: certificateBlockType, Bytes: caDERBytes}); err != nil { + return nil, nil, err + } + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{Type: keyutil.RSAPrivateKeyBlockType, Bytes: x509.MarshalPKCS1PrivateKey(caKey)}); err != nil { + return nil, nil, err + } + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} diff --git a/pkg/controller/certificatesigningrequest/signer_controller_test.go b/pkg/controller/certificatesigningrequest/signer_controller_test.go new file mode 100644 index 00000000000..c3e1131c501 --- /dev/null +++ b/pkg/controller/certificatesigningrequest/signer_controller_test.go @@ -0,0 +1,142 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certificatesigningrequest + +import ( + "context" + "crypto/x509" + "crypto/x509/pkix" + "testing" + "time" + + config "antrea.io/antrea/pkg/config/controller" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + certutil "k8s.io/client-go/util/cert" +) + +func TestIPsecCertificateApproverAndSigner(t *testing.T) { + validX509CertificateRequest := x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"antrea.io"}, + CommonName: "worker-node-1", + }, + DNSNames: []string{"worker-node-1"}, + } + _, crBytes := x509CRtoPEM(t, &validX509CertificateRequest) + tests := []struct { + name string + objects []runtime.Object + csr *certificatesv1.CertificateSigningRequest + approvePolicy config.AutoApprovePolicy + expectedError error + expectedApproved, expectedDenied bool + }{ + { + name: "verify and sign valid IPsec CSR", + objects: []runtime.Object{ + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1", + }, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "antrea-agent-8r5f9", + UID: "1206ba75-7d75-474c-8110-99255502178c", + }, + Spec: corev1.PodSpec{ + NodeName: "worker-node-1", + }, + }, + }, + csr: &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-node-1-ipsec", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: crBytes, + SignerName: "antrea.io/signer", + Extra: map[string]certificatesv1.ExtraValue{ + "authentication.kubernetes.io/pod-name": {"antrea-agent-8r5f9"}, + "authentication.kubernetes.io/pod-uid": {"1206ba75-7d75-474c-8110-99255502178c"}, + }, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageIPsecTunnel, + }, + }, + }, + approvePolicy: config.AutoApprovePolicyValidateAgentPodName, + expectedApproved: true, + expectedDenied: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewSimpleClientset(tt.objects...) + informerFactory := informers.NewSharedInformerFactory(clientset, 0) + stopCh := make(chan struct{}) + defer close(stopCh) + + csrInformer := informerFactory.Certificates().V1().CertificateSigningRequests() + approverController := NewCSRApproverController(clientset, csrInformer, config.AutoApprovePolicyValidateAgentPodName) + signerController, err := NewCSRSignerController(clientset, csrInformer, true) + require.NoError(t, err) + + informerFactory.Start(stopCh) + informerFactory.WaitForCacheSync(stopCh) + + go approverController.Run(stopCh) + go signerController.Run(stopCh) + + csr, err := clientset.CertificatesV1().CertificateSigningRequests().Create(context.TODO(), tt.csr, metav1.CreateOptions{}) + require.NoError(t, err) + err = wait.PollImmediate(200*time.Millisecond, 2*time.Second, func() (done bool, err error) { + csr, err = clientset.CertificatesV1().CertificateSigningRequests().Get(context.TODO(), tt.csr.Name, metav1.GetOptions{}) + require.NoError(t, err) + if !isCertificateRequestApproved(csr) { + return false, nil + } + if len(csr.Status.Certificate) == 0 { + return false, nil + } + return true, nil + }) + assert.NoError(t, err) + issued := csr.Status.Certificate + parsed, err := certutil.ParseCertsPEM(issued) + assert.NoError(t, err) + assert.Len(t, parsed, 1) + roots := x509.NewCertPool() + roots.AddCert(signerController.certificateAuthority.Load().(*certificateAuthority).Certificate) + verifyOptions := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageIPSECTunnel, + }, + } + _, err = parsed[0].Verify(verifyOptions) + assert.NoError(t, err) + }) + } +} diff --git a/pkg/ovs/ovsconfig/interfaces.go b/pkg/ovs/ovsconfig/interfaces.go index f4b2bd07115..c1a4b3e8fb5 100644 --- a/pkg/ovs/ovsconfig/interfaces.go +++ b/pkg/ovs/ovsconfig/interfaces.go @@ -40,7 +40,7 @@ type OVSBridgeClient interface { CreateAccessPort(name, ifDev string, externalIDs map[string]interface{}, vlanID uint16) (string, Error) CreateInternalPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) - CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, psk string, externalIDs map[string]interface{}) (string, Error) + CreateTunnelPortExt(name string, tunnelType TunnelType, ofPortRequest int32, csum bool, localIP string, remoteIP string, remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) CreateUplinkPort(name string, ofPortRequest int32, externalIDs map[string]interface{}) (string, Error) DeletePort(portUUID string) Error DeletePorts(portUUIDList []string) Error diff --git a/pkg/ovs/ovsconfig/ovs_client.go b/pkg/ovs/ovsconfig/ovs_client.go index 0d0f01ac3b0..c6050444507 100644 --- a/pkg/ovs/ovsconfig/ovs_client.go +++ b/pkg/ovs/ovsconfig/ovs_client.go @@ -373,7 +373,7 @@ func (br *OVSBridge) CreateInternalPort(name string, ofPortRequest int32, extern // the bridge. // If ofPortRequest is not zero, it will be passed to the OVS port creation. func (br *OVSBridge) CreateTunnelPort(name string, tunnelType TunnelType, ofPortRequest int32) (string, Error) { - return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", nil) + return br.createTunnelPort(name, tunnelType, ofPortRequest, false, "", "", "", "", nil) } // CreateTunnelPortExt creates a tunnel port with the specified name and type @@ -393,12 +393,16 @@ func (br *OVSBridge) CreateTunnelPortExt( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { if psk != "" && remoteIP == "" { return "", newInvalidArgumentsError("IPsec tunnel can not be flow based. remoteIP must be set") } - return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, psk, externalIDs) + if psk != "" && remoteName != "" { + return "", newInvalidArgumentsError("Cannot set psk and remoteName together") + } + return br.createTunnelPort(name, tunnelType, ofPortRequest, csum, localIP, remoteIP, remoteName, psk, externalIDs) } func (br *OVSBridge) createTunnelPort( @@ -408,6 +412,7 @@ func (br *OVSBridge) createTunnelPort( csum bool, localIP string, remoteIP string, + remoteName string, psk string, externalIDs map[string]interface{}) (string, Error) { @@ -429,7 +434,9 @@ func (br *OVSBridge) createTunnelPort( if localIP != "" { options["local_ip"] = localIP } - + if remoteName != "" { + options["remote_name"] = remoteName + } if psk != "" { options["psk"] = psk } diff --git a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go index 882fb72bcf8..cb9856946d2 100644 --- a/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go +++ b/pkg/ovs/ovsconfig/testing/mock_ovsconfig.go @@ -151,18 +151,18 @@ func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPort(arg0, arg1, arg2 int } // CreateTunnelPortExt mocks base method -func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6 string, arg7 map[string]interface{}) (string, ovsconfig.Error) { +func (m *MockOVSBridgeClient) CreateTunnelPortExt(arg0 string, arg1 ovsconfig.TunnelType, arg2 int32, arg3 bool, arg4, arg5, arg6, arg7 string, arg8 map[string]interface{}) (string, ovsconfig.Error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + ret := m.ctrl.Call(m, "CreateTunnelPortExt", arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) ret0, _ := ret[0].(string) ret1, _ := ret[1].(ovsconfig.Error) return ret0, ret1 } // CreateTunnelPortExt indicates an expected call of CreateTunnelPortExt -func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7 interface{}) *gomock.Call { +func (mr *MockOVSBridgeClientMockRecorder) CreateTunnelPortExt(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTunnelPortExt", reflect.TypeOf((*MockOVSBridgeClient)(nil).CreateTunnelPortExt), arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) } // CreateUplinkPort mocks base method diff --git a/test/e2e/ipsec_test.go b/test/e2e/ipsec_test.go index 67790def112..4d19fca9a95 100644 --- a/test/e2e/ipsec_test.go +++ b/test/e2e/ipsec_test.go @@ -40,6 +40,11 @@ func TestIPSec(t *testing.T) { } defer teardownTest(t, data) + t.Logf("Redeploy Antrea with IPsec tunnel enabled") + data.redeployAntrea(t, deployAntreaIPsec) + // Restore normal Antrea deployment with IPsec disabled. + defer data.redeployAntrea(t, deployAntreaDefault) + t.Run("testIPSecTunnelConnectivity", func(t *testing.T) { testIPSecTunnelConnectivity(t, data) }) t.Run("testIPSecDeleteStaleTunnelPorts", func(t *testing.T) { testIPSecDeleteStaleTunnelPorts(t, data) }) } @@ -76,9 +81,6 @@ func (data *TestData) readSecurityAssociationsStatus(nodeName string) (up int, c // the IPsec tunnel, by creating multiple Pods across distinct Nodes and having // them ping each other. func testIPSecTunnelConnectivity(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) - data.testPodConnectivityDifferentNodes(t) // We know that testPodConnectivityDifferentNodes always creates a Pod on Node 0 for the @@ -91,17 +93,12 @@ func testIPSecTunnelConnectivity(t *testing.T, data *TestData) { } else { t.Logf("Found %d 'up' SecurityAssociation(s) for Node '%s'", up, nodeName) } - - // Restore normal Antrea deployment with IPsec disabled. - data.redeployAntrea(t, deployAntreaDefault) } // testIPSecDeleteStaleTunnelPorts checks that when switching from IPsec mode to // non-encrypted mode, the previously created tunnel ports are deleted // correctly. func testIPSecDeleteStaleTunnelPorts(t *testing.T, data *TestData) { - t.Logf("Redeploy Antrea with IPsec tunnel enabled") - data.redeployAntrea(t, deployAntreaIPsec) nodeName0 := nodeName(0) nodeName1 := nodeName(1) diff --git a/test/integration/ovs/ovs_client_test.go b/test/integration/ovs/ovs_client_test.go index 499ca91cc4c..b4f7aebb0b9 100644 --- a/test/integration/ovs/ovs_client_test.go +++ b/test/integration/ovs/ovs_client_test.go @@ -242,7 +242,7 @@ func TestTunnelOptionCsum(t *testing.T) { defer data.teardown(t) name := "vxlan0" - _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", nil) + _, err := data.br.CreateTunnelPortExt(name, ovsconfig.VXLANTunnel, ofPortRequest, testCase.initialCsum, "", "", "", "", nil) require.Nil(t, err, "Error when creating tunnel port") options, err := data.br.GetInterfaceOptions(name) require.Nil(t, err, "Error when getting interface options")