Merge pull request rook#10204 from thotz/rgw-sse-s3-vault

object: adding support for sse s3 for RGW

Documentation/CRDs/Object-Storage/ceph-object-store-crd.md

@@ -201,9 +201,9 @@ Rook-Ceph always keeps the bucket and the user for the health check, it just doe
 
 ## Security settings
 
-Ceph RGW supports encryption via Key Management System (KMS) using HashiCorp Vault. Refer to the [vault kms section](../Cluster/ceph-cluster-crd.md#vault-kms) for detailed explanation.
-If these settings are defined, then RGW establish a connection between Vault and whenever S3 client sends a request with Server Side Encryption,
-it encrypts that using the key specified by the client. For more details w.r.t RGW, please refer [Ceph Vault documentation](https://docs.ceph.com/en/latest/radosgw/vault/)
+Ceph RGW supports Server Side Encryption as defined in [AWS S3 protocol](https://docs.aws.amazon.com/AmazonS3/latest/userguide/serv-side-encryption.html) with three different modes: AWS-SSE:C, AWS-SSE:KMS and AWS-SSE:S3. The last two modes require a Key Management System (KMS) like HashiCorp Vault. Currently, Vault is the only supported KMS backend for CephObjectStore.
+
+Refer to the [Vault KMS section](../../Storage-Configuration/Advanced/key-management-system.md#vault) for details about Vault. If these settings are defined, then RGW will establish a connection between Vault and whenever S3 client sends request with Server Side Encryption. [Ceph's Vault documentation](https://docs.ceph.com/en/latest/radosgw/vault/) has more details.
 
 The `security` section contains settings related to KMS encryption of the RGW.
 
@@ -217,13 +217,21 @@ security:
       VAULT_SECRET_ENGINE: kv
       VAULT_BACKEND: v2
     # name of the k8s secret containing the kms authentication token
-    tokenSecretName: rgw-vault-token
+    tokenSecretName: rgw-vault-kms-token
+  s3:
+    connectionDetails:
+      KMS_PROVIDER: vault
+      VAULT_ADDR: http://vault.default.svc.cluster.local:8200
+      VAULT_BACKEND_PATH: rgw
+      VAULT_SECRET_ENGINE: transit
+    # name of the k8s secret containing the kms authentication token
+    tokenSecretName: rgw-vault-s3-token
 ```
 
 For RGW, please note the following:
 
-* `VAULT_SECRET_ENGINE` option is specifically for RGW to mention about the secret engine which can be used, currently supports two: [kv](https://www.vaultproject.io/docs/secrets/kv) and [transit](https://www.vaultproject.io/docs/secrets/transit). And for kv engine only version 2 is supported.
-* The Storage administrator needs to create a secret in the Vault server so that S3 clients use that key for encryption
+* `VAULT_SECRET_ENGINE`: the secret engine which Vault should use. Currently supports [kv](https://www.vaultproject.io/docs/secrets/kv) and [transit](https://www.vaultproject.io/docs/secrets/transit). AWS-SSE:KMS supports `transit` engine and `kv` engine version 2. AWS-SSE:S3 only supports `transit` engine.
+* The Storage administrator needs to create a secret in the Vault server so that S3 clients use that key for encryption for AWS-SSE:KMS
 
 ```console
 vault kv put rook/<mybucketkey> key=$(openssl rand -base64 32) # kv engine
@@ -232,6 +240,10 @@ vault write -f transit/keys/<mybucketkey> exportable=true # transit engine
 
 * TLS authentication with custom certificates between Vault and CephObjectStore RGWs are supported from ceph v16.2.6 onwards
 
+* `tokenSecretName` can be (and often will be) the same for both kms and s3 configurations.
+
+* `AWS-SSE:S3` requires Ceph Quincy (v17.2.3) and later.
+
 ## Deleting a CephObjectStore
 
 During deletion of a CephObjectStore resource, Rook protects against accidental or premature

PendingReleaseNotes.md

@@ -7,3 +7,4 @@
 ## Features
 
 - The toolbox pod now uses the Ceph image directly instead of the Rook image
+- Add support for AWS Server Side Encryption with (AWS-SSE:S3)[https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html] for RGW.

cmd/rook/secret.go

@@ -73,7 +73,7 @@ func startSecret() *kms.Config {
 	}
 
 	// Validate connection details
-	err = kms.ValidateConnectionDetails(ctx, context, &cephCluster.Spec.Security, namespace)
+	err = kms.ValidateConnectionDetails(ctx, context, &cephCluster.Spec.Security.KeyManagementService, namespace)
 	if err != nil {
 		rook.TerminateFatal(errors.Wrap(err, "failed to validate kms connection details"))
 	}

deploy/charts/rook-ceph/templates/resources.yaml

@@ -8767,6 +8767,21 @@ spec:
                           description: TokenSecretName is the kubernetes secret containing the KMS token
                           type: string
                       type: object
+                    s3:
+                      description: The settings for supporting AWS-SSE:S3 with RGW
+                      nullable: true
+                      properties:
+                        connectionDetails:
+                          additionalProperties:
+                            type: string
+                          description: ConnectionDetails contains the KMS connection details (address, port etc)
+                          nullable: true
+                          type: object
+                          x-kubernetes-preserve-unknown-fields: true
+                        tokenSecretName:
+                          description: TokenSecretName is the kubernetes secret containing the KMS token
+                          type: string
+                      type: object
                   type: object
                 zone:
                   description: The multisite info

deploy/examples/crds.yaml

@@ -8759,6 +8759,21 @@ spec:
                           description: TokenSecretName is the kubernetes secret containing the KMS token
                           type: string
                       type: object
+                    s3:
+                      description: The settings for supporting AWS-SSE:S3 with RGW
+                      nullable: true
+                      properties:
+                        connectionDetails:
+                          additionalProperties:
+                            type: string
+                          description: ConnectionDetails contains the KMS connection details (address, port etc)
+                          nullable: true
+                          type: object
+                          x-kubernetes-preserve-unknown-fields: true
+                        tokenSecretName:
+                          description: TokenSecretName is the kubernetes secret containing the KMS token
+                          type: string
+                      type: object
                   type: object
                 zone:
                   description: The multisite info

deploy/examples/object-openshift.yaml

@@ -109,7 +109,7 @@ spec:
   # security oriented settings
   # security:
   # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file
-  #   kms:
+  #   kms: # configures RGW with AWS-SSE:KMS settings
   #     # name of the config map containing all the kms connection details
   #     connectionDetails:
   #        KMS_PROVIDER: "vault"
@@ -119,6 +119,15 @@ spec:
   #        VAULT_BACKEND: v2
   #     # name of the secret containing the kms authentication token
   #     tokenSecretName: rook-vault-token
+  #   s3: # configures RGW with AWS-SSE:S3 settings
+  #     # name of the config map containing all the kms connection details
+  #     connectionDetails:
+  #        KMS_PROVIDER: "vault"
+  #        VAULT_ADDR: VAULT_ADDR_CHANGE_ME # e,g: http://vault.my-domain.com:8200
+  #        VAULT_BACKEND_PATH: "rook"
+  #        VAULT_SECRET_ENGINE: "transit"
+  #     # name of the secret containing the kms authentication token
+  #     tokenSecretName: rook-vault-token
 # # UNCOMMENT THIS TO ENABLE A KMS CONNECTION
 # # Also, do not forget to replace both:
 # #  * ROOK_TOKEN_CHANGE_ME: with a base64 encoded value of the token to use

deploy/examples/object.yaml

@@ -116,8 +116,8 @@ spec:
       disabled: false
   # security oriented settings
   # security:
-  # To enable the KMS configuration properly don't forget to uncomment the Secret at the end of the file
-  #   kms:
+  # To enable the Server Side Encryption configuration properly don't forget to uncomment the Secret at the end of the file
+  #   kms: # configures RGW with AWS-SSE:KMS
   #     # name of the config map containing all the kms connection details
   #     connectionDetails:
   #        KMS_PROVIDER: "vault"
@@ -127,6 +127,15 @@ spec:
   #        VAULT_BACKEND: v2
   #     # name of the secret containing the kms authentication token
   #     tokenSecretName: rook-vault-token
+  #   s3: # configures RGW with AWS-SSE:S3
+  #     # name of the config map containing all the kms connection details
+  #     connectionDetails:
+  #        KMS_PROVIDER: "vault"
+  #        VAULT_ADDR: VAULT_ADDR_CHANGE_ME # e,g: http://vault.my-domain.com:8200
+  #        VAULT_BACKEND_PATH: "rook"
+  #        VAULT_SECRET_ENGINE: "transit"
+  #     # name of the secret containing the kms authentication token
+  #     tokenSecretName: rook-vault-token
 # # UNCOMMENT THIS TO ENABLE A KMS CONNECTION
 # # Also, do not forget to replace both:
 # #  * ROOK_TOKEN_CHANGE_ME: with a base64 encoded value of the token to use

pkg/apis/ceph.rook.io/v1/types.go

@@ -244,6 +244,18 @@ type SecuritySpec struct {
 	KeyManagementService KeyManagementServiceSpec `json:"kms,omitempty"`
 }
 
+// ObjectStoreSecuritySpec is spec to define security features like encryption
+type ObjectStoreSecuritySpec struct {
+	// +optional
+	// +nullable
+	SecuritySpec `json:""`
+
+	// The settings for supporting AWS-SSE:S3 with RGW
+	// +optional
+	// +nullable
+	ServerSideEncryptionS3 KeyManagementServiceSpec `json:"s3,omitempty"`
+}
+
 // KeyManagementServiceSpec represent various details of the KMS server
 type KeyManagementServiceSpec struct {
 	// ConnectionDetails contains the KMS connection details (address, port etc)
@@ -1324,7 +1336,7 @@ type ObjectStoreSpec struct {
 	// Security represents security settings
 	// +optional
 	// +nullable
-	Security *SecuritySpec `json:"security,omitempty"`
+	Security *ObjectStoreSecuritySpec `json:"security,omitempty"`
 
 	// Whether host networking is enabled for the rgw daemon. If not set, the network settings from the cluster CR will be applied.
 	// +kubebuilder:pruning:PreserveUnknownFields

pkg/daemon/ceph/osd/kms/kms.go

@@ -209,37 +209,37 @@ func GetParam(kmsConfig map[string]string, param string) string {
 }
 
 // ValidateConnectionDetails validates mandatory KMS connection details
-func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Context, securitySpec *cephv1.SecuritySpec, ns string) error {
+func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Context, kms *cephv1.KeyManagementServiceSpec, ns string) error {
 	// Lookup mandatory connection details
 	for _, config := range kmsMandatoryConnectionDetails {
-		if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
+		if GetParam(kms.ConnectionDetails, config) == "" {
 			return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
 		}
 	}
 
 	// A token must be specified if token-auth is used
-	if !securitySpec.KeyManagementService.IsK8sAuthEnabled() && securitySpec.KeyManagementService.TokenSecretName == "" {
-		if !securitySpec.KeyManagementService.IsTokenAuthEnabled() {
+	if !kms.IsK8sAuthEnabled() && kms.TokenSecretName == "" {
+		if !kms.IsTokenAuthEnabled() {
 			return errors.New("failed to validate kms configuration (missing token in spec)")
 		}
 	}
 
 	// KMS provider must be specified
-	provider := GetParam(securitySpec.KeyManagementService.ConnectionDetails, Provider)
+	provider := GetParam(kms.ConnectionDetails, Provider)
 
 	// Validate potential token Secret presence
-	if securitySpec.KeyManagementService.IsTokenAuthEnabled() {
-		kmsToken, err := clusterdContext.Clientset.CoreV1().Secrets(ns).Get(ctx, securitySpec.KeyManagementService.TokenSecretName, metav1.GetOptions{})
+	if kms.IsTokenAuthEnabled() {
+		kmsToken, err := clusterdContext.Clientset.CoreV1().Secrets(ns).Get(ctx, kms.TokenSecretName, metav1.GetOptions{})
 		if err != nil {
-			return errors.Wrapf(err, "failed to fetch kms token secret %q", securitySpec.KeyManagementService.TokenSecretName)
+			return errors.Wrapf(err, "failed to fetch kms token secret %q", kms.TokenSecretName)
 		}
 
 		switch provider {
 		case secrets.TypeVault:
 			// Check for empty token
 			token, ok := kmsToken.Data[KMSTokenSecretNameKey]
 			if !ok || len(token) == 0 {
-				return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, securitySpec.KeyManagementService.TokenSecretName)
+				return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", KMSTokenSecretNameKey, kms.TokenSecretName)
 			}
 
 			// Set the env variable
@@ -252,38 +252,38 @@ func ValidateConnectionDetails(ctx context.Context, clusterdContext *clusterd.Co
 			for _, config := range kmsIBMKeyProtectMandatoryTokenDetails {
 				v, ok := kmsToken.Data[config]
 				if !ok || len(v) == 0 {
-					return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", config, securitySpec.KeyManagementService.TokenSecretName)
+					return errors.Errorf("failed to read k8s kms secret %q key %q (not found or empty)", config, kms.TokenSecretName)
 				}
 				// Append the token secret details to the connection details
-				securitySpec.KeyManagementService.ConnectionDetails[config] = strings.TrimSuffix(strings.TrimSpace(string(v)), "\n")
+				kms.ConnectionDetails[config] = strings.TrimSuffix(strings.TrimSpace(string(v)), "\n")
 			}
 		}
 	}
 
 	// Validate KMS provider connection details for each provider
 	switch provider {
 	case secrets.TypeVault:
-		err := validateVaultConnectionDetails(ctx, clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
+		err := validateVaultConnectionDetails(ctx, clusterdContext, ns, kms.ConnectionDetails)
 		if err != nil {
 			return errors.Wrap(err, "failed to validate vault connection details")
 		}
 
-		secretEngine := securitySpec.KeyManagementService.ConnectionDetails[VaultSecretEngineKey]
+		secretEngine := kms.ConnectionDetails[VaultSecretEngineKey]
 		switch secretEngine {
 		case VaultKVSecretEngineKey:
 			// Append Backend Version if not already present
-			if GetParam(securitySpec.KeyManagementService.ConnectionDetails, vault.VaultBackendKey) == "" {
-				backendVersion, err := BackendVersion(ctx, clusterdContext, ns, securitySpec.KeyManagementService.ConnectionDetails)
+			if GetParam(kms.ConnectionDetails, vault.VaultBackendKey) == "" {
+				backendVersion, err := BackendVersion(ctx, clusterdContext, ns, kms.ConnectionDetails)
 				if err != nil {
 					return errors.Wrap(err, "failed to get backend version")
 				}
-				securitySpec.KeyManagementService.ConnectionDetails[vault.VaultBackendKey] = backendVersion
+				kms.ConnectionDetails[vault.VaultBackendKey] = backendVersion
 			}
 		}
 
 	case TypeIBM:
 		for _, config := range kmsIBMKeyProtectMandatoryConnectionDetails {
-			if GetParam(securitySpec.KeyManagementService.ConnectionDetails, config) == "" {
+			if GetParam(kms.ConnectionDetails, config) == "" {
 				return errors.Errorf("failed to validate kms config %q. cannot be empty", config)
 			}
 		}