From ec18fd5dfb5ab96e7c2c159f6d12feb9ce9b422f Mon Sep 17 00:00:00 2001 From: Konstantin Shalygin Date: Thu, 16 Sep 2021 13:38:04 +0300 Subject: [PATCH] rbd: added RBD features support for krbd Added support for `object-map, fast-diff` Signed-off-by: Konstantin Shalygin (cherry picked from commit 7411773f7367f4ef05f9012f71361bfcd6cf6a70) --- charts/ceph-csi-rbd/values.yaml | 9 +++++---- docs/deploy-rbd.md | 33 ++++++++++++++++---------------- examples/rbd/storageclass.yaml | 9 +++++---- internal/rbd/rbd_util.go | 34 +++++++++++++++++++++------------ internal/rbd/rbd_util_test.go | 30 ++++++++++++++++++++++++++--- 5 files changed, 75 insertions(+), 40 deletions(-) diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 1e967cc2073d..97d91844f647 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -277,10 +277,11 @@ storageClass: thickProvision: false # (required) RBD image features, CSI creates image with image-format 2 - # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock` - # features. If `journaling` is enabled, must enable `exclusive-lock` too. - # imageFeatures: layering,journaling,exclusive-lock - imageFeatures: layering + # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, + # `object-map`, `fast-diff` features. If `journaling` is enabled, must + # enable `exclusive-lock` too. + # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff + imageFeatures: "layering" # (optional) uncomment the following to use rbd-nbd as mounter # on supported nodes diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index b80a5aa28c25..daf0cf077b77 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -48,23 +48,22 @@ make image-cephcsi **Available volume parameters:** -| Parameter | Required | Description | -| --------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | -| `pool` | yes | Ceph pool into which the RBD image shall be created | -| `dataPool` | no | Ceph pool used for the data of the RBD images. | -| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). | -| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). | -| `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. | -| | -| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | -| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | -| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | -| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | -| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | -| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** | -| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases | -| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it | +| Parameter | Required | Description | +| --------------------------------------------------------------------------------------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | +| `pool` | yes | Ceph pool into which the RBD image shall be created | +| `dataPool` | no | Ceph pool used for the data of the RBD images. | +| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). | +| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). | +| `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. | +| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | +| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | +| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | +| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | +| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | +| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** | +| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases | +| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it | **NOTE:** An accompanying CSI configuration file, needs to be provided to the running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration) diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 80b421412892..f31c59ff4365 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -33,10 +33,11 @@ parameters: # creation (thin provisioning is the default). thickProvision: "false" # (required) RBD image features, CSI creates image with image-format 2 - # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock` - # features. If `journaling` is enabled, must enable `exclusive-lock` too. - # imageFeatures: layering,journaling,exclusive-lock - imageFeatures: layering + # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, + # `object-map`, `fast-diff` features. If `journaling` is enabled, must + # enable `exclusive-lock` too. + # imageFeatures: layering,journaling,exclusive-lock,object-map,fast-diff + imageFeatures: "layering" # (optional) mapOptions is a comma-separated list of map options. # For krbd options refer diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 5a2f334ac79a..1b5536d1f744 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -169,18 +169,28 @@ type imageFeature struct { dependsOn []string } -var supportedFeatures = map[string]imageFeature{ - librbd.FeatureNameLayering: { - needRbdNbd: false, - }, - librbd.FeatureNameExclusiveLock: { - needRbdNbd: true, - }, - librbd.FeatureNameJournaling: { - needRbdNbd: true, - dependsOn: []string{librbd.FeatureNameExclusiveLock}, - }, -} +var ( + supportedFeatures = map[string]imageFeature{ + librbd.FeatureNameLayering: { + needRbdNbd: false, + }, + librbd.FeatureNameExclusiveLock: { + needRbdNbd: false, + }, + librbd.FeatureNameObjectMap: { + needRbdNbd: false, + dependsOn: []string{librbd.FeatureNameExclusiveLock}, + }, + librbd.FeatureNameFastDiff: { + needRbdNbd: false, + dependsOn: []string{librbd.FeatureNameObjectMap}, + }, + librbd.FeatureNameJournaling: { + needRbdNbd: true, + dependsOn: []string{librbd.FeatureNameExclusiveLock}, + }, + } +) // Connect an rbdVolume to the Ceph cluster. func (ri *rbdImage) Connect(cr *util.Credentials) error { diff --git a/internal/rbd/rbd_util_test.go b/internal/rbd/rbd_util_test.go index a121daaf8960..e1894a0275f7 100644 --- a/internal/rbd/rbd_util_test.go +++ b/internal/rbd/rbd_util_test.go @@ -85,21 +85,45 @@ func TestValidateImageFeatures(t *testing.T) { false, "", }, + { + "layering,exclusive-lock,object-map,fast-diff", + &rbdVolume{ + Mounter: rbdDefaultMounter, + }, + false, + "", + }, { "layering,journaling", &rbdVolume{ - Mounter: rbdNbdMounter, + Mounter: rbdDefaultMounter, }, true, "feature journaling requires exclusive-lock to be set", }, + { + "object-map,fast-diff", + &rbdVolume{ + Mounter: rbdDefaultMounter, + }, + true, + "feature object-map requires exclusive-lock to be set", + }, + { + "fast-diff", + &rbdVolume{ + Mounter: rbdDefaultMounter, + }, + true, + "feature fast-diff requires object-map to be set", + }, { "layering,exclusive-lock,journaling", &rbdVolume{ Mounter: rbdDefaultMounter, }, true, - "feature exclusive-lock requires rbd-nbd for mounter", + "feature journaling requires rbd-nbd for mounter", }, { "layering,exclusive-lock,journaling", @@ -107,7 +131,7 @@ func TestValidateImageFeatures(t *testing.T) { Mounter: rbdDefaultMounter, }, true, - "feature exclusive-lock requires rbd-nbd for mounter", + "feature journaling requires rbd-nbd for mounter", }, { "layering,exclusive-loc,journaling",