diff --git a/Makefile b/Makefile index 9a2a529f..681c847d 100644 --- a/Makefile +++ b/Makefile @@ -58,10 +58,20 @@ bin/chwrap.tar: build-chwrap cmd/chwrap/chwrap.sh fi; \ done -# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime -# container, push, or push-multiarch are made. Additional prerequisites and the recipes for -# container and push are defined in release-tools/build.make. -container: build-chwrap bin/chwrap.tar + +# This target is mainly used for development to first rebuild the driver binary before building the +# container for local testing. Since the beegfs-csi-driver container requires chwrap to be built and +# included, we also build it anytime container, push, or push-multiarch are made. Additional +# prerequisites and the recipes for container and push are defined in release-tools/build.make. +# +# IMPORTANT: Because the release tool's build.make file specifies BUILD_PLATFORMS= and cannot be +# modified, a default set of build platforms cannot be specified in this file and thus must be +# always provided on the command line otherwise the resulting files will not work correctly with how +# the project's Dockerfile expects them to be named. +# +# For ARM: `make BUILD_PLATFORMS="linux arm64 arm64 arm64" container` For x86: `make +# BUILD_PLATFORMS="linux amd64 amd64 amd64" container` +container: all push-multiarch: build-chwrap bin/chwrap.tar push: container # not explicitly executed in release-tools/build.make diff --git a/README.md b/README.md index faf00c9e..7288eda2 100644 --- a/README.md +++ b/README.md @@ -204,7 +204,7 @@ deployment guide](operator/README.md). ``` 4. Run `kubectl apply -k deploy/k8s/overlays/default`. Note by default the beegfs-csi-driver image will be pulled from - [DockerHub](https://hub.docker.com/r/netapp/beegfs-csi-driver). + [GitHub Container Registry](https://github.com/ThinkParQ/beegfs-csi-driver/pkgs/container/beegfs-csi-driver). 5. Verify all components are installed and operational: `kubectl get pods -n beegfs-csi`. diff --git a/deploy/k8s/bases/csi-beegfs-controller.yaml b/deploy/k8s/bases/csi-beegfs-controller.yaml index bcdcb435..3a8063b7 100644 --- a/deploy/k8s/bases/csi-beegfs-controller.yaml +++ b/deploy/k8s/bases/csi-beegfs-controller.yaml @@ -48,6 +48,27 @@ spec: requests: cpu: 80m memory: 24Mi + - name: csi-resizer + image: registry.k8s.io/sig-storage/csi-resizer:v1.11.1 + args: + - "--csi-address=/csi/csi.sock" + - -v=$(LOG_LEVEL) + securityContext: + # On SELinux enabled systems, a non-privileged sidecar container cannot access the unix domain socket + # created by the privileged driver container. + privileged: true + env: + - name: LOG_LEVEL + value: "3" + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 500Mi + requests: + cpu: 10m + memory: 20Mi - name: beegfs image: ghcr.io/thinkparq/beegfs-csi-driver:v1.6.0 args: diff --git a/deploy/k8s/bases/csi-beegfs-rbac.yaml b/deploy/k8s/bases/csi-beegfs-rbac.yaml index 5d33a60d..f2c702b9 100644 --- a/deploy/k8s/bases/csi-beegfs-rbac.yaml +++ b/deploy/k8s/bases/csi-beegfs-rbac.yaml @@ -24,10 +24,10 @@ metadata: rules: - apiGroups: [""] resources: ["persistentvolumes"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: [""] resources: ["persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] @@ -40,6 +40,12 @@ rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] --- diff --git a/deploy/k8s/deploy.go b/deploy/k8s/deploy.go index 8f4c2abb..06d875ad 100644 --- a/deploy/k8s/deploy.go +++ b/deploy/k8s/deploy.go @@ -42,6 +42,7 @@ const ( ContainerNameBeegfsCsiDriver = "beegfs" ContainerNameCsiNodeDriverRegistrar = "node-driver-registrar" ContainerNameCsiProvisioner = "csi-provisioner" + ContainerNameCsiResizer = "csi-resizer" ContainerNameLivenessProbe = "liveness-probe" ) diff --git a/deploy/k8s/overlays/default/kustomization.yaml b/deploy/k8s/overlays/default/kustomization.yaml index 092cf466..058b5091 100644 --- a/deploy/k8s/overlays/default/kustomization.yaml +++ b/deploy/k8s/overlays/default/kustomization.yaml @@ -43,6 +43,9 @@ secretGenerator: # - name: k8s.gcr.io/sig-storage/csi-provisioner # newName: # newTag: +# - name: k8s.gcr.io/sig-storage/csi-resizer +# newName: +# newTag: # - name: k8s.gcr.io/sig-storage/livenessprobe # newName: # newTag: diff --git a/docs/usage.md b/docs/usage.md index 0eb409b4..ef25d5bd 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -69,6 +69,13 @@ any POSIX) filesystem. The driver does provide integration with BeeGFS permissio and quotas which provides ways to limit the capacity consumed by containers. For more details refer to the documentation on [Quotas](quotas.md). +Starting with v1.7.0 the driver also supports [volume +expansion](https://kubernetes-csi.github.io/docs/volume-expansion.html), which is useful for +instances where the persistent volume claim request size has meaning for the application. As with +the initial capacity request, the size of the PVC and PV are simply updated in the Kubernetes API to +reflect the requested new capacity, and there are no checks there is actually sufficient space +available to satisfy the requested capacity. + ### Static vs Dynamic Provisioning diff --git a/examples/k8s/dyn/dyn-sc.yaml b/examples/k8s/dyn/dyn-sc.yaml index 73523837..508ee26a 100644 --- a/examples/k8s/dyn/dyn-sc.yaml +++ b/examples/k8s/dyn/dyn-sc.yaml @@ -20,4 +20,4 @@ parameters: # permissions/mode: "0644" reclaimPolicy: Delete volumeBindingMode: Immediate -allowVolumeExpansion: false +allowVolumeExpansion: true diff --git a/operator/Makefile b/operator/Makefile index 16fce52b..734688e7 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -6,10 +6,12 @@ VERSION ?= 1.6.0 # BUILD_PLATFORMS contains a set of tuples [os arch buildx_platform suffix base_image addon_image] -# separated by semicolon. An empty variable or empty entry (= just a -# semicolon) builds for the default platform of the current Go -# toolchain. This approach was adapted from the CSI driver release-tools.1 -BUILD_PLATFORMS = +# separated by semicolon. All supported architectures are listed here so other targets used for +# development such as docker-build work correctly even when not building multi-arch images. This is +# because the Dockerfile expects all binaries will have an architecture suffix. Note for the +# official images/binaries published using GitHub actions, the default here is overridden by the +# RELEASE_TOOLS_BUILD_PLATFORMS variable (see the notes there for why). +BUILD_PLATFORMS ?= linux amd64 amd64 amd64;linux arm64 arm64 arm64 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") diff --git a/operator/README.md b/operator/README.md index 5fe3f309..d258a02c 100644 --- a/operator/README.md +++ b/operator/README.md @@ -209,6 +209,9 @@ spec: csiProvisioner: image: some.registry/sig-storage/csi-provisioner tag: # Changing this tag is not supported. + csiResizer: + image: some.registry/sig-storage/csi-resizer + tag: # Changing this tag is not supported. livenessProbe: image: some.registry/sig-storage/livenessprobe tag: # Changing this tag is not supported. diff --git a/operator/api/v1/beegfsdriver_types.go b/operator/api/v1/beegfsdriver_types.go index 6bc07ac3..78171551 100644 --- a/operator/api/v1/beegfsdriver_types.go +++ b/operator/api/v1/beegfsdriver_types.go @@ -113,6 +113,9 @@ type ContainerImageOverrides struct { // Defaults to registry.k8s.io/sig-storage/csi-provisioner:. //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CSI Provisioner" CsiProvisioner ContainerImageOverride `json:"csiProvisioner"` + // Defaults to registry.k8s.io/sig-storage/csi-resizer:. + //+operator-sdk:csv:customresourcedefinitions:type=spec,displayName="CSI Resizer" + CsiResizer ContainerImageOverride `json:"csiResizer"` // Defaults to registry.k8s.io/sig-storage/livenessprobe:. //+operator-sdk:csv:customresourcedefinitions:type=spec LivenessProbe ContainerImageOverride `json:"livenessProbe"` diff --git a/operator/api/v1/zz_generated.deepcopy.go b/operator/api/v1/zz_generated.deepcopy.go index 53c5ffea..d8c3c967 100644 --- a/operator/api/v1/zz_generated.deepcopy.go +++ b/operator/api/v1/zz_generated.deepcopy.go @@ -209,6 +209,7 @@ func (in *ContainerImageOverrides) DeepCopyInto(out *ContainerImageOverrides) { out.BeegfsCsiDriver = in.BeegfsCsiDriver out.CsiNodeDriverRegistrar = in.CsiNodeDriverRegistrar out.CsiProvisioner = in.CsiProvisioner + out.CsiResizer = in.CsiResizer out.LivenessProbe = in.LivenessProbe } diff --git a/operator/bundle/manifests/beegfs-csi-driver-operator.clusterserviceversion.yaml b/operator/bundle/manifests/beegfs-csi-driver-operator.clusterserviceversion.yaml index 40718acf..75b1b66e 100644 --- a/operator/bundle/manifests/beegfs-csi-driver-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/beegfs-csi-driver-operator.clusterserviceversion.yaml @@ -128,6 +128,17 @@ spec: - description: A tag (e.g. v2.2.2 or latest). displayName: Tag path: containerImageOverrides.csiProvisioner.tag + - description: Defaults to registry.k8s.io/sig-storage/csi-resizer:. + displayName: CSI Resizer + path: containerImageOverrides.csiResizer + - description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner + or ghcr.io/thinkparq/beegfs-csi-driver). + displayName: Image + path: containerImageOverrides.csiResizer.image + - description: A tag (e.g. v2.2.2 or latest). + displayName: Tag + path: containerImageOverrides.csiResizer.tag - description: Defaults to registry.k8s.io/sig-storage/livenessprobe:. displayName: Liveness Probe @@ -472,8 +483,15 @@ spec: verbs: - get - list + - patch - update - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - patch - apiGroups: - "" resources: @@ -483,6 +501,15 @@ spec: - delete - get - list + - patch + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list - watch - apiGroups: - "" diff --git a/operator/bundle/manifests/beegfs.csi.netapp.com_beegfsdrivers.yaml b/operator/bundle/manifests/beegfs.csi.netapp.com_beegfsdrivers.yaml index 54ec93e5..d521f7b4 100644 --- a/operator/bundle/manifests/beegfs.csi.netapp.com_beegfsdrivers.yaml +++ b/operator/bundle/manifests/beegfs.csi.netapp.com_beegfsdrivers.yaml @@ -87,6 +87,18 @@ spec: description: A tag (e.g. v2.2.2 or latest). type: string type: object + csiResizer: + description: Defaults to registry.k8s.io/sig-storage/csi-resizer:. + properties: + image: + description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner + or ghcr.io/thinkparq/beegfs-csi-driver). + type: string + tag: + description: A tag (e.g. v2.2.2 or latest). + type: string + type: object livenessProbe: description: Defaults to registry.k8s.io/sig-storage/livenessprobe:. diff --git a/operator/config/crd/bases/beegfs.csi.netapp.com_beegfsdrivers.yaml b/operator/config/crd/bases/beegfs.csi.netapp.com_beegfsdrivers.yaml index dcdd1906..ffa38287 100644 --- a/operator/config/crd/bases/beegfs.csi.netapp.com_beegfsdrivers.yaml +++ b/operator/config/crd/bases/beegfs.csi.netapp.com_beegfsdrivers.yaml @@ -81,6 +81,18 @@ spec: description: A tag (e.g. v2.2.2 or latest). type: string type: object + csiResizer: + description: Defaults to registry.k8s.io/sig-storage/csi-resizer:. + properties: + image: + description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner + or ghcr.io/thinkparq/beegfs-csi-driver). + type: string + tag: + description: A tag (e.g. v2.2.2 or latest). + type: string + type: object livenessProbe: description: Defaults to registry.k8s.io/sig-storage/livenessprobe:. diff --git a/operator/config/manifests/bases/beegfs-csi-driver-operator.clusterserviceversion.yaml b/operator/config/manifests/bases/beegfs-csi-driver-operator.clusterserviceversion.yaml index 60435532..d6b1a2c8 100644 --- a/operator/config/manifests/bases/beegfs-csi-driver-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/bases/beegfs-csi-driver-operator.clusterserviceversion.yaml @@ -68,6 +68,17 @@ spec: - description: A tag (e.g. v2.2.2 or latest). displayName: Tag path: containerImageOverrides.csiProvisioner.tag + - description: Defaults to registry.k8s.io/sig-storage/csi-resizer:. + displayName: CSI Resizer + path: containerImageOverrides.csiResizer + - description: A combination of registry and image (e.g. registry.k8s.io/csi-provisioner + or ghcr.io/thinkparq/beegfs-csi-driver). + displayName: Image + path: containerImageOverrides.csiResizer.image + - description: A tag (e.g. v2.2.2 or latest). + displayName: Tag + path: containerImageOverrides.csiResizer.tag - description: Defaults to registry.k8s.io/sig-storage/livenessprobe:. displayName: Liveness Probe diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 5a88ae1a..4fb66858 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -84,8 +84,15 @@ rules: verbs: - get - list + - patch - update - watch +- apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - patch - apiGroups: - "" resources: @@ -95,6 +102,15 @@ rules: - delete - get - list + - patch + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list - watch - apiGroups: - "" diff --git a/operator/config/samples/beegfs_v1_beegfsdriver.yaml b/operator/config/samples/beegfs_v1_beegfsdriver.yaml index bc088225..63c73d53 100644 --- a/operator/config/samples/beegfs_v1_beegfsdriver.yaml +++ b/operator/config/samples/beegfs_v1_beegfsdriver.yaml @@ -40,6 +40,9 @@ spec: # tag: # csiProvisioner: # image: + # tag: + # csiResizer: + # image: # tag: # livenessProbe: # image: diff --git a/operator/controllers/beegfsdriver_controller.go b/operator/controllers/beegfsdriver_controller.go index 1a0a67c7..580266d2 100644 --- a/operator/controllers/beegfsdriver_controller.go +++ b/operator/controllers/beegfsdriver_controller.go @@ -71,12 +71,14 @@ type BeegfsDriverReconciler struct { //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,resourceNames=privileged,verbs=use // The operator must have the following permissions in order to grant them to the driver. -//+kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;delete -//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;delete;patch +//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=events,verbs=list;watch;create;update;patch //+kubebuilder:rbac:groups=storage.k8s.io,resources=csinodes,verbs=get;list;watch //+kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch +//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims/status,verbs=patch // Reconcile is part of the main Kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -598,6 +600,7 @@ func setImages(log logr.Logger, containers []corev1.Container, overrides beegfsv deploy.ContainerNameBeegfsCsiDriver: overrides.BeegfsCsiDriver, deploy.ContainerNameCsiNodeDriverRegistrar: overrides.CsiNodeDriverRegistrar, deploy.ContainerNameCsiProvisioner: overrides.CsiProvisioner, + deploy.ContainerNameCsiResizer: overrides.CsiResizer, deploy.ContainerNameLivenessProbe: overrides.LivenessProbe, } diff --git a/operator/controllers/beegfsdriver_controller_test.go b/operator/controllers/beegfsdriver_controller_test.go index 0a5ded93..94611620 100644 --- a/operator/controllers/beegfsdriver_controller_test.go +++ b/operator/controllers/beegfsdriver_controller_test.go @@ -370,6 +370,7 @@ var _ = Describe("Unit tests of helper functions", func() { {Name: deploy.ContainerNameBeegfsCsiDriver, Image: "default.domain/default-driver-image:default-driver-tag"}, {Name: deploy.ContainerNameLivenessProbe, Image: "default.domain/default-liveness-image:default-liveness-tag"}, {Name: deploy.ContainerNameCsiNodeDriverRegistrar, Image: "default.domain/default-registrar-image:default-registrar-tag"}, + {Name: deploy.ContainerNameCsiResizer, Image: "default.domain/default-resizer-image:default-resizer-tag"}, } }) @@ -380,11 +381,13 @@ var _ = Describe("Unit tests of helper functions", func() { BeegfsCsiDriver: beegfsv1.ContainerImageOverride{Image: "override.domain/override-driver", Tag: "override-tag"}, CsiNodeDriverRegistrar: beegfsv1.ContainerImageOverride{Image: "override.domain/override-registrar", Tag: "override-tag"}, CsiProvisioner: beegfsv1.ContainerImageOverride{Image: "override.domain/override-provisioner", Tag: "override-tag"}, + CsiResizer: beegfsv1.ContainerImageOverride{Image: "override.domain/override-resizer", Tag: "override-tag"}, LivenessProbe: beegfsv1.ContainerImageOverride{Image: "override.domain/override-liveness", Tag: "override-tag"}, } setImages(ctrl.Log, containers, overrides) Expect(getContainerImageForName(deploy.ContainerNameCsiNodeDriverRegistrar, containers)).To(Equal("override.domain/override-registrar:override-tag")) Expect(getContainerImageForName(deploy.ContainerNameCsiProvisioner, containers)).To(Equal("override.domain/override-provisioner:override-tag")) + Expect(getContainerImageForName(deploy.ContainerNameCsiResizer, containers)).To(Equal("override.domain/override-resizer:override-tag")) Expect(getContainerImageForName(deploy.ContainerNameBeegfsCsiDriver, containers)).To(Equal("override.domain/override-driver:override-tag")) Expect(getContainerImageForName(deploy.ContainerNameLivenessProbe, containers)).To(Equal("override.domain/override-liveness:override-tag")) }) diff --git a/pkg/beegfs/controllerserver.go b/pkg/beegfs/controllerserver.go index 67b3c552..cdd591bb 100644 --- a/pkg/beegfs/controllerserver.go +++ b/pkg/beegfs/controllerserver.go @@ -43,6 +43,7 @@ var ( // controllerCaps represents the capabilities of the controller service controllerCaps = []csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_EXPAND_VOLUME, } ) @@ -378,7 +379,17 @@ func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnap } func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + // While currently volume "capacity" has no meaning as far as the driver is concerned, some + // applications rely on the capacity of the PV/PVC in the K8s API to make certain decisions. + // For these applications it is helpful to support volume resizing. + return &csi.ControllerExpandVolumeResponse{ + CapacityBytes: req.CapacityRange.RequiredBytes, + NodeExpansionRequired: false, + }, nil } func (cs *controllerServer) ControllerGetVolume(ctx context.Context, in *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error) { diff --git a/pkg/beegfs/identityserver.go b/pkg/beegfs/identityserver.go index a9b3ef0e..2e0d1a67 100644 --- a/pkg/beegfs/identityserver.go +++ b/pkg/beegfs/identityserver.go @@ -72,6 +72,13 @@ func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.G }, }, }, + { + Type: &csi.PluginCapability_VolumeExpansion_{ + VolumeExpansion: &csi.PluginCapability_VolumeExpansion{ + Type: csi.PluginCapability_VolumeExpansion_ONLINE, + }, + }, + }, }, }, nil } diff --git a/test/e2e/driver/driver.go b/test/e2e/driver/driver.go index 8991d70c..ff6be98e 100644 --- a/test/e2e/driver/driver.go +++ b/test/e2e/driver/driver.go @@ -97,7 +97,7 @@ func initBaseBeegfsDriver(dynamicVolDirBasePathBeegfsRoot, staticVolDirPathBeegf storageframework.CapPVCDataSource: false, storageframework.CapMultiPODs: true, storageframework.CapRWX: true, - storageframework.CapControllerExpansion: false, + storageframework.CapControllerExpansion: true, storageframework.CapNodeExpansion: false, storageframework.CapVolumeLimits: false, // This setting is only used in two places, both in the multivolume test suite. Setting this to true