diff --git a/e2e/test/csi_driver_test.go b/e2e/test/csi_driver_test.go index c2d186f2..ad1094bd 100644 --- a/e2e/test/csi_driver_test.go +++ b/e2e/test/csi_driver_test.go @@ -4,6 +4,7 @@ import ( "e2e_test/test/framework" "fmt" "strconv" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -55,6 +56,70 @@ var _ = Describe("Linode CSI Driver", func() { return nil }, f.Timeout, f.RetryInterval).Should(Succeed()) }) + }) + }) + + Describe("Test", func() { + Context("Simple", func() { + Context("Block Storage", func() { + JustBeforeEach(func() { + By("Creating Persistent Volume Claim") + pvc = f.GetPersistentVolumeClaimObject(size, f.StorageClass, false) + err = f.CreatePersistentVolumeClaim(pvc) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Pod with PVC") + pod = f.GetPodObject(podName1, pvc.Name) + err = f.CreatePod(pod) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("Deleting the Pod with PVC") + err = f.DeletePod(pod.Name) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the Volume to be Detached") + waitForOperation() + + By("Deleting the PVC") + err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the Volume to be Deleted") + waitForOperation() + }) + + Context("1Gi Storage", func() { + BeforeEach(func() { + size = "1Gi" + }) + It("should write and read", func() { + writeFile(file) + readFile(file) + }) + }) + + Context("10Gi Storage", func() { + BeforeEach(func() { + size = "10Gi" + }) + It("should write and read", func() { + writeFile(file) + readFile(file) + }) + }) + + Context("20Gi Storage", func() { + BeforeEach(func() { + size = "20Gi" + }) + It("should write and read", func() { + writeFile(file) + readFile(file) + }) + }) + }) AfterEach(func() { By("Deleting the StatefulSet") @@ -265,4 +330,50 @@ var _ = Describe("Linode CSI Driver", func() { }) }) }) + + Describe("Test", func() { + Context("Block Storage", func() { + Context("in Raw Block Mode", func() { + JustBeforeEach(func() { + By("Creating Persistent Volume Claim") + pvc = f.GetPersistentVolumeClaimObject(size, f.StorageClass, true) + err = f.CreatePersistentVolumeClaim(pvc) + Expect(err).NotTo(HaveOccurred()) + + By("Creating Pod with PVC") + pod = f.GetPodObjectWithBlockVolume(pvc.Name) + err = f.CreatePod(pod) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + By("Deleting the Pod with PVC") + err = f.DeletePod(pod.ObjectMeta) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the Volume to be Detached") + time.Sleep(2 * time.Minute) + + By("Deleting the PVC") + err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the Volume to be Deleted") + time.Sleep(1 * time.Minute) + }) + + Context("Creating Raw Block Storage", func() { + BeforeEach(func() { + size = "10Gi" + }) + + It("should check that raw block storage works", func() { + By("Creating a ext3 Filesystem on the Pod") + err := framework.MkfsInPod(pod) + Expect(err).NotTo(HaveOccurred()) + }) + }) + }) + }) + }) }) diff --git a/e2e/test/framework/pod.go b/e2e/test/framework/pod.go index 046343a4..9d00dc27 100644 --- a/e2e/test/framework/pod.go +++ b/e2e/test/framework/pod.go @@ -44,6 +44,40 @@ func GetPodObject(name, namespace, pvc string) *core.Pod { } } +func (f *Invocation) GetPodObjectWithBlockVolume(pvc string) *core.Pod { + return &core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.app, + Namespace: f.namespace, + }, + Spec: core.PodSpec{ + Containers: []core.Container{ + { + Name: f.app, + Image: "ubuntu", + VolumeDevices: []core.VolumeDevice{ + { + DevicePath: "/dev/block", + Name: "csi-volume", + }, + }, + Command: []string{"sleep", "1000000"}, + }, + }, + Volumes: []core.Volume{ + { + Name: "csi-volume", + VolumeSource: core.VolumeSource{ + PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ + ClaimName: pvc, + }, + }, + }, + }, + }, + } +} + func (f *Invocation) CreatePod(pod *core.Pod) error { pod, err := f.kubeClient.CoreV1().Pods(pod.ObjectMeta.Namespace).Create(pod) if err != nil { @@ -91,3 +125,7 @@ func (f *Invocation) CheckIfFileIsInPod(filename string, pod *core.Pod) error { } return errors.Wrap(err, fmt.Sprintf("file name %v not found", filename)) } + +func MkfsInPod(pod *core.Pod) error { + return runCommand("kubectl", "exec", "--kubeconfig", KubeConfigFile, "-it", "-n", pod.Namespace, pod.Name, "--", "/bin/bash", "-c", "mkfs.ext4 -F /dev/block") +} diff --git a/e2e/test/framework/pvc.go b/e2e/test/framework/pvc.go index 4ba921b3..8ac4fbb8 100644 --- a/e2e/test/framework/pvc.go +++ b/e2e/test/framework/pvc.go @@ -11,7 +11,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func GetPersistentVolumeClaimObject(name, namespace, size, storageClass string) *core.PersistentVolumeClaim { +func (f *Invocation) GetPersistentVolumeClaimObject(size, storageClass string, isblock bool) *core.PersistentVolumeClaim { + + var volType core.PersistentVolumeMode + + if isblock { + volType = core.PersistentVolumeBlock + } else { + volType = core.PersistentVolumeFilesystem + } + return &core.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -21,6 +30,7 @@ func GetPersistentVolumeClaimObject(name, namespace, size, storageClass string) AccessModes: []core.PersistentVolumeAccessMode{ core.ReadWriteOnce, }, + VolumeMode: &volType, StorageClassName: &storageClass, Resources: core.ResourceRequirements{ Requests: core.ResourceList{ diff --git a/pkg/linode-bs/controllerserver.go b/pkg/linode-bs/controllerserver.go index 7ba93b0d..a62be526 100644 --- a/pkg/linode-bs/controllerserver.go +++ b/pkg/linode-bs/controllerserver.go @@ -21,6 +21,7 @@ import ( const gigabyte = 1024 * 1024 * 1024 const minProviderVolumeBytes = 10 * gigabyte const waitTimeout = 300 +const devicePathKey = "devicePath" type LinodeControllerServer struct { Driver *LinodeDriver @@ -269,9 +270,10 @@ func (linodeCS *LinodeControllerServer) ControllerPublishVolume(ctx context.Cont if err != nil { return nil, err } - glog.V(4).Infof("volume %d is attached to instance %d", volume.ID, *volume.LinodeID) + glog.V(4).Infof("volume %d is attached to instance %d with path '%s'", volume.ID, *volume.LinodeID, volume.FilesystemPath) - return &csi.ControllerPublishVolumeResponse{}, nil + pvInfo := map[string]string{devicePathKey: volume.FilesystemPath} + return &csi.ControllerPublishVolumeResponse{PublishContext: pvInfo}, nil } // ControllerUnpublishVolume deattaches the given volume from the node @@ -310,7 +312,7 @@ func (linodeCS *LinodeControllerServer) ControllerUnpublishVolume(ctx context.Co // ValidateVolumeCapabilities checks whether the volume capabilities requested are supported. func (linodeCS *LinodeControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - volumeID, statusErr := common.VolumeIdAsInt("ControllerUnpublishVolume", req) + volumeID, statusErr := common.VolumeIdAsInt("ControllerValidateVolumeCapabilities", req) if statusErr != nil { return nil, statusErr } diff --git a/pkg/linode-bs/examples/kubernetes/csi-app-block.yaml b/pkg/linode-bs/examples/kubernetes/csi-app-block.yaml new file mode 100644 index 00000000..c0ec37ee --- /dev/null +++ b/pkg/linode-bs/examples/kubernetes/csi-app-block.yaml @@ -0,0 +1,17 @@ +kind: Pod +apiVersion: v1 +metadata: + name: csi-block-example-pod +spec: + containers: + - name: csi-block-example-container + image: busybox + volumeMounts: + volumeDevices: + - name: csi-block-example-volume + devicePath: /dev/linode/csi-block-example-dev + command: [ "/bin/sh", "-c", "stat /dev/linode/csi-block-example-dev && sleep 1000000" ] + volumes: + - name: csi-block-example-volume + persistentVolumeClaim: + claimName: csi-block-example-pvc diff --git a/pkg/linode-bs/examples/kubernetes/csi-pvc-block.yaml b/pkg/linode-bs/examples/kubernetes/csi-pvc-block.yaml new file mode 100644 index 00000000..ae9d4bb9 --- /dev/null +++ b/pkg/linode-bs/examples/kubernetes/csi-pvc-block.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: csi-block-example-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + storageClassName: linode-block-storage-retain + resources: + requests: + storage: 10Gi diff --git a/pkg/linode-bs/nodeserver.go b/pkg/linode-bs/nodeserver.go index cfe33b4f..2a315bb0 100644 --- a/pkg/linode-bs/nodeserver.go +++ b/pkg/linode-bs/nodeserver.go @@ -18,11 +18,12 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strconv" "strings" "sync" - csi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" "github.com/linode/linode-blockstorage-csi-driver/pkg/common" linodeclient "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-client" @@ -57,10 +58,12 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node // Validate Arguments targetPath := req.GetTargetPath() + targetPathDir := filepath.Dir(targetPath) stagingTargetPath := req.GetStagingTargetPath() readOnly := req.GetReadonly() volumeID := req.GetVolumeId() volumeCapability := req.GetVolumeCapability() + if len(volumeID) == 0 { return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume ID must be provided") } @@ -74,6 +77,14 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume Capability must be provided") } + // Set mount options: + // - bind mount to the full path to allow duplicate mounts of the same PD. + // - read-only if specified + options := []string{"bind"} + if readOnly { + options = append(options, "ro") + } + notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath) if err != nil && !os.IsNotExist(err) { glog.Errorf("cannot validate mount point: %s %v", targetPath, err) @@ -90,17 +101,38 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node return &csi.NodePublishVolumeResponse{}, nil } - if err := os.MkdirAll(targetPath, os.FileMode(0755)); err != nil { - glog.Errorf("mkdir failed on disk %s (%v)", targetPath, err) - return nil, err - } + if blk := volumeCapability.GetBlock(); blk != nil { + // VolumeMode: Block + glog.V(5).Infof("NodePublishVolume[block]: making targetPathDir %s", targetPathDir) + if err := os.MkdirAll(targetPathDir, os.FileMode(0755)); err != nil { + glog.Errorf("mkdir failed on disk %s (%v)", targetPathDir, err) + return nil, err + } - // Perform a bind mount to the full path to allow duplicate mounts of the same PD. - options := []string{"bind"} - if readOnly { - options = append(options, "ro") + // Update staging path to devicePath + stagingTargetPath = req.PublishContext["devicePath"] + glog.V(5).Infof("NodePublishVolume[block]: set stagingTargetPath to devicePath %s", stagingTargetPath) + + // Make file to bind mount device to file + glog.V(5).Infof("NodePublishVolume[block]: making target block bind mount device file %s", targetPath) + file, err := os.OpenFile(targetPath, os.O_CREATE, 0660) + if err != nil { + if removeErr := os.Remove(targetPath); removeErr != nil { + return nil, status.Errorf(codes.Internal, "Failed remove mount target %s: %v", targetPath, err) + } + return nil, status.Errorf(codes.Internal, "Failed to create file %s: %v", targetPath, err) + } + file.Close() + } else { + // VolumeMode: Filesystem + glog.V(5).Infof("NodePublishVolume[filesystem]: making targetPath %s", targetPath) + if err := os.MkdirAll(targetPath, os.FileMode(0755)); err != nil { + glog.Errorf("mkdir failed on disk %s (%v)", targetPath, err) + return nil, err + } } + // Mount Source to Target err = ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, "ext4", options) if err != nil { notMnt, mntErr := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath) @@ -129,7 +161,7 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume mount of disk failed: %v", err)) } - glog.V(4).Infof("Successfully mounted %s", targetPath) + glog.V(4).Infof("NodePublishVolume successfully mounted %s", targetPath) return &csi.NodePublishVolumeResponse{}, nil } @@ -246,7 +278,12 @@ func (ns *LinodeNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeSt */ return &csi.NodeStageVolumeResponse{}, nil + } + // VolumeMode=block + // Do nothing else with the mount point for stage + if blk := volumeCapability.GetBlock(); blk != nil { + return &csi.NodeStageVolumeResponse{}, nil } // Part 3: Mount device to stagingTargetPath @@ -258,9 +295,6 @@ func (ns *LinodeNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeSt fstype = mnt.FsType } options = append(options, mnt.MountFlags...) - } else if blk := volumeCapability.GetBlock(); blk != nil { - // TODO(#64): Block volume support - return nil, status.Error(codes.Unimplemented, "Block volume support is not yet implemented") } fmtAndMountSource := devicePath