Merge pull request #126 from srust/raw_block_device

VolumeMode=block support

e2e/test/csi_driver_test.go

@@ -4,6 +4,7 @@ import (
 	"e2e_test/test/framework"
 	"fmt"
 	"strconv"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -55,6 +56,70 @@ var _ = Describe("Linode CSI Driver", func() {
 					return nil
 				}, f.Timeout, f.RetryInterval).Should(Succeed())
 			})
+		})
+	})
+
+	Describe("Test", func() {
+		Context("Simple", func() {
+			Context("Block Storage", func() {
+				JustBeforeEach(func() {
+					By("Creating Persistent Volume Claim")
+					pvc = f.GetPersistentVolumeClaimObject(size, f.StorageClass, false)
+					err = f.CreatePersistentVolumeClaim(pvc)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Creating Pod with PVC")
+					pod = f.GetPodObject(podName1, pvc.Name)
+					err = f.CreatePod(pod)
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				AfterEach(func() {
+					By("Deleting the Pod with PVC")
+					err = f.DeletePod(pod.Name)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Waiting for the Volume to be Detached")
+					waitForOperation()
+
+					By("Deleting the PVC")
+					err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Waiting for the Volume to be Deleted")
+					waitForOperation()
+				})
+
+				Context("1Gi Storage", func() {
+					BeforeEach(func() {
+						size = "1Gi"
+					})
+					It("should write and read", func() {
+						writeFile(file)
+						readFile(file)
+					})
+				})
+
+				Context("10Gi Storage", func() {
+					BeforeEach(func() {
+						size = "10Gi"
+					})
+					It("should write and read", func() {
+						writeFile(file)
+						readFile(file)
+					})
+				})
+
+				Context("20Gi Storage", func() {
+					BeforeEach(func() {
+						size = "20Gi"
+					})
+					It("should write and read", func() {
+						writeFile(file)
+						readFile(file)
+					})
+				})
+			})
 
 			AfterEach(func() {
 				By("Deleting the StatefulSet")
@@ -265,4 +330,50 @@ var _ = Describe("Linode CSI Driver", func() {
 			})
 		})
 	})
+
+	Describe("Test", func() {
+		Context("Block Storage", func() {
+			Context("in Raw Block Mode", func() {
+				JustBeforeEach(func() {
+					By("Creating Persistent Volume Claim")
+					pvc = f.GetPersistentVolumeClaimObject(size, f.StorageClass, true)
+					err = f.CreatePersistentVolumeClaim(pvc)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Creating Pod with PVC")
+					pod = f.GetPodObjectWithBlockVolume(pvc.Name)
+					err = f.CreatePod(pod)
+					Expect(err).NotTo(HaveOccurred())
+				})
+
+				AfterEach(func() {
+					By("Deleting the Pod with PVC")
+					err = f.DeletePod(pod.ObjectMeta)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Waiting for the Volume to be Detached")
+					time.Sleep(2 * time.Minute)
+
+					By("Deleting the PVC")
+					err = f.DeletePersistentVolumeClaim(pvc.ObjectMeta)
+					Expect(err).NotTo(HaveOccurred())
+
+					By("Waiting for the Volume to be Deleted")
+					time.Sleep(1 * time.Minute)
+				})
+
+				Context("Creating Raw Block Storage", func() {
+					BeforeEach(func() {
+						size = "10Gi"
+					})
+
+					It("should check that raw block storage works", func() {
+						By("Creating a ext3 Filesystem on the Pod")
+						err := framework.MkfsInPod(pod)
+						Expect(err).NotTo(HaveOccurred())
+					})
+				})
+			})
+		})
+	})
 })

e2e/test/framework/pod.go

@@ -44,6 +44,40 @@ func GetPodObject(name, namespace, pvc string) *core.Pod {
 	}
 }
 
+func (f *Invocation) GetPodObjectWithBlockVolume(pvc string) *core.Pod {
+	return &core.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      f.app,
+			Namespace: f.namespace,
+		},
+		Spec: core.PodSpec{
+			Containers: []core.Container{
+				{
+					Name:  f.app,
+					Image: "ubuntu",
+					VolumeDevices: []core.VolumeDevice{
+						{
+							DevicePath: "/dev/block",
+							Name:       "csi-volume",
+						},
+					},
+					Command: []string{"sleep", "1000000"},
+				},
+			},
+			Volumes: []core.Volume{
+				{
+					Name: "csi-volume",
+					VolumeSource: core.VolumeSource{
+						PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{
+							ClaimName: pvc,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
 func (f *Invocation) CreatePod(pod *core.Pod) error {
 	pod, err := f.kubeClient.CoreV1().Pods(pod.ObjectMeta.Namespace).Create(pod)
 	if err != nil {
@@ -91,3 +125,7 @@ func (f *Invocation) CheckIfFileIsInPod(filename string, pod *core.Pod) error {
 	}
 	return errors.Wrap(err, fmt.Sprintf("file name %v not found", filename))
 }
+
+func MkfsInPod(pod *core.Pod) error {
+	return runCommand("kubectl", "exec", "--kubeconfig", KubeConfigFile, "-it", "-n", pod.Namespace, pod.Name, "--", "/bin/bash", "-c", "mkfs.ext4 -F /dev/block")
+}

e2e/test/framework/pvc.go

@@ -11,7 +11,16 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func GetPersistentVolumeClaimObject(name, namespace, size, storageClass string) *core.PersistentVolumeClaim {
+func (f *Invocation) GetPersistentVolumeClaimObject(size, storageClass string, isblock bool) *core.PersistentVolumeClaim {
+
+	var volType core.PersistentVolumeMode
+
+	if isblock {
+		volType = core.PersistentVolumeBlock
+	} else {
+		volType = core.PersistentVolumeFilesystem
+	}
+
 	return &core.PersistentVolumeClaim{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -21,6 +30,7 @@ func GetPersistentVolumeClaimObject(name, namespace, size, storageClass string)
 			AccessModes: []core.PersistentVolumeAccessMode{
 				core.ReadWriteOnce,
 			},
+			VolumeMode:       &volType,
 			StorageClassName: &storageClass,
 			Resources: core.ResourceRequirements{
 				Requests: core.ResourceList{

pkg/linode-bs/controllerserver.go

@@ -21,6 +21,7 @@ import (
 const gigabyte = 1024 * 1024 * 1024
 const minProviderVolumeBytes = 10 * gigabyte
 const waitTimeout = 300
+const devicePathKey = "devicePath"
 
 type LinodeControllerServer struct {
 	Driver          *LinodeDriver
@@ -269,9 +270,10 @@ func (linodeCS *LinodeControllerServer) ControllerPublishVolume(ctx context.Cont
 	if err != nil {
 		return nil, err
 	}
-	glog.V(4).Infof("volume %d is attached to instance %d", volume.ID, *volume.LinodeID)
+	glog.V(4).Infof("volume %d is attached to instance %d with path '%s'", volume.ID, *volume.LinodeID, volume.FilesystemPath)
 
-	return &csi.ControllerPublishVolumeResponse{}, nil
+	pvInfo := map[string]string{devicePathKey: volume.FilesystemPath}
+	return &csi.ControllerPublishVolumeResponse{PublishContext: pvInfo}, nil
 }
 
 // ControllerUnpublishVolume deattaches the given volume from the node
@@ -310,7 +312,7 @@ func (linodeCS *LinodeControllerServer) ControllerUnpublishVolume(ctx context.Co
 
 // ValidateVolumeCapabilities checks whether the volume capabilities requested are supported.
 func (linodeCS *LinodeControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
-	volumeID, statusErr := common.VolumeIdAsInt("ControllerUnpublishVolume", req)
+	volumeID, statusErr := common.VolumeIdAsInt("ControllerValidateVolumeCapabilities", req)
 	if statusErr != nil {
 		return nil, statusErr
 	}

pkg/linode-bs/examples/kubernetes/csi-app-block.yaml

@@ -0,0 +1,17 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: csi-block-example-pod
+spec:
+  containers:
+    - name: csi-block-example-container
+      image: busybox
+      volumeMounts:
+      volumeDevices:
+        - name: csi-block-example-volume
+          devicePath: /dev/linode/csi-block-example-dev
+      command: [ "/bin/sh", "-c", "stat /dev/linode/csi-block-example-dev && sleep 1000000" ]
+  volumes:
+    - name: csi-block-example-volume
+      persistentVolumeClaim:
+        claimName: csi-block-example-pvc

pkg/linode-bs/examples/kubernetes/csi-pvc-block.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-block-example-pvc
+spec:
+  accessModes:
+  - ReadWriteOnce
+  volumeMode: Block
+  storageClassName: linode-block-storage-retain
+  resources:
+    requests:
+      storage: 10Gi

pkg/linode-bs/nodeserver.go

@@ -18,11 +18,12 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"sync"
 
-	csi "github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/golang/glog"
 	"github.com/linode/linode-blockstorage-csi-driver/pkg/common"
 	linodeclient "github.com/linode/linode-blockstorage-csi-driver/pkg/linode-client"
@@ -57,10 +58,12 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node
 
 	// Validate Arguments
 	targetPath := req.GetTargetPath()
+	targetPathDir := filepath.Dir(targetPath)
 	stagingTargetPath := req.GetStagingTargetPath()
 	readOnly := req.GetReadonly()
 	volumeID := req.GetVolumeId()
 	volumeCapability := req.GetVolumeCapability()
+
 	if len(volumeID) == 0 {
 		return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume ID must be provided")
 	}
@@ -74,6 +77,14 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node
 		return nil, status.Error(codes.InvalidArgument, "NodePublishVolume Volume Capability must be provided")
 	}
 
+	// Set mount options:
+	//  - bind mount to the full path to allow duplicate mounts of the same PD.
+	//  - read-only if specified
+	options := []string{"bind"}
+	if readOnly {
+		options = append(options, "ro")
+	}
+
 	notMnt, err := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath)
 	if err != nil && !os.IsNotExist(err) {
 		glog.Errorf("cannot validate mount point: %s %v", targetPath, err)
@@ -90,17 +101,38 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
 
-	if err := os.MkdirAll(targetPath, os.FileMode(0755)); err != nil {
-		glog.Errorf("mkdir failed on disk %s (%v)", targetPath, err)
-		return nil, err
-	}
+	if blk := volumeCapability.GetBlock(); blk != nil {
+		// VolumeMode: Block
+		glog.V(5).Infof("NodePublishVolume[block]: making targetPathDir %s", targetPathDir)
+		if err := os.MkdirAll(targetPathDir, os.FileMode(0755)); err != nil {
+			glog.Errorf("mkdir failed on disk %s (%v)", targetPathDir, err)
+			return nil, err
+		}
 
-	// Perform a bind mount to the full path to allow duplicate mounts of the same PD.
-	options := []string{"bind"}
-	if readOnly {
-		options = append(options, "ro")
+		// Update staging path to devicePath
+		stagingTargetPath = req.PublishContext["devicePath"]
+		glog.V(5).Infof("NodePublishVolume[block]: set stagingTargetPath to devicePath %s", stagingTargetPath)
+
+		// Make file to bind mount device to file
+		glog.V(5).Infof("NodePublishVolume[block]: making target block bind mount device file %s", targetPath)
+		file, err := os.OpenFile(targetPath, os.O_CREATE, 0660)
+		if err != nil {
+			if removeErr := os.Remove(targetPath); removeErr != nil {
+				return nil, status.Errorf(codes.Internal, "Failed remove mount target %s: %v", targetPath, err)
+			}
+			return nil, status.Errorf(codes.Internal, "Failed to create file %s: %v", targetPath, err)
+		}
+		file.Close()
+	} else {
+		// VolumeMode: Filesystem
+		glog.V(5).Infof("NodePublishVolume[filesystem]: making targetPath %s", targetPath)
+		if err := os.MkdirAll(targetPath, os.FileMode(0755)); err != nil {
+			glog.Errorf("mkdir failed on disk %s (%v)", targetPath, err)
+			return nil, err
+		}
 	}
 
+	// Mount Source to Target
 	err = ns.Mounter.Interface.Mount(stagingTargetPath, targetPath, "ext4", options)
 	if err != nil {
 		notMnt, mntErr := ns.Mounter.Interface.IsLikelyNotMountPoint(targetPath)
@@ -129,7 +161,7 @@ func (ns *LinodeNodeServer) NodePublishVolume(ctx context.Context, req *csi.Node
 		return nil, status.Error(codes.Internal, fmt.Sprintf("NodePublishVolume mount of disk failed: %v", err))
 	}
 
-	glog.V(4).Infof("Successfully mounted %s", targetPath)
+	glog.V(4).Infof("NodePublishVolume successfully mounted %s", targetPath)
 	return &csi.NodePublishVolumeResponse{}, nil
 }
 
@@ -246,7 +278,12 @@ func (ns *LinodeNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeSt
 
 		*/
 		return &csi.NodeStageVolumeResponse{}, nil
+	}
 
+	// VolumeMode=block
+	// Do nothing else with the mount point for stage
+	if blk := volumeCapability.GetBlock(); blk != nil {
+		return &csi.NodeStageVolumeResponse{}, nil
 	}
 
 	// Part 3: Mount device to stagingTargetPath
@@ -258,9 +295,6 @@ func (ns *LinodeNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeSt
 			fstype = mnt.FsType
 		}
 		options = append(options, mnt.MountFlags...)
-	} else if blk := volumeCapability.GetBlock(); blk != nil {
-		// TODO(#64): Block volume support
-		return nil, status.Error(codes.Unimplemented, "Block volume support is not yet implemented")
 	}
 
 	fmtAndMountSource := devicePath