Skip to content

Commit

Permalink
Support Direct VolumeMount for PVC (kubeflow#2738)
Browse files Browse the repository at this point in the history 
* support direct volume mount for pvc

Signed-off-by: Lize Cai <lize.cai@sap.com>

* add unit tests

Signed-off-by: Lize Cai <lize.cai@sap.com>

* update flag name to enableDirectPvcVolumeMount

Signed-off-by: Lize Cai <lize.cai@sap.com>

* change parameter type to bool

Signed-off-by: Lize Cai <lize.cai@sap.com>

---------

Signed-off-by: Lize Cai <lize.cai@sap.com>
  • Loading branch information
@lizzzcai
lizzzcai authored Mar 25, 2023
1 parent b2546ef commit 1c5409f
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 17 deletions.
3 changes: 2 additions & 1 deletion config/configmap/inferenceservice.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ data:
"memoryLimit": "1Gi",
"cpuRequest": "100m",
"cpuLimit": "1",
"storageSpecSecretName": "storage-config"
"storageSpecSecretName": "storage-config",
"enableDirectPvcVolumeMount": false
}
# ====================================== CREDENTIALS ======================================
# For a quick reference about AWS ENV variables:
Expand Down
46 changes: 40 additions & 6 deletions pkg/webhook/admission/pod/storage_initializer_injector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,13 @@ const (
)

type StorageInitializerConfig struct {
Image string `json:"image"`
CpuRequest string `json:"cpuRequest"`
CpuLimit string `json:"cpuLimit"`
MemoryRequest string `json:"memoryRequest"`
MemoryLimit string `json:"memoryLimit"`
StorageSpecSecretName string `json:"storageSpecSecretName"`
Image string `json:"image"`
CpuRequest string `json:"cpuRequest"`
CpuLimit string `json:"cpuLimit"`
MemoryRequest string `json:"memoryRequest"`
MemoryLimit string `json:"memoryLimit"`
StorageSpecSecretName string `json:"storageSpecSecretName"`
EnableDirectPvcVolumeMount bool `json:"enableDirectPvcVolumeMount"`
}

type StorageInitializerInjector struct {
Expand Down Expand Up @@ -135,6 +136,39 @@ func (mi *StorageInitializerInjector) InjectStorageInitializer(pod *v1.Pod) erro
}
podVolumes = append(podVolumes, pvcSourceVolume)

// check if using direct volume mount to mount the pvc
// if yes, mount the pvc to model local mount path and return
if mi.config.EnableDirectPvcVolumeMount == true {

// add a corresponding pvc volume mount to the userContainer
// pvc will be mount to /mnt/models rather than /mnt/pvc
// pvcPath will be injected via SubPath, pvcPath must be a root or Dir
// it is user responsibility to ensure it is a root or Dir
pvcSourceVolumeMount := v1.VolumeMount{
Name: PvcSourceMountName,
MountPath: constants.DefaultModelLocalMountPath,
// only path to volume's root ("") or folder is supported
SubPath: pvcPath,
ReadOnly: true,
}
userContainer.VolumeMounts = append(userContainer.VolumeMounts, pvcSourceVolumeMount)

// change the CustomSpecStorageUri env variable value
// to the default model path if present
for index, envVar := range userContainer.Env {
if envVar.Name == constants.CustomSpecStorageUriEnvVarKey && envVar.Value != "" {
userContainer.Env[index].Value = constants.DefaultModelLocalMountPath
}
}

// add volumes to the PodSpec
pod.Spec.Volumes = append(pod.Spec.Volumes, podVolumes...)

// not inject the storage initializer
return nil
}

// below use storage initializer to handle the pvc
// add a corresponding PVC volume mount to the INIT container
pvcSourceVolumeMount := v1.VolumeMount{
Name: PvcSourceMountName,
Expand Down
145 changes: 135 additions & 10 deletions pkg/webhook/admission/pod/storage_initializer_injector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,22 @@ import (
)

const (
StorageInitializerDefaultCPURequest = "100m"
StorageInitializerDefaultCPULimit = "1"
StorageInitializerDefaultMemoryRequest = "200Mi"
StorageInitializerDefaultMemoryLimit = "1Gi"
StorageInitializerDefaultStorageSpecSecretName = "storage-config"
StorageInitializerDefaultCPURequest = "100m"
StorageInitializerDefaultCPULimit = "1"
StorageInitializerDefaultMemoryRequest = "200Mi"
StorageInitializerDefaultMemoryLimit = "1Gi"
StorageInitializerDefaultStorageSpecSecretName = "storage-config"
StorageInitializerDefaultEnableDirectPvcVolumeMount = false
)

var (
storageInitializerConfig = &StorageInitializerConfig{
CpuRequest: StorageInitializerDefaultCPURequest,
CpuLimit: StorageInitializerDefaultCPULimit,
MemoryRequest: StorageInitializerDefaultMemoryRequest,
MemoryLimit: StorageInitializerDefaultMemoryLimit,
StorageSpecSecretName: StorageInitializerDefaultStorageSpecSecretName,
CpuRequest: StorageInitializerDefaultCPURequest,
CpuLimit: StorageInitializerDefaultCPULimit,
MemoryRequest: StorageInitializerDefaultMemoryRequest,
MemoryLimit: StorageInitializerDefaultMemoryLimit,
StorageSpecSecretName: StorageInitializerDefaultStorageSpecSecretName,
EnableDirectPvcVolumeMount: StorageInitializerDefaultEnableDirectPvcVolumeMount,
}

resourceRequirement = v1.ResourceRequirements{
Expand Down Expand Up @@ -1155,3 +1157,126 @@ func TestParsePvcURI(t *testing.T) {

}
}

func TestDirectVolumeMountForPvc(t *testing.T) {
scenarios := map[string]struct {
original *v1.Pod
expected *v1.Pod
}{
"StorageInitializerNotInjectedAndMountsPvcViaVolumeMount": {
original: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
constants.StorageInitializerSourceUriInternalAnnotationKey: "pvc://mypvcname/some/path/on/pvc",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: constants.InferenceServiceContainerName,
},
},
},
},
expected: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
constants.StorageInitializerSourceUriInternalAnnotationKey: "pvc://mypvcname/some/path/on/pvc",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: constants.InferenceServiceContainerName,
VolumeMounts: []v1.VolumeMount{
{
Name: "kserve-pvc-source",
MountPath: "/mnt/models",
SubPath: "some/path/on/pvc",
ReadOnly: true,
},
},
},
},
Volumes: []v1.Volume{
{
Name: "kserve-pvc-source",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: "mypvcname",
ReadOnly: false,
},
},
},
},
},
},
},
"StorageInitializerNotInjectedAndMountsPvcViaVolumeMountShortestPath": {
original: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
constants.StorageInitializerSourceUriInternalAnnotationKey: "pvc://mypvcname",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: constants.InferenceServiceContainerName,
},
},
},
},
expected: &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
constants.StorageInitializerSourceUriInternalAnnotationKey: "pvc://mypvcname",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: constants.InferenceServiceContainerName,
VolumeMounts: []v1.VolumeMount{
{
Name: "kserve-pvc-source",
MountPath: "/mnt/models",
SubPath: "", // volume's root
ReadOnly: true,
},
},
},
},
Volumes: []v1.Volume{
{
Name: "kserve-pvc-source",
VolumeSource: v1.VolumeSource{
PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
ClaimName: "mypvcname",
ReadOnly: false,
},
},
},
},
},
},
},
}

for name, scenario := range scenarios {
injector := &StorageInitializerInjector{
credentialBuilder: credentials.NewCredentialBulder(c, &v1.ConfigMap{
Data: map[string]string{},
}),
config: &StorageInitializerConfig{
EnableDirectPvcVolumeMount: true, // enable direct volume mount for PVC
},
}
if err := injector.InjectStorageInitializer(scenario.original); err != nil {
t.Errorf("Test %q unexpected result: %s", name, err)
}
if diff, _ := kmp.SafeDiff(scenario.expected.Spec, scenario.original.Spec); diff != "" {
t.Errorf("Test %q unexpected result (-want +got): %v", name, diff)
}
}
}

0 comments on commit 1c5409f

Please sign in to comment.