Skip to content

Commit

Permalink
e2e test commented
Browse files Browse the repository at this point in the history 
Signed-off-by: Bangqi Zhu <bangqizhu@microsoft.com>
  • Loading branch information
Bangqi Zhu committed Aug 6, 2024
1 parent 5bde8f4 commit 4736725
Show file tree
Hide file tree
Showing 4 changed files with 267 additions and 243 deletions.
24 changes: 22 additions & 2 deletions pkg/controllers/workspace_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,9 @@ func (c *WorkspaceReconciler) updateControllerRevision(ctx context.Context, wObj
Revision: revisionNum,
Data: runtime.RawExtension{Raw: jsonData},
}

if annotations == nil {
annotations = make(map[string]string)
} // in case annotations is nil. TODO: have the solution of update status
annotations[WorkspaceRevisionAnnotation] = currentHash
wObj.SetAnnotations(annotations)
deployment := &appsv1.Deployment{}
Expand All @@ -287,11 +289,29 @@ func (c *WorkspaceReconciler) updateControllerRevision(ctx context.Context, wObj

if hash, exists := deployment.Annotations[WorkspaceRevisionAnnotation]; !exists || (hash != currentHash) {

initContainers, envs := resources.GenerateInitContainers(wObj)
var volumes []corev1.Volume
var volumeMounts []corev1.VolumeMount
shmVolume, shmVolumeMount := utils.ConfigSHMVolume(*wObj.Resource.Count)
if shmVolume.Name != "" {
volumes = append(volumes, shmVolume)
}
if shmVolumeMount.Name != "" {
volumeMounts = append(volumeMounts, shmVolumeMount)
}

if len(wObj.Inference.Adapters) > 0 {
adapterVolume, adapterVolumeMount := utils.ConfigAdapterVolume()
volumes = append(volumes, adapterVolume)
volumeMounts = append(volumeMounts, adapterVolumeMount)
}

initContainers, envs := resources.GenerateInitContainers(wObj, volumeMounts)
spec := &deployment.Spec
spec.Template.Spec.InitContainers = initContainers
spec.Template.Spec.Containers[0].Env = envs
spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
deployment.Annotations[WorkspaceRevisionAnnotation] = currentHash
spec.Template.Spec.Volumes = volumes

if err := c.Update(ctx, deployment); err != nil {
return fmt.Errorf("failed to update deployment: %w", err)
Expand Down
6 changes: 3 additions & 3 deletions pkg/resources/manifests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ func GenerateDeploymentManifest(ctx context.Context, workspaceObj *kaitov1alpha1
initContainers := []corev1.Container{}
envs := []corev1.EnvVar{}
if len(workspaceObj.Inference.Adapters) > 0 {
initContainers, envs = GenerateInitContainers(workspaceObj)
initContainers, envs = GenerateInitContainers(workspaceObj, volumeMount)
}

return &appsv1.Deployment{
Expand Down Expand Up @@ -334,7 +334,7 @@ func GenerateDeploymentManifest(ctx context.Context, workspaceObj *kaitov1alpha1
}
}

func GenerateInitContainers(wObj *kaitov1alpha1.Workspace) ([]corev1.Container, []corev1.EnvVar) {
func GenerateInitContainers(wObj *kaitov1alpha1.Workspace, volumeMount []corev1.VolumeMount) ([]corev1.Container, []corev1.EnvVar) {
initContainers := []corev1.Container{}
envs := []corev1.EnvVar{}
if len(wObj.Inference.Adapters) > 0 {
Expand All @@ -343,7 +343,7 @@ func GenerateInitContainers(wObj *kaitov1alpha1.Workspace) ([]corev1.Container,
Name: adapter.Source.Name,
Image: adapter.Source.Image,
Command: []string{"/bin/sh", "-c", fmt.Sprintf("mkdir -p /mnt/adapter/%s && cp -r /data/* /mnt/adapter/%s", adapter.Source.Name, adapter.Source.Name)},
VolumeMounts: []corev1.VolumeMount{},
VolumeMounts: volumeMount,
ImagePullPolicy: corev1.PullAlways,
}
initContainers = append(initContainers, initContainer)
Expand Down
192 changes: 97 additions & 95 deletions test/e2e/preset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,149 +561,151 @@ var _ = Describe("Workspace Preset", func() {
Fail("Fail threshold reached")
}
})
/*
It("should create a mistral workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createMistralWorkspaceWithPresetPublicMode(numOfNode)
It("should create a mistral workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createMistralWorkspaceWithPresetPublicMode(numOfNode)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a Phi-2 workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createPhi2WorkspaceWithPresetPublicMode(numOfNode)
It("should create a Phi-2 workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createPhi2WorkspaceWithPresetPublicMode(numOfNode)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a falcon workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createFalconWorkspaceWithPresetPublicMode(numOfNode)
It("should create a falcon workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createFalconWorkspaceWithPresetPublicMode(numOfNode)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a llama 7b workspace with preset private mode successfully", func() {
numOfNode := 1
modelVersion, ok := modelInfo[PresetLlama2AChat]
if !ok {
Fail(fmt.Sprintf("Model version for %s not found", PresetLlama2AChat))
}
workspaceObj := createLlama7BWorkspaceWithPresetPrivateMode(aiModelsRegistry, aiModelsRegistrySecret, modelVersion, numOfNode)
It("should create a llama 7b workspace with preset private mode successfully", func() {
numOfNode := 1
modelVersion, ok := modelInfo[PresetLlama2AChat]
if !ok {
Fail(fmt.Sprintf("Model version for %s not found", PresetLlama2AChat))
}
workspaceObj := createLlama7BWorkspaceWithPresetPrivateMode(aiModelsRegistry, aiModelsRegistrySecret, modelVersion, numOfNode)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a llama 13b workspace with preset private mode successfully", func() {
if !runLlama13B {
Skip("Skipping llama 13b workspace test")
}
numOfNode := 2
modelVersion, ok := modelInfo[PresetLlama2BChat]
if !ok {
Fail(fmt.Sprintf("Model version for %s not found", PresetLlama2AChat))
}
workspaceObj := createLlama13BWorkspaceWithPresetPrivateMode(aiModelsRegistry, aiModelsRegistrySecret, modelVersion, numOfNode)
It("should create a llama 13b workspace with preset private mode successfully", func() {
if !runLlama13B {
Skip("Skipping llama 13b workspace test")
}
numOfNode := 2
modelVersion, ok := modelInfo[PresetLlama2BChat]
if !ok {
Fail(fmt.Sprintf("Model version for %s not found", PresetLlama2AChat))
}
workspaceObj := createLlama13BWorkspaceWithPresetPrivateMode(aiModelsRegistry, aiModelsRegistrySecret, modelVersion, numOfNode)
defer cleanupResources(workspaceObj)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), true)
validateInferenceResource(workspaceObj, int32(numOfNode), true)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a custom template workspace successfully", func() {
numOfNode := 1
imageName := "nginx:latest"
workspaceObj := createCustomWorkspaceWithPresetCustomMode(imageName, numOfNode)
It("should create a custom template workspace successfully", func() {
numOfNode := 1
imageName := "nginx:latest"
workspaceObj := createCustomWorkspaceWithPresetCustomMode(imageName, numOfNode)
defer cleanupResources(workspaceObj)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
It("should create a Phi-3-mini-128k-instruct workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createPhi3WorkspaceWithPresetPublicMode(numOfNode)
It("should create a Phi-3-mini-128k-instruct workspace with preset public mode successfully", func() {
numOfNode := 1
workspaceObj := createPhi3WorkspaceWithPresetPublicMode(numOfNode)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
defer cleanupResources(workspaceObj)
time.Sleep(30 * time.Second)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
validateMachineCreation(workspaceObj, numOfNode)
validateResourceStatus(workspaceObj)
time.Sleep(30 * time.Second)
time.Sleep(30 * time.Second)
validateAssociatedService(workspaceObj)
validateAssociatedService(workspaceObj)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateInferenceResource(workspaceObj, int32(numOfNode), false)
validateWorkspaceReadiness(workspaceObj)
})
validateWorkspaceReadiness(workspaceObj)
})
*/

It("should create a workspace for tuning successfully", func() {
numOfNode := 1
Expand Down
Loading

0 comments on commit 4736725

Please sign in to comment.