Skip to content

Commit

Permalink
chore: tuning e2e webhook test (#487)
Browse files Browse the repository at this point in the history
**Reason for Change**:
<!-- What does this PR improve or fix in Kaito? Why is it needed? -->

**Requirements**

- [ ] added unit tests and e2e tests (if applicable).

**Issue Fixed**:
<!-- If this PR fixes GitHub issue 4321, add "Fixes #4321" to the next
line. -->

**Notes for Reviewers**:

Signed-off-by: Bangqi Zhu <bangqizhu@microsoft.com>
Co-authored-by: Bangqi Zhu <bangqizhu@microsoft.com>
  • Loading branch information
bangqipropel and Bangqi Zhu authored Jun 28, 2024
1 parent f534355 commit 5242169
Show file tree
Hide file tree
Showing 2 changed files with 124 additions and 0 deletions.
26 changes: 26 additions & 0 deletions test/e2e/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,32 @@ func GenerateInferenceWorkspaceManifest(name, namespace, imageName string, resou
return workspace
}

func GenerateTuningWorkspaceManifest(name, namespace, imageName string, resourceCount int, instanceType string,
labelSelector *metav1.LabelSelector, preferredNodes []string, input *kaitov1alpha1.DataSource,
output *kaitov1alpha1.DataDestination, preset *kaitov1alpha1.PresetSpec, method kaitov1alpha1.TuningMethod) *kaitov1alpha1.Workspace {

workspace := &kaitov1alpha1.Workspace{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Resource: kaitov1alpha1.ResourceSpec{
Count: lo.ToPtr(resourceCount),
InstanceType: instanceType,
LabelSelector: labelSelector,
PreferredNodes: preferredNodes,
},
Tuning: &kaitov1alpha1.TuningSpec{
Method: method,
Input: input,
Output: output,
Preset: preset,
},
}

return workspace
}

func GeneratePodTemplate(name, namespace, image string, labels map[string]string) *corev1.PodTemplateSpec {
return &corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Expand Down
98 changes: 98 additions & 0 deletions test/e2e/webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client"
)

var (
testModelImage = utils.GetEnv("AI_MODELS_REGISTRY") + "/e2e-test:0.0.1"
testDataSourceConfig = &kaitov1alpha1.DataSource{Name: PresetFalcon7BModel, Image: testModelImage}
testDataDestinationConfig = &kaitov1alpha1.DataDestination{Image: testModelImage, ImagePushSecret: utils.GetEnv("AI_MODELS_REGISTRY_SECRET")}

initialPresetSpec = &kaitov1alpha1.PresetSpec{PresetMeta: kaitov1alpha1.PresetMeta{Name: PresetFalcon7BModel}}
updatedPresetSpec = &kaitov1alpha1.PresetSpec{PresetMeta: kaitov1alpha1.PresetMeta{Name: PresetFalcon40BModel}}

initialTuningMethod = kaitov1alpha1.TuningMethodLora
alternativeTuningMethod = kaitov1alpha1.TuningMethodQLora
)

var _ = Describe("Workspace Validation Webhook", func() {
It("should validate the workspace resource spec at creation ", func() {
workspaceObj := utils.GenerateInferenceWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_Bad",
Expand Down Expand Up @@ -47,6 +59,51 @@ var _ = Describe("Workspace Validation Webhook", func() {
})
})

It("should validate the workspace tuning spec at creation ", func() {
workspaceObj := utils.GenerateTuningWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_NC12s_v3",
&metav1.LabelSelector{
MatchLabels: map[string]string{"kaito-workspace": "webhook-e2e-test"},
}, nil, nil, testDataDestinationConfig, initialPresetSpec, initialTuningMethod)

By("Creating a workspace with nil input", func() {
// Create workspace
Eventually(func() error {
return TestingCluster.KubeClient.Create(ctx, workspaceObj, &client.CreateOptions{})
}, 20*time.Minute, utils.PollInterval).
Should(HaveOccurred(), "Failed to create workspace %s", workspaceObj.Name)
})
})

It("should validate the workspace tuning spec at creation ", func() {
workspaceObj := utils.GenerateTuningWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_NC12s_v3",
&metav1.LabelSelector{
MatchLabels: map[string]string{"kaito-workspace": "webhook-e2e-test"},
}, nil, testDataSourceConfig, nil, initialPresetSpec, initialTuningMethod)

By("Creating a workspace with nil output", func() {
// Create workspace
Eventually(func() error {
return TestingCluster.KubeClient.Create(ctx, workspaceObj, &client.CreateOptions{})
}, 20*time.Minute, utils.PollInterval).
Should(HaveOccurred(), "Failed to create workspace %s", workspaceObj.Name)
})
})

It("should validate the workspace tuning spec at creation ", func() {
workspaceObj := utils.GenerateTuningWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_NC12s_v3",
&metav1.LabelSelector{
MatchLabels: map[string]string{"kaito-workspace": "webhook-e2e-test"},
}, nil, testDataSourceConfig, testDataDestinationConfig, nil, initialTuningMethod)

By("Creating a workspace with nil preset", func() {
// Create workspace
Eventually(func() error {
return TestingCluster.KubeClient.Create(ctx, workspaceObj, &client.CreateOptions{})
}, 20*time.Minute, utils.PollInterval).
Should(HaveOccurred(), "Failed to create workspace %s", workspaceObj.Name)
})
})

//TODO preset private mode
//TODO custom template

Expand Down Expand Up @@ -93,6 +150,47 @@ var _ = Describe("Workspace Validation Webhook", func() {

})

It("should validate the workspace tuning spec at update ", func() {
workspaceObj := utils.GenerateTuningWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_NC12s_v3",
&metav1.LabelSelector{
MatchLabels: map[string]string{"kaito-workspace": "webhook-e2e-test"},
}, nil, testDataSourceConfig, testDataDestinationConfig, initialPresetSpec, initialTuningMethod)

By("Creating a valid tuning workspace", func() {
// Create workspace
Eventually(func() error {
return TestingCluster.KubeClient.Create(ctx, workspaceObj, &client.CreateOptions{})
}, 20*time.Minute, utils.PollInterval).
Should(Succeed(), "Failed to create workspace %s", workspaceObj.Name)
})

By("Updating the tuning preset", func() {
updatedObj := workspaceObj
updatedObj.Tuning.Preset = updatedPresetSpec
// update workspace
Eventually(func() error {
return TestingCluster.KubeClient.Update(ctx, updatedObj, &client.UpdateOptions{})
}, utils.PollTimeout, utils.PollInterval).
Should(HaveOccurred(), "Failed to update workspace %s", updatedObj.Name)
})

By("Updating the Method", func() {
updatedObj := workspaceObj
updatedObj.Tuning.Method = alternativeTuningMethod
// update workspace
Eventually(func() error {
return TestingCluster.KubeClient.Update(ctx, updatedObj, &client.UpdateOptions{})
}, utils.PollTimeout, utils.PollInterval).
Should(HaveOccurred(), "Failed to update workspace %s", updatedObj.Name)
})

// delete workspace
Eventually(func() error {
return TestingCluster.KubeClient.Delete(ctx, workspaceObj, &client.DeleteOptions{})
}, utils.PollTimeout, utils.PollInterval).Should(Succeed(), "Failed to delete workspace")

})

It("should validate the workspace inference spec at update ", func() {
workspaceObj := utils.GenerateInferenceWorkspaceManifest(fmt.Sprint("webhook-", rand.Intn(1000)), namespaceName, "", 1, "Standard_NC12s_v3",
&metav1.LabelSelector{
Expand Down

0 comments on commit 5242169

Please sign in to comment.