-
Notifications
You must be signed in to change notification settings - Fork 70
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore: e2e test for adapters and validation of adapters (#483)
**Reason for Change**: <!-- What does this PR improve or fix in Kaito? Why is it needed? --> **Requirements** - [ ] added unit tests and e2e tests (if applicable). **Issue Fixed**: <!-- If this PR fixes GitHub issue 4321, add "Fixes #4321" to the next line. --> **Notes for Reviewers**: --------- Signed-off-by: Bangqi Zhu <bangqizhu@microsoft.com> Co-authored-by: Bangqi Zhu <bangqizhu@microsoft.com>
- Loading branch information
1 parent
5242169
commit f613679
Showing
11 changed files
with
250 additions
and
41 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,6 @@ | ||
| FROM busybox:latest | ||
|
|
||
| RUN mkdir -p /data | ||
|
|
||
| COPY docker/adapter/adapter_config.json /data/ | ||
| COPY docker/adapter/adapter_model.safetensors /data/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| # E2E Adapter Test Files | ||
|
|
||
| ## Overview | ||
|
|
||
| These files are part of a set used for conducting end-to-end (E2E) testing of an adapter component. The Dockerfile builds an image incorporating the configuration and model files, which is then used within an Init Container for testing. The adapter is training from [dolly-15k-oai-style](https://huggingface.co/datasets/philschmid/dolly-15k-oai-style) dataset | ||
| and was trained using default [qlora-params.yaml](../../charts/kaito/workspace/templates/qlora-params.yaml) | ||
|
|
||
| ## Files | ||
|
|
||
| - **Dockerfile**: Builds the Docker image for the E2E tests. | ||
|
|
||
| - **adapter_config.json**: Contains settings for configuring the adapter in the test environment. | ||
|
|
||
| - **adapter_model.safetensors**: Provides the adapter's machine learning model in SafeTensors format. | ||
|
|
||
| ## Usage | ||
|
|
||
| Build the Docker image with the following command: | ||
|
|
||
| ```bash | ||
|
|
||
| make docker-build-adapter | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| { | ||
| "alpha_pattern": {}, | ||
| "auto_mapping": { | ||
| "base_model_class": "FalconForCausalLM", | ||
| "parent_library": "transformers.models.falcon.modeling_falcon" | ||
| }, | ||
| "base_model_name_or_path": "/workspace/tfs/weights", | ||
| "bias": "none", | ||
| "fan_in_fan_out": false, | ||
| "inference_mode": true, | ||
| "init_lora_weights": true, | ||
| "layers_pattern": null, | ||
| "layers_to_transform": null, | ||
| "loftq_config": {}, | ||
| "lora_alpha": 8, | ||
| "lora_dropout": 0.0, | ||
| "megatron_config": null, | ||
| "megatron_core": "megatron.core", | ||
| "modules_to_save": null, | ||
| "peft_type": "LORA", | ||
| "r": 8, | ||
| "rank_pattern": {}, | ||
| "revision": null, | ||
| "target_modules": [ | ||
| "query_key_value" | ||
| ], | ||
| "task_type": null, | ||
| "use_rslora": false | ||
| } |
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // Copyright (c) Microsoft Corporation. | ||
| // Licensed under the MIT license. | ||
|
|
||
| package e2e | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| kaitov1alpha1 "github.com/azure/kaito/api/v1alpha1" | ||
| "github.com/azure/kaito/test/e2e/utils" | ||
| . "github.com/onsi/ginkgo/v2" | ||
| . "github.com/onsi/gomega" | ||
| appsv1 "k8s.io/api/apps/v1" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| var DefaultStrength = "1.0" | ||
|
|
||
| var imageName = "e2e-adapter" | ||
| var fullImageName = utils.GetEnv("REGISTRY") + "/" + imageName + ":0.0.1" | ||
|
|
||
| var validAdapters = []kaitov1alpha1.AdapterSpec{ | ||
| { | ||
| Source: &kaitov1alpha1.DataSource{ | ||
| Name: imageName, | ||
| Image: fullImageName, | ||
| }, | ||
| Strength: &DefaultStrength, | ||
| }, | ||
| } | ||
|
|
||
| var expectedInitContainers = []corev1.Container{ | ||
| { | ||
| Name: imageName, | ||
| Image: fullImageName, | ||
| }, | ||
| } | ||
|
|
||
| func validateAdapters(workspaceObj *kaitov1alpha1.Workspace, expectedInitContainers []corev1.Container) { | ||
| By("Checking the Adapters", func() { | ||
| Eventually(func() bool { | ||
| var err error | ||
| var initContainers []corev1.Container | ||
|
|
||
| dep := &appsv1.Deployment{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Name: workspaceObj.Name, | ||
| Namespace: workspaceObj.Namespace, | ||
| }, | ||
| } | ||
| err = TestingCluster.KubeClient.Get(ctx, client.ObjectKey{ | ||
| Namespace: workspaceObj.Namespace, | ||
| Name: workspaceObj.Name, | ||
| }, dep) | ||
| initContainers = dep.Spec.Template.Spec.InitContainers | ||
|
|
||
| if err != nil { | ||
| GinkgoWriter.Printf("Error fetching resource: %v\n", err) | ||
| return false | ||
| } | ||
|
|
||
| if len(initContainers) != len(expectedInitContainers) { | ||
| return false | ||
| } | ||
| initContainer, expectedInitContainer := initContainers[0], expectedInitContainers[0] | ||
|
|
||
| // GinkgoWriter.Printf("Resource '%s' not ready. Ready replicas: %d\n", workspaceObj.Name, readyReplicas) | ||
| return initContainer.Image == expectedInitContainer.Image && initContainer.Name == expectedInitContainer.Name | ||
| }, 20*time.Minute, utils.PollInterval).Should(BeTrue(), "Failed to wait for adapter resource to be ready") | ||
| }) | ||
| } | ||
|
|
||
| var _ = Describe("Workspace Preset", func() { | ||
| BeforeEach(func() { | ||
| loadTestEnvVars() | ||
|
|
||
| loadModelVersions() | ||
| }) | ||
|
|
||
| It("should create a falcon workspace with adapter", func() { | ||
| numOfNode := 1 | ||
| workspaceObj := createCustomWorkspaceWithAdapter(numOfNode) | ||
|
|
||
| defer cleanupResources(workspaceObj) | ||
| time.Sleep(30 * time.Second) | ||
|
|
||
| validateMachineCreation(workspaceObj, numOfNode) | ||
| validateResourceStatus(workspaceObj) | ||
|
|
||
| time.Sleep(30 * time.Second) | ||
|
|
||
| validateAssociatedService(workspaceObj) | ||
|
|
||
| validateInferenceResource(workspaceObj, int32(numOfNode), false) | ||
|
|
||
| validateWorkspaceReadiness(workspaceObj) | ||
|
|
||
| validateAdapters(workspaceObj, expectedInitContainers) | ||
| }) | ||
|
|
||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.