Fix kubeflow pipeline and reuse command (#4009)

microsoft · Aug 4, 2021 · 4b5212d · 4b5212d
1 parent a9ebcf4
commit 4b5212d
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 9 deletions.
diff --git a/pipelines/integration-test-frameworkcontroller.yml b/pipelines/integration-test-frameworkcontroller.yml
@@ -42,10 +42,10 @@ jobs:
       cd test
       python3 nni_test/nnitest/generate_ts_config.py \
           --ts frameworkcontroller \
-          --keyvault_vaultname NNIKeyVault \
-          --keyvault_name AzureStorageAccountKey \
-          --azs_account nniazurestorage \
-          --azs_share nni \
+          --keyvault_vaultname $(keyvault_vaultname}) \
+          --keyvault_name $(keyvault_name) \
+          --azs_account $(azs_account) \
+          --azs_share $(azs_share) \
           --nni_docker_image nnidev/nni-nightly \
           --nni_manager_ip $(manager_ip)
       python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts frameworkcontroller --exclude multi-phase,multi-thread

diff --git a/pipelines/integration-test-kubeflow.yml b/pipelines/integration-test-kubeflow.yml
@@ -52,10 +52,10 @@ jobs:
       cd test
       python3 nni_test/nnitest/generate_ts_config.py \
           --ts kubeflow \
-          --keyvault_vaultname NNIKeyVault \
-          --keyvault_name AzureStorageAccountKey \
-          --azs_account nniazurestorage \
-          --azs_share nni \
+          --keyvault_vaultname $(keyvault_vaultname) \
+          --keyvault_name $(keyvault_name) \
+          --azs_account $(azs_account) \
+          --azs_share $(azs_share) \
           --nni_docker_image nnidev/nni-nightly \
           --nni_manager_ip $(manager_ip)
       python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts kubeflow --exclude multi-phase,multi-thread

diff --git a/...i_manager/training_service/reusable/environments/kubernetes/kubeflowEnvironmentService.ts b/...i_manager/training_service/reusable/environments/kubernetes/kubeflowEnvironmentService.ts
@@ -81,7 +81,9 @@ export class KubeflowEnvironmentService extends KubernetesEnvironmentService {
         if (this.createStoragePromise) {
             await this.createStoragePromise;
         }
-        environment.command = `mv envs outputs/envs && cd outputs && ${environment.command}`;
+        const expFolder = `${this.CONTAINER_MOUNT_PATH}/nni/${this.experimentId}`;
+        environment.command = `cd ${expFolder} && ${environment.command} \
+        1>${expFolder}/envs/${environment.id}/trialrunner_stdout 2>${expFolder}/envs/${environment.id}/trialrunner_stderr`;
         if (this.config.deprecated && this.config.deprecated.useActiveGpu !== undefined) {
             environment.useActiveGpu = this.config.deprecated.useActiveGpu;
         }