hotfix unhandled TrainingService is not assigned and extend exec ti…

…me in pipeline (#3442)
microsoft · Mar 10, 2021 · a272da9 · a272da9
1 parent 62af469
commit a272da9
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 16 deletions.
diff --git a/test/config/assessors/curvefitting.yml b/test/config/assessors/curvefitting.yml
@@ -1,6 +1,6 @@
 authorName: nni
 experimentName: default_test
-maxExecDuration: 5m
+maxExecDuration: 10m
 maxTrialNum: 8
 trialConcurrency: 8
 searchSpacePath: ../naive_trial/search_space.json

diff --git a/test/nni_test/nnitest/run_tests.py b/test/nni_test/nnitest/run_tests.py
@@ -260,9 +260,9 @@ def run(args):
             continue
         # remote mode need more time to cleanup 
         if args.ts == 'remote':
-            wait_for_port_available(8080, 180)
+            wait_for_port_available(8080, 240)
         else:
-            wait_for_port_available(8080, 30)
+            wait_for_port_available(8080, 60)
 
         # adl mode need more time to cleanup PVC
         if args.ts == 'adl' and name == 'nnictl-resume-2':

diff --git a/ts/nni_manager/core/nnimanager.ts b/ts/nni_manager/core/nnimanager.ts
@@ -326,22 +326,26 @@ class NNIManager implements Manager {
     }
 
     public async stopExperimentBottomHalf(): Promise<void> {
-        const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs();
-
-        // DON'T try to make it in parallel, the training service may not handle it well.
-        // If there is performance concern, consider to support batch cancellation on training service.
-        for (const trialJob of trialJobList) {
-            if (trialJob.status === 'RUNNING' ||
-                trialJob.status === 'WAITING') {
-                try {
-                    this.log.info(`cancelTrialJob: ${trialJob.id}`);
-                    await this.trainingService.cancelTrialJob(trialJob.id);
-                } catch (error) {
-                    this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`);
+        try {
+            const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs();
+
+            // DON'T try to make it in parallel, the training service may not handle it well.
+            // If there is performance concern, consider to support batch cancellation on training service.
+            for (const trialJob of trialJobList) {
+                if (trialJob.status === 'RUNNING' ||
+                    trialJob.status === 'WAITING') {
+                    try {
+                        this.log.info(`cancelTrialJob: ${trialJob.id}`);
+                        await this.trainingService.cancelTrialJob(trialJob.id);
+                    } catch (error) {
+                        this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`);
+                    }
                 }
             }
+            await this.trainingService.cleanUp();
+        } catch (err) {
+            this.log.error(`${err.stack}`);
         }
-        await this.trainingService.cleanUp();
         if (this.experimentProfile.endTime === undefined) {
             this.setEndtime();
         }