Add UT for trial dispatcher, and fix some bugs

src/nni_manager/training_service/reusable/channels/amlCommandChannel.ts

@@ -3,7 +3,6 @@
 
 'use strict';
 
-import { EventEmitter } from 'events';
 import { delay } from "../../../common/utils";
 import { AMLEnvironmentInformation } from '../aml/amlConfig';
 import { CommandChannel, RunnerConnection } from "../commandChannel";
@@ -16,11 +15,7 @@ export class AMLCommandChannel extends CommandChannel {
     private stopping: boolean = false;
     private currentMessageIndex: number = -1;
     private sendQueues: [EnvironmentInformation, string][] = [];
-    private readonly NNI_METRICS_PATTERN: string = `NNISDK_MEb'(?<metrics>.*?)'`;
-
-    public constructor(commandEmitter: EventEmitter) {
-        super(commandEmitter);
-    }
+
     public get channelName(): Channel {
         return "aml";
     }
@@ -98,11 +93,11 @@ export class AMLCommandChannel extends CommandChannel {
                     const messages = command['trial_runner'];
                     if (messages) {
                         if (messages instanceof Object && this.currentMessageIndex < messages.length - 1) {
-                            for (let index = this.currentMessageIndex + 1; index < messages.length; index ++) {
+                            for (let index = this.currentMessageIndex + 1; index < messages.length; index++) {
                                 this.handleCommand(runnerConnection.environment, messages[index]);
                             }
                             this.currentMessageIndex = messages.length - 1;
-                        } else if (this.currentMessageIndex === -1){
+                        } else if (this.currentMessageIndex === -1) {
                             this.handleCommand(runnerConnection.environment, messages);
                             this.currentMessageIndex += 1;
                         }

src/nni_manager/training_service/reusable/environment.ts

@@ -69,7 +69,20 @@ export abstract class EnvironmentService {
     public abstract startEnvironment(environment: EnvironmentInformation): Promise<void>;
     public abstract stopEnvironment(environment: EnvironmentInformation): Promise<void>;
 
-    public getCommandChannel(commandEmitter: EventEmitter): CommandChannel {
+    // It depends on environment pressure and settings
+    // for example, OpenPAI relies on API calls, and there is an limitation for frequence, so it need to be bigger.
+    public get environmentMaintenceLoopInterval(): number {
+        return 5000;
+    }
+
+    // it's needed in two scenario
+    // 1. remote machine has fixed number, so it can return false, when all environment are assigned.
+    // 2. If there are consistent error on requested environments, for example, authentication failure on platform.
+    public get hasMoreEnvironments(): boolean {
+        return true;
+    }
+
+    public createCommandChannel(commandEmitter: EventEmitter): CommandChannel {
         return new WebCommandChannel(commandEmitter);
     }
 

src/nni_manager/training_service/reusable/environments/amlEnvironmentService.ts

@@ -41,7 +41,7 @@ export class AMLEnvironmentService extends EnvironmentService {
         return false;
     }
 
-    public getCommandChannel(commandEmitter: EventEmitter): CommandChannel {
+    public createCommandChannel(commandEmitter: EventEmitter): CommandChannel {
         return new AMLCommandChannel(commandEmitter);
     }
 

src/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts

@@ -40,6 +40,10 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
         this.experimentId = getExperimentId();
     }
 
+    public get environmentMaintenceLoopInterval(): number {
+        return 5000;
+    }
+
     public get hasStorageService(): boolean {
         return true;
     }

src/nni_manager/training_service/reusable/test/mountedStorageService.test.ts

@@ -101,25 +101,25 @@ describe('Unit Test for MountedStorageService', () => {
         chai.assert.isTrue(fs.existsSync(localNewName));
     })
 
-    // it('FileContentTest', async () => {
-    //     const savedFileName = "savedfile.txt";
-    //     await service.save("01234", savedFileName);
-    //     chai.expect(fs.existsSync(savedFileName));
+    it('FileContentTest', async () => {
+        const savedFileName = "savedfile.txt";
+        await service.save("01234", savedFileName);
+        chai.expect(fs.existsSync(savedFileName));
 
-    //     let content = await service.readFileContent(savedFileName, 0, -1);
-    //     chai.assert.equal(content, "01234");
+        let content = await service.readFileContent(savedFileName, 0, -1);
+        chai.assert.equal(content, "01234");
 
-    //     await service.save("56789", savedFileName, true);
-    //     content = await service.readFileContent(savedFileName, 0, -1);
-    //     chai.assert.equal(content, "0123456789");
+        await service.save("56789", savedFileName, true);
+        content = await service.readFileContent(savedFileName, 0, -1);
+        chai.assert.equal(content, "0123456789");
 
-    //     content = await service.readFileContent(savedFileName, -1, 1);
-    //     chai.assert.equal(content, "0");
+        content = await service.readFileContent(savedFileName, -1, 1);
+        chai.assert.equal(content, "0");
 
-    //     content = await service.readFileContent(savedFileName, 5, 1);
-    //     chai.assert.equal(content, "5");
+        content = await service.readFileContent(savedFileName, 5, 1);
+        chai.assert.equal(content, "5");
 
-    //     content = await service.readFileContent(savedFileName, 5, -1);
-    //     chai.assert.equal(content, "56789");
-    // });
+        content = await service.readFileContent(savedFileName, 5, -1);
+        chai.assert.equal(content, "56789");
+    });
 });