Merge pull request #119 from Microsoft/master

merge master

Makefile

@@ -1,6 +1,6 @@
 # Setting variables
 SHELL := /bin/bash
-PIP_INSTALL := python3 -m pip install --no-cache-dir
+PIP_INSTALL := python3 -m pip install
 PIP_UNINSTALL := python3 -m pip uninstall
 
 ## Colorful output
@@ -19,22 +19,25 @@ else
 endif
 
 ## Install directories
-ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
-IS_SYS_PYTHON ?= $(shell [[ $(ROOT_FOLDER) == /usr* || $(ROOT_FOLDER) == /Library* ]] && echo TRUE || echo FALSE)
-
-ifeq ($(shell id -u), 0)  # is root
-    _ROOT := 1
-    BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions
-else  # is normal user
-    ifeq (TRUE, $(IS_SYS_PYTHON))
-        ROOT_FOLDER := $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
-    endif
-    ifndef VIRTUAL_ENV
-        ifeq (, $(shell echo $$PATH | grep 'conda'))
+
+## For apt-get or pip installed virtualenv
+ifdef VIRTUAL_ENV
+    ROOT_FOLDER ?= $(VIRTUAL_ENV)
+    BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d
+else
+    ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
+    IS_SYS_PYTHON ?= $(shell [[ $(ROOT_FOLDER) == /usr* || $(ROOT_FOLDER) == /Library* ]] && echo TRUE || echo FALSE)
+
+    ifeq ($(shell id -u), 0)  # is root
+        _ROOT := 1
+        BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions
+    else  # is normal user
+        ifeq (TRUE, $(IS_SYS_PYTHON))
+            ROOT_FOLDER := $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
             PIP_MODE ?= --user
         endif
+        BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d
     endif
-    BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d
 endif
 BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl
 

README.md

@@ -4,7 +4,7 @@
 
 -----------
 
-[![MIT licensed](https://img.shields.io/badge/license-MIT-yellow.svg)](https://github.com/Microsoft/nni/blob/master/LICENSE)
+[![MIT licensed](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://github.com/Microsoft/nni/blob/master/LICENSE)
 [![Build Status](https://msrasrg.visualstudio.com/NNIOpenSource/_apis/build/status/Microsoft.nni)](https://msrasrg.visualstudio.com/NNIOpenSource/_build/latest?definitionId=6)
 [![Issues](https://img.shields.io/github/issues-raw/Microsoft/nni.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen)
 [![Bugs](https://img.shields.io/github/issues/Microsoft/nni/bug.svg)](https://github.com/Microsoft/nni/issues?q=is%3Aissue+is%3Aopen+label%3Abug)

docs/ExperimentConfig.md

@@ -177,8 +177,18 @@ machineList:
     __nniManagerIp__ set the IP address of the machine on which nni manager process runs. This field is optional, and if it's not set, eth0 device IP will be used instead.
 
     Note: run ifconfig on NNI manager's machine to check if eth0 device exists. If not, we recommend to set nnimanagerIp explicitly.
-
-
+
+* __logDir__
+  * Description
+
+    __logDir__ configures the directory to store logs and data of the experiment. The default value is `<user home directory>/nni/experiment`
+
+* __logLevel__
+  * Description
+
+    __logLevel__ sets log level for the experiment, available log levels are: `trace, debug, info, warning, error, fatal`. The default value is `info`.
+
+
 * __tuner__
   * Description
 

docs/NNICTLDOC.md

@@ -43,6 +43,7 @@ nnictl --version
     | ------ | ------ | ------ |------ |
     | --config, -c|  True| |yaml configure file of the experiment|
     | --port, -p  |  False| |the port of restful server|
+    | --debug, -d |  False| |Set log level to debug|
 
 * __nnictl resume__
 
@@ -62,6 +63,7 @@ nnictl --version
     | ------ | ------ | ------ |------ |
     | id|  False| |The id of the experiment you want to resume|  
     | --port, -p|  False| |Rest port of the experiment you want to resume|
+    | --debug, -d |  False| |Set log level to debug|
 
 * __nnictl stop__
   * Description

src/nni_manager/common/experimentStartupInfo.ts

@@ -20,6 +20,8 @@
 'use strict';
 
 import * as assert from 'assert';
+import * as os from 'os';
+import * as path from 'path';
 import * as component from '../common/component';
 
 @component.Singleton
@@ -29,15 +31,27 @@ class ExperimentStartupInfo {
     private basePort: number = -1;
     private initialized: boolean = false;
     private initTrialSequenceID: number = 0;
+    private logDir: string = '';
+    private logLevel: string = '';
 
-    public setStartupInfo(newExperiment: boolean, experimentId: string, basePort: number): void {
+    public setStartupInfo(newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string): void {
         assert(!this.initialized);
         assert(experimentId.trim().length > 0);
 
         this.newExperiment = newExperiment;
         this.experimentId = experimentId;
         this.basePort = basePort;
         this.initialized = true;
+
+        if (logDir !== undefined && logDir.length > 0) {
+            this.logDir = path.join(logDir, getExperimentId());
+        } else {
+            this.logDir = path.join(os.homedir(), 'nni', 'experiments', getExperimentId());
+        }
+
+        if (logLevel !== undefined && logLevel.length > 1) {
+            this.logLevel = logLevel;
+        }
     }
 
     public getExperimentId(): string {
@@ -58,6 +72,18 @@ class ExperimentStartupInfo {
         return this.newExperiment;
     }
 
+    public getLogDir(): string {
+        assert(this.initialized);
+
+        return this.logDir;
+    }
+
+    public getLogLevel(): string {
+        assert(this.initialized);
+
+        return this.logLevel;
+    }
+
     public setInitTrialSequenceId(initSequenceId: number): void {
         assert(this.initialized);
         this.initTrialSequenceID = initSequenceId;
@@ -90,9 +116,15 @@ function getInitTrialSequenceId(): number {
     return component.get<ExperimentStartupInfo>(ExperimentStartupInfo).getInitTrialSequenceId();
 }
 
-function setExperimentStartupInfo(newExperiment: boolean, experimentId: string, basePort: number): void {
-    component.get<ExperimentStartupInfo>(ExperimentStartupInfo).setStartupInfo(newExperiment, experimentId, basePort);
+function getExperimentStartupInfo(): ExperimentStartupInfo {
+    return component.get<ExperimentStartupInfo>(ExperimentStartupInfo);
+}
+
+function setExperimentStartupInfo(
+    newExperiment: boolean, experimentId: string, basePort: number, logDir?: string, logLevel?: string): void {
+    component.get<ExperimentStartupInfo>(ExperimentStartupInfo)
+    .setStartupInfo(newExperiment, experimentId, basePort, logDir, logLevel);
 }
 
-export { ExperimentStartupInfo, getBasePort, getExperimentId, isNewExperiment,
+export { ExperimentStartupInfo, getBasePort, getExperimentId, isNewExperiment, getExperimentStartupInfo,
     setExperimentStartupInfo, setInitTrialSequenceId, getInitTrialSequenceId };

src/nni_manager/common/log.ts

@@ -26,13 +26,18 @@ import { Writable } from 'stream';
 import { WritableStreamBuffer } from 'stream-buffers';
 import { format } from 'util';
 import * as component from '../common/component';
+import { getExperimentStartupInfo } from './experimentStartupInfo';
 import { getLogDir } from './utils';
 
-const CRITICAL: number = 1;
+const FATAL: number = 1;
 const ERROR: number = 2;
 const WARNING: number = 3;
 const INFO: number = 4;
 const DEBUG: number = 5;
+const TRACE: number = 6;
+
+const logLevelNameMap: Map<string, number> = new Map([['fatal', FATAL],
+    ['error', ERROR], ['warning', WARNING], ['info', INFO], ['debug', DEBUG], ['trace', TRACE]]);
 
 class BufferSerialEmitter {
     private buffer: Buffer;
@@ -83,12 +88,25 @@ class Logger {
             autoClose: true
         });
         this.bufferSerialEmitter = new BufferSerialEmitter(this.writable);
+
+        const logLevelName: string = getExperimentStartupInfo()
+                                    .getLogLevel();
+        const logLevel: number | undefined = logLevelNameMap.get(logLevelName);
+        if (logLevel !== undefined) {
+            this.level = logLevel;
+        }
     }
 
     public close() {
         this.writable.destroy();
     }
 
+    public trace(...param: any[]): void {
+        if (this.level >= TRACE) {
+            this.log('TRACE', param);
+        }
+    }
+
     public debug(...param: any[]): void {
         if (this.level >= DEBUG) {
             this.log('DEBUG', param);
@@ -113,8 +131,8 @@ class Logger {
         }
     }
 
-    public critical(...param: any[]): void {
-        this.log('CRITICAL', param);
+    public fatal(...param: any[]): void {
+        this.log('FATAL', param);
     }
 
     private log(level: string, param: any[]): void {

src/nni_manager/common/utils.ts

@@ -30,13 +30,14 @@ import { Container } from 'typescript-ioc';
 import * as util from 'util';
 
 import { Database, DataStore } from './datastore';
-import { ExperimentStartupInfo, getExperimentId, setExperimentStartupInfo } from './experimentStartupInfo';
+import { ExperimentStartupInfo, getExperimentId, getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
 import { Manager } from './manager';
 import { HyperParameters, TrainingService, TrialJobStatus } from './trainingService';
 import { getLogger } from './log';
 
 function getExperimentRootDir(): string {
-    return path.join(os.homedir(), 'nni', 'experiments', getExperimentId());
+    return getExperimentStartupInfo()
+            .getLogDir();
 }
 
 function getLogDir(): string{

src/nni_manager/core/nnimanager.ts

@@ -35,7 +35,7 @@ import {
 import {
     TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus
 } from '../common/trainingService';
-import { delay, getCheckpointDir, getLogDir, getMsgDispatcherCommand, mkDirP } from '../common/utils';
+import { delay, getCheckpointDir, getExperimentRootDir, getLogDir, getMsgDispatcherCommand, mkDirP } from '../common/utils';
 import {
     ADD_CUSTOMIZED_TRIAL_JOB, INITIALIZE, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, NO_MORE_TRIAL_JOBS, PING,
     REPORT_METRIC_DATA, REQUEST_TRIAL_JOBS, SEND_TRIAL_JOB_PARAMETER, TERMINATE, TRIAL_END, UPDATE_SEARCH_SPACE
@@ -670,7 +670,7 @@ class NNIManager implements Manager {
             id: getExperimentId(),
             revision: 0,
             execDuration: 0,
-            logDir: getLogDir(),
+            logDir: getExperimentRootDir(),
             maxSequenceId: 0,
             params: {
                 authorName: '',

src/nni_manager/main.ts

@@ -22,6 +22,7 @@
 import { Container, Scope } from 'typescript-ioc';
 
 import * as component from './common/component';
+import * as fs from 'fs';
 import { Database, DataStore } from './common/datastore';
 import { setExperimentStartupInfo } from './common/experimentStartupInfo';
 import { getLogger, Logger } from './common/log';
@@ -40,10 +41,10 @@ import { PAITrainingService } from './training_service/pai/paiTrainingService';
 import { KubeflowTrainingService } from './training_service/kubernetes/kubeflow/kubeflowTrainingService';
 import { FrameworkControllerTrainingService } from './training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService';
 
-function initStartupInfo(startExpMode: string, resumeExperimentId: string, basePort: number) {
+function initStartupInfo(startExpMode: string, resumeExperimentId: string, basePort: number, logDirectory: string, experimentLogLevel: string) {
     const createNew: boolean = (startExpMode === 'new');
     const expId: string = createNew ? uniqueString(8) : resumeExperimentId;
-    setExperimentStartupInfo(createNew, expId, basePort);
+    setExperimentStartupInfo(createNew, expId, basePort, logDirectory, experimentLogLevel);
 }
 
 async function initContainer(platformMode: string): Promise<void> {
@@ -102,7 +103,19 @@ if (startMode === 'resume' && experimentId.trim().length < 1) {
     process.exit(1);
 }
 
-initStartupInfo(startMode, experimentId, port);
+const logDir: string = parseArg(['--log_dir', '-ld']);
+if (logDir.length > 0) {
+    if (!fs.existsSync(logDir)) {
+        console.log(`FATAL: log_dir ${logDir} does not exist`);
+    }
+}
+
+const logLevel: string = parseArg(['--log_level', '-ll']);
+if (logLevel.length > 0 && !['debug', 'info', 'error', 'warning', 'critical'].includes(logLevel)) {
+    console.log(`FATAL: invalid log_level: ${logLevel}`);
+}
+
+initStartupInfo(startMode, experimentId, port, logDir, logLevel);
 
 mkDirP(getLogDir()).then(async () => {
     const log: Logger = getLogger();

src/nni_manager/rest_server/restHandler.ts

@@ -105,7 +105,7 @@ class NNIRestHandler {
 
         // If it's a fatal error, exit process
         if(isFatal) {
-            this.log.critical(err);
+            this.log.fatal(err);
             process.exit(1);
         }
 

src/webui/src/components/SlideBar.tsx

@@ -221,12 +221,12 @@ class SlideBar extends React.Component<{}, SliderState> {
               Download <Icon type="down" />
             </a>
           </Dropdown>
-          <a href="https://github.com/Microsoft/nni/issues/new" target="_blank">
+          <a href="https://github.com/Microsoft/nni/issues/new?labels=v0.5" target="_blank">
             <img
               src={require('../static/img/icon/issue.png')}
               alt="NNI github issue"
             />
-            FeedBack
+            Feedback
           </a>
           <span className="version">Version: {version}</span>
         </Col>

src/webui/src/components/overview/BasicInfo.tsx

@@ -43,7 +43,7 @@ class BasicInfo extends React.Component<BasicInfoProps, {}> {
                     </div>
                 </Col>
                 <Col span={8} className="padItem basic">
-                    <p>LogPath</p>
+                    <p>Log Directory</p>
                     <div className="nowrap">
                         <Tooltip placement="top" title={trialProfile.logDir}>
                             {trialProfile.logDir}

tools/nni_cmd/config_schema.py

@@ -33,6 +33,8 @@
 Optional('multiPhase'): bool,
 Optional('multiThread'): bool,
 Optional('nniManagerIp'): str,
+Optional('logDir'): os.path.isdir,
+Optional('logLevel'): Or('trace', 'debug', 'info', 'warning', 'error', 'fatal'),
 'useAnnotation': bool,
 Optional('advisor'): Or({
     'builtinAdvisorName': Or('Hyperband'),