Refactor nnictl to support list multiple experiment

deployment/README.md

@@ -2,6 +2,8 @@ Dockerfile
 ===
 ## 1.Description  
 This is the Dockerfile of nni project, including the most kinds of deeplearning frameworks and nni source code.  You can run your nni experiment in this docker container directly.
+Dockerfile.build.base could build the base Docker image, users can get a docker image with Ubuntu and NNI environment after building this file. 
+Dockerfile could build the customized docker image, users could build their customized docker image using this file.
 ## 2.Including Libraries  
 
 ```

tools/nnicmd/common_utils.py

@@ -21,6 +21,7 @@
 import json
 import yaml
 import psutil
+import socket
 from .constants import ERROR_INFO, NORMAL_INFO, WARNING_INFO, COLOR_RED_FORMAT, COLOR_YELLOW_FORMAT
 
 def get_yml_content(file_path):
@@ -60,3 +61,13 @@ def detect_process(pid):
         return process.is_running()
     except:
         return False
+
+def detect_port(port):
+    '''Detect if the port is used'''
+    socket_test = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
+    try:
+        socket_test.connect(('127.0.0.1', int(port)))
+        socket_test.shutdown(2)
+        return True
+    except:
+        return False

tools/nnicmd/config_utils.py

@@ -46,12 +46,6 @@ def get_config(self, key):
         '''get a value according to key'''
         return self.config.get(key)
 
-    def copy_metadata_to_new_path(self, path):
-        '''copy metadata to a new path'''
-        if not os.path.exists(path):
-            os.mkdir(path)
-        shutil.copy(self.config_file, path)
-
     def write_file(self):
         '''save config to local file'''
         if self.config:
@@ -71,3 +65,44 @@ def read_file(self):
             except ValueError:
                 return {}
         return {}
+
+class Experiments:
+    '''Maintain experiment list'''
+    def __init__(self):
+        os.makedirs(HOME_DIR, exist_ok=True)
+        self.experiment_file = os.path.join(HOME_DIR, '.experiment')
+        self.experiments = self.read_file()
+
+    def add_experiment(self, id, port, time):
+        '''set {key:value} paris to self.experiment'''
+        self.experiments[id] = [port, time]
+        self.write_file()
+
+    def remove_experiment(self, id):
+        '''remove an experiment by id'''
+        if id in self.experiments:
+            self.experiments.pop(id)
+        self.write_file()
+
+    def get_all_experiments(self):
+        '''return all of experiments'''
+        return self.experiments
+
+    def write_file(self):
+        '''save config to local file'''
+        try:
+            with open(self.experiment_file, 'w') as file:
+                json.dump(self.experiments, file)
+        except IOError as error:
+            print('Error:', error)
+            return
+
+    def read_file(self):
+        '''load config from local file'''
+        if os.path.exists(self.experiment_file):
+            try:
+                with open(self.experiment_file, 'r') as file:
+                    return json.load(file)
+            except ValueError:
+                return {}
+        return {} 

tools/nnicmd/constants.py

@@ -28,12 +28,12 @@
 
 WARNING_INFO = 'WARNING: %s'
 
-DEFAULT_REST_PORT = 51188
+DEFAULT_REST_PORT = 8080
 
 EXPERIMENT_SUCCESS_INFO = '\033[1;32;32mSuccessfully started experiment!\n\033[0m' \
                           '-----------------------------------------------------------------------\n' \
                           'The experiment id is %s\n'\
-                          'The restful server post is %s\n' \
+                          'The Web UI urls are: %s\n' \
                           '-----------------------------------------------------------------------\n\n' \
                           'You can use these commands to get more information about the experiment\n' \
                           '-----------------------------------------------------------------------\n' \
@@ -42,11 +42,23 @@
                           '2. nnictl trial ls               list all of trial jobs\n' \
                           '3. nnictl log stderr             show stderr log content\n' \
                           '4. nnictl log stdout             show stdout log content\n' \
-                          '5. nnictl stop                   stop a experiment\n' \
+                          '5. nnictl stop                   stop an experiment\n' \
                           '6. nnictl trial kill             kill a trial job by id\n' \
                           '7. nnictl --help                 get help information about nnictl\n' \
                           '-----------------------------------------------------------------------\n' \
 
+LOG_HEADER = '-----------------------------------------------------------------------\n' \
+             '                Experiment start time %s\n' \
+             '-----------------------------------------------------------------------\n'
+
+EXPERIMENT_START_FAILED_INFO = 'There is an experiment running in the port %d, please stop it first or set another port!\n' \
+                               'You could use \'nnictl stop --port [PORT]\' command to stop an experiment!\nOr you could use \'nnictl create --config [CONFIG_PATH] --port [PORT]\' to set port!\n'
+
+EXPERIMENT_ID_INFO = '-----------------------------------------------------------------------\n' \
+                     '                Experiment information\n' \
+                     '%s\n' \
+                     '-----------------------------------------------------------------------\n'
+
 PACKAGE_REQUIREMENTS = {
     'SMAC': 'smac_tuner'
 }

tools/nnicmd/launcher.py

@@ -29,9 +29,11 @@
 from .launcher_utils import validate_all_content
 from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick, check_response
 from .url_utils import cluster_metadata_url, experiment_url
-from .config_utils import Config
-from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process
+from .config_utils import Config, Experiments
+from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
 from .constants import *
+from .webui_utils import *
+import time
 
 def start_rest_server(port, platform, mode, experiment_id=None):
     '''Run nni manager process'''
@@ -40,9 +42,12 @@ def start_rest_server(port, platform, mode, experiment_id=None):
     rest_port = nni_config.get_config('restServerPort')
     running, _ = check_rest_server_quick(rest_port)
     if rest_port and running:
-        print_error('There is an experiment running, please stop it first...')
-        print_normal('You can use \'nnictl stop\' command to stop an experiment!')
-        exit(0)
+        print_error(EXPERIMENT_START_FAILED_INFO % port)
+        exit(1)
+
+    if detect_port(port):
+        print_error('Port %s is used by another process, please reset the port!' % port)
+        exit(1)
 
     print_normal('Starting restful server...')
     manager = os.environ.get('NNI_MANAGER', 'nnimanager')
@@ -53,8 +58,12 @@ def start_rest_server(port, platform, mode, experiment_id=None):
     stderr_full_path = os.path.join(HOME_DIR, str(port), 'stderr')
     stdout_file = open(stdout_full_path, 'a+')
     stderr_file = open(stderr_full_path, 'a+')
+    time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
+    log_header = LOG_HEADER % str(time_now)
+    stdout_file.write(log_header)
+    stderr_file.write(log_header)
     process = Popen(cmds, stdout=stdout_file, stderr=stderr_file)
-    return process
+    return process, str(time_now)
 
 def set_trial_config(experiment_config, port):
     '''set trial configuration'''
@@ -115,7 +124,8 @@ def set_pai_config(experiment_config, port):
     if not response or not response.status_code == 200:
         if response is not None:
             err_message = response.text
-            with open(STDERR_FULL_PATH, 'a+') as fout:
+            stderr_full_path = os.path.join(HOME_DIR, str(port), 'stderr')
+            with open(stderr_full_path, 'a+') as fout:
                 fout.write(json.dumps(json.loads(err_message), indent=4, sort_keys=True, separators=(',', ':')))
         return False, err_message
 
@@ -189,14 +199,8 @@ def set_experiment(experiment_config, mode, port):
 def launch_experiment(args, experiment_config, mode, experiment_id=None):
     '''follow steps to start rest server and start experiment'''
     nni_config = Config(args.port)
-    #Check if there is an experiment running
-    origin_rest_pid = nni_config.get_config('restServerPid')
-    if origin_rest_pid and detect_process(origin_rest_pid):
-        print_error('There is an experiment running, please stop it first...')
-        print_normal('You can use \'nnictl stop\' command to stop an experiment!')
-        exit(1)
     # start rest server
-    rest_process = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, experiment_id)
+    rest_process, start_time = start_rest_server(args.port, experiment_config['trainingServicePlatform'], mode, experiment_id)
     nni_config.set_config('restServerPid', rest_process.pid)
     # Deal with annotation
     if experiment_config.get('useAnnotation'):
@@ -233,7 +237,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None):
         print_normal('Setting remote config...')
         config_result, err_msg = set_remote_config(experiment_config, args.port)
         if config_result:
-            print_normal('Success!')
+            print_normal('Successfully set remote config!')
         else:
             print_error('Failed! Error is: {}'.format(err_msg))
             try:
@@ -288,7 +292,13 @@ def launch_experiment(args, experiment_config, mode, experiment_id=None):
         except Exception:
             raise Exception(ERROR_INFO % 'Restful server stopped!')
         exit(1)
-    print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, args.port))
+    web_ui_url_list = get_web_ui_urls(args.port)
+
+    #save experiment information
+    experiment_config = Experiments()
+    experiment_config.add_experiment(experiment_id, args.port, start_time)
+
+    print_normal(EXPERIMENT_SUCCESS_INFO % (experiment_id, '   '.join(web_ui_url_list)))
 
 def resume_experiment(args):
     '''resume an experiment'''

tools/nnicmd/nnictl.py

@@ -55,32 +55,32 @@ def parse_args():
     #add subparsers for parser_updater
     parser_updater_subparsers = parser_updater.add_subparsers()
     parser_updater_searchspace = parser_updater_subparsers.add_parser('searchspace', help='update searchspace')
-    parser_updater_searchspace.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_updater_searchspace.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_updater_searchspace.add_argument('--filename', '-f', required=True)
     parser_updater_searchspace.set_defaults(func=update_searchspace)
     parser_updater_concurrency = parser_updater_subparsers.add_parser('concurrency', help='update concurrency')
-    parser_updater_concurrency.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_updater_concurrency.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_updater_concurrency.add_argument('--value', '-v', required=True)
     parser_updater_concurrency.set_defaults(func=update_concurrency)
     parser_updater_duration = parser_updater_subparsers.add_parser('duration', help='update duration')
-    parser_updater_duration.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_updater_duration.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_updater_duration.add_argument('--value', '-v', required=True)
     parser_updater_duration.set_defaults(func=update_duration)
 
     #parse stop command
     parser_stop = subparsers.add_parser('stop', help='stop the experiment')
-    parser_stop.add_argument('--port', '-p', required=True, dest='port', help='the port of restful server')
+    parser_stop.add_argument('id', nargs='?', help='the id of experiment')
     parser_stop.set_defaults(func=stop_experiment)
 
     #parse trial command
     parser_trial = subparsers.add_parser('trial', help='get trial information')
     #add subparsers for parser_trial
     parser_trial_subparsers = parser_trial.add_subparsers()
     parser_trial_ls = parser_trial_subparsers.add_parser('ls', help='list trial jobs')
-    parser_trial_ls.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_trial_ls.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_trial_ls.set_defaults(func=trial_ls)
     parser_trial_kill = parser_trial_subparsers.add_parser('kill', help='kill trial jobs')
-    parser_trial_kill.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_trial_kill.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_trial_kill.add_argument('--trialid', '-t', required=True, dest='trialid', help='the id of trial to be killed')
     parser_trial_kill.set_defaults(func=trial_kill)
 
@@ -89,41 +89,51 @@ def parse_args():
     #add subparsers for parser_experiment
     parser_experiment_subparsers = parser_experiment.add_subparsers()
     parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment')
-    parser_experiment_show.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_experiment_show.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_experiment_show.set_defaults(func=list_experiment)
     parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment')
-    parser_experiment_status.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_experiment_status.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_experiment_status.set_defaults(func=experiment_status)
+    parser_experiment_list = parser_experiment_subparsers.add_parser('list', help='list all of running experiment ids')
+    parser_experiment_list.set_defaults(func=experiment_id)
+
+    #TODO:finish webui function
+    #parse board command
+    parser_webui = subparsers.add_parser('webui', help='get web ui information')
+    #add subparsers for parser_board
+    parser_webui_subparsers = parser_webui.add_subparsers()
+    parser_webui_url = parser_webui_subparsers.add_parser('url', help='show the url of web ui')
+    parser_webui_url.add_argument('--id', '-i', dest='id', help='the id of experiment')
+    parser_webui_url.set_defaults(func=webui_url)
 
     #parse config command
     parser_config = subparsers.add_parser('config', help='get config information')
     parser_config_subparsers = parser_config.add_subparsers()
     parser_config_show = parser_config_subparsers.add_parser('show', help='show the information of config')
-    parser_config_show.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_config_show.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_config_show.set_defaults(func=get_config)
 
     #parse log command
     parser_log = subparsers.add_parser('log', help='get log information')
     # add subparsers for parser_log
     parser_log_subparsers = parser_log.add_subparsers()
     parser_log_stdout = parser_log_subparsers.add_parser('stdout', help='get stdout information')
-    parser_log_stdout.add_argument('--port', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_log_stdout.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_log_stdout.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stdout')
     parser_log_stdout.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stdout')
     parser_log_stdout.add_argument('--path', action='store_true', default=False, help='get the path of stdout file')
     parser_log_stdout.set_defaults(func=log_stdout)
     parser_log_stderr = parser_log_subparsers.add_parser('stderr', help='get stderr information')
-    parser_log_stderr.add_argument('--port', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
+    parser_log_stderr.add_argument('--id', '-i', dest='id', help='the id of experiment')
     parser_log_stderr.add_argument('--tail', '-T', dest='tail', type=int, help='get tail -100 content of stderr')
     parser_log_stderr.add_argument('--head', '-H', dest='head', type=int, help='get head -100 content of stderr')
     parser_log_stderr.add_argument('--path', action='store_true', default=False, help='get the path of stderr file')
     parser_log_stderr.set_defaults(func=log_stderr)
     parser_log_trial = parser_log_subparsers.add_parser('trial', help='get trial log path')
-    parser_log_trial.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of restful server')
-    parser_log_trial.add_argument('--id', '-I', dest='id', help='find trial log path by id')
+    parser_log_trial.add_argument('--id', '-i', dest='id', help='the id of experiment')
+    parser_log_trial.add_argument('--trialid', '-T', dest='trialid', help='find trial log path by id')
     parser_log_trial.set_defaults(func=log_trial)
 
-
     #parse package command
     parser_package = subparsers.add_parser('package', help='control nni tuner and assessor packages')
     # add subparsers for parser_package
@@ -134,7 +144,6 @@ def parse_args():
     parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages')
     parser_package_show.set_defaults(func=package_show)
 
-
     args = parser.parse_args()
     args.func(args)