Fix nnictl bugs and add new feature

tools/nnicmd/launcher.py

@@ -31,7 +31,7 @@
 from .rest_utils import rest_put, rest_post, check_rest_server, check_rest_server_quick
 from .url_utils import cluster_metadata_url, experiment_url
 from .config_utils import Config
-from .common_utils import get_yml_content, get_json_content, print_error, print_normal
+from .common_utils import get_yml_content, get_json_content, print_error, print_normal, detect_process
 from .constants import EXPERIMENT_SUCCESS_INFO, STDOUT_FULL_PATH, STDERR_FULL_PATH, LOG_DIR, REST_PORT, ERROR_INFO, NORMAL_INFO
 from .webui_utils import start_web_ui, check_web_ui
 
@@ -40,7 +40,7 @@ def start_rest_server(port, platform, mode, experiment_id=None):
     print_normal('Checking experiment...')
     nni_config = Config()
     rest_port = nni_config.get_config('restServerPort')
-    if rest_port and check_rest_server_quick(rest_port):
+    if rest_port and check_rest_server_quick(rest_port)[0]:
         print_error('There is an experiment running, please stop it first...')
         print_normal('You can use \'nnictl stop\' command to stop an experiment!')
         exit(0)
@@ -122,6 +122,12 @@ def set_experiment(experiment_config, mode, port):
 def launch_experiment(args, experiment_config, mode, webuiport, experiment_id=None):
     '''follow steps to start rest server and start experiment'''
     nni_config = Config()
+    #Check if there is an experiment running
+    origin_rest_pid = nni_config.get_config('restServerPid')
+    if origin_rest_pid and detect_process(origin_rest_pid):
+        print_error('There is an experiment running, please stop it first...')
+        print_normal('You can use \'nnictl stop\' command to stop an experiment!')
+        exit(0)
     # start rest server
     rest_process = start_rest_server(REST_PORT, experiment_config['trainingServicePlatform'], mode, experiment_id)
     nni_config.set_config('restServerPid', rest_process.pid)
@@ -144,7 +150,7 @@ def launch_experiment(args, experiment_config, mode, webuiport, experiment_id=No
 
     # check rest server
     print_normal('Checking restful server...')
-    if check_rest_server(REST_PORT):
+    if check_rest_server(REST_PORT)[0]:
         print_normal('Restful server start success!')
     else:
         print_error('Restful server start failed!')

tools/nnicmd/nnictl.py

@@ -29,7 +29,7 @@ def nni_help_info(*args):
 
 def parse_args():
     '''Definite the arguments users need to follow and input'''
-    parser = argparse.ArgumentParser(prog='nni ctl', description='use nni control')
+    parser = argparse.ArgumentParser(prog='nnictl', description='use nnictl command to control nni experiments')
     parser.set_defaults(func=nni_help_info)
 
     # create subparsers for args with sub values
@@ -95,6 +95,8 @@ def parse_args():
     parser_experiment_subparsers = parser_experiment.add_subparsers()
     parser_experiment_show = parser_experiment_subparsers.add_parser('show', help='show the information of experiment')
     parser_experiment_show.set_defaults(func=list_experiment)
+    parser_experiment_status = parser_experiment_subparsers.add_parser('status', help='show the status of experiment')
+    parser_experiment_status.set_defaults(func=experiment_status)
 
     #parse config command
     parser_config = subparsers.add_parser('config', help='get config information')

tools/nnicmd/nnictl_utils.py

@@ -47,7 +47,7 @@ def check_rest(args):
     '''check if restful server is running'''
     nni_config = Config()
     rest_port = nni_config.get_config('restServerPort')
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         print_normal('Restful server is running...')
     else:
         print_normal('Restful server is not running...')
@@ -62,7 +62,7 @@ def stop_experiment(args):
         print_normal('Experiment is not running...')
         stop_web_ui()
         return
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         response = rest_delete(experiment_url(rest_port), 20)
         if not response or response.status_code != 200:
             print_error('Stop experiment failed!')
@@ -82,7 +82,7 @@ def trial_ls(args):
     if not detect_process(rest_pid):
         print_error('Experiment is not running...')
         return
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         response = rest_get(trial_jobs_url(rest_port), 20)
         if response and response.status_code == 200:
             content = json.loads(response.text)
@@ -102,7 +102,7 @@ def trial_kill(args):
     if not detect_process(rest_pid):
         print_error('Experiment is not running...')
         return
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         response = rest_delete(trial_job_id_url(rest_port, args.trialid), 20)
         if response and response.status_code == 200:
             print(response.text)
@@ -119,7 +119,7 @@ def list_experiment(args):
     if not detect_process(rest_pid):
         print_error('Experiment is not running...')
         return
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         response = rest_get(experiment_url(rest_port), 20)
         if response and response.status_code == 200:
             content = convert_time_stamp_to_date(json.loads(response.text))
@@ -129,6 +129,16 @@ def list_experiment(args):
     else:
         print_error('Restful server is not running...')
 
+def experiment_status(args):
+    '''Show the status of experiment'''
+    nni_config = Config()
+    rest_port = nni_config.get_config('restServerPort')
+    result, response = check_rest_server_quick(rest_port)
+    if not result:
+        print_normal('Restful server is not running...')
+    else:
+        print(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
+
 def get_log_content(file_name, cmds):
     '''use cmds to read config content'''
     if os.path.exists(file_name):

tools/nnicmd/rest_utils.py

@@ -64,16 +64,16 @@ def check_rest_server(rest_port):
         response = rest_get(check_status_url(rest_port), 20)
         if response:
             if response.status_code == 200:
-                return True
+                return True, response
             else:
-                return False
+                return False, response
         else:
             time.sleep(3)
-    return  False
+    return  False, response
 
 def check_rest_server_quick(rest_port):
     '''Check if restful server is ready, only check once'''
     response = rest_get(check_status_url(rest_port), 5)
     if response and response.status_code == 200:
-        return True
-    return False
+        return True, response
+    return False, None

tools/nnicmd/updater.py

@@ -56,7 +56,7 @@ def update_experiment_profile(key, value):
     '''call restful server to update experiment profile'''
     nni_config = Config()
     rest_port = nni_config.get_config('restServerPort')
-    if check_rest_server_quick(rest_port):
+    if check_rest_server_quick(rest_port)[0]:
         response = rest_get(experiment_url(rest_port), 20)
         if response and response.status_code == 200:
             experiment_profile = json.loads(response.text)