feature: export experiment results

docs/en_US/Tutorial/Nnictl.md

@@ -24,6 +24,7 @@ nnictl support commands:
 * [nnictl package](#package)
 * [nnictl ss_gen](#ss_gen)
 * [nnictl --version](#version)
+* [nnictl export-results](#export-results)
 
 ### Manage an experiment
 
@@ -851,3 +852,25 @@ Debug mode will disable version check function in Trialkeeper.
     nnictl --version
     ```
 
+
+<a name="export-results"></a>
+
+### Export experiment results
+
+* __nnictl export-results__
+  * Description
+
+    Export experiment settings. All trials' intermediate results and final results are dumped into a json file. The file's default name is `exp_{experiment id}_{timestamp}.json`. Its content and format are aligned with `Download > Experiment Summary` in webUI.
+
+  * Usage
+
+    ```bash
+    nnictl export-results [options]
+    ```
+
+  * Options
+
+  |Name, shorthand|Required|Default|Description|
+  |---|---|---|---|
+  |--name, -n|False|'exp_{experiment id}_{timestamp}.json'|output json file name|
+  |--id|False||ID of the experiment you want to export results|

tools/nni_cmd/nnictl.py

@@ -11,7 +11,7 @@
 from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\
                           log_trial, experiment_clean, platform_clean, experiment_list, \
                           monitor_experiment, export_trials_data, trial_codegen, webui_url, \
-                          get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas
+                          get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, export_results
 from .package_management import package_install, package_uninstall, package_show, package_list
 from .constants import DEFAULT_REST_PORT
 from .tensorboard_utils import start_tensorboard, stop_tensorboard
@@ -230,6 +230,13 @@ def parse_args():
     'the unit is second')
     parser_top.set_defaults(func=monitor_experiment)
 
+    #parse export results
+    parser_export_results = subparsers.add_parser('export-results', help='dump all intermediate and final results into a json file')
+    parser_export_results.add_argument('--name', '-n', default='', type=str,
+                                       help='custom name of the dump file, default is \'exp_{Experiment ID}_{timestamp}.json\'')
+    parser_export_results.add_argument('id', nargs='?', help='the id of experiment')
+    parser_export_results.set_defaults(func=export_results)
+
     args = parser.parse_args()
     args.func(args)
 

tools/nni_cmd/nnictl_utils.py

@@ -16,7 +16,7 @@
 from nni.package_utils import get_nni_installation_path
 from nni_annotation import expand_annotations
 from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_response
-from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url
+from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_data_url, metric_data_url
 from .config_utils import Config, Experiments
 from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
      EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
@@ -736,3 +736,64 @@ def search_space_auto_gen(args):
         print_warning('Expected search space file \'{}\' generated, but not found.'.format(file_path))
     else:
         print_normal('Generate search space done: \'{}\'.'.format(file_path))
+
+def export_results(args):
+    '''dump all intermediate results and final results to json file
+    '''
+
+    def groupby_trial_id(intermediate_results):
+        sorted(intermediate_results, key=lambda x: x['timestamp'])
+        groupby = dict()
+        for content in intermediate_results:
+            groupby.setdefault(content['trialJobId'], []).append(content)
+        return groupby
+
+    update_experiment()
+
+    nni_config = Config(get_config_filename(args))
+    rest_port = nni_config.get_config('restServerPort')
+    rest_pid = nni_config.get_config('restServerPid')
+    if not detect_process(rest_pid):
+        print_error('Experiment is not running...')
+        return
+    running, _ = check_rest_server_quick(rest_port)
+    if running:
+        experiment_info = rest_get(experiment_url(rest_port), REST_TIME_OUT)
+        trial_info = rest_get(metric_data_url(rest_port), REST_TIME_OUT)
+        trial_jobs = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
+
+        if experiment_info and check_response(experiment_info) and \
+                trial_info and check_response(trial_info) and \
+                trial_jobs and check_response(trial_jobs):
+            results = {}
+            experiment_info = json.loads(experiment_info.text)
+            results['experimentParameters'] = experiment_info
+            experiment_id = experiment_info.get('id')
+            trial_info = json.loads(trial_info.text)
+            group_trial_info = groupby_trial_id(trial_info)
+            trial_jobs = json.loads(trial_jobs.text)
+            trial_message = []
+            for trial in trial_jobs:
+                trial_id = trial['id']
+                trial['intermediate'] = group_trial_info[trial_id]
+                trial_message.append(trial)
+            results['trialMessage'] = trial_message
+        else:
+            print_error('Ops. Error occured connecting to the server.')
+            exit(1)
+    else:
+        args = vars(args)
+        experiment_config = Experiments()
+        experiment_dict = experiment_config.get_all_experiments()
+        print_error('Ops. The experiment to export isn\'t running now. Please use \n    \
+        nnictl resume {}\nto restart.'.format(args.id if args.id in experiment_dict else ''))
+        exit(1)
+    args = vars(args)
+    filename = 'exp_' + str(experiment_id) + '_' + str(time.time()) + '.json' \
+        if args.get('name') == '' else args.get('name')
+    if os.path.exists(filename):
+        print_error('File {0} has existed.'.format(filename))
+        exit(1)
+    with open(filename, 'w') as f:
+        f.write(json.dumps(results))
+    print_normal('Export expriment {0} done: {1}'.format(experiment_id, filename))

tools/nni_cmd/url_utils.py

@@ -22,6 +22,12 @@
 
 TENSORBOARD_API = '/tensorboard'
 
+METRIC_DATA_API = '/metric-data'
+
+def metric_data_url(port):
+    '''get metric_data url'''
+    return '{0}:{1}{2}{3}'.format(BASE_URL, port, API_ROOT_URL, METRIC_DATA_API)
+
 
 def check_status_url(port):
     '''get check_status url'''