From af8b52b6f63d6e7b88accffe4b67e837af1bb632 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 24 Jun 2021 15:16:15 +0800 Subject: [PATCH 1/9] init --- docs/en_US/Tutorial/Nnictl.rst | 54 ++++++++++++++++++++++++++++++++++ nni/tools/nnictl/launcher.py | 27 +++++++++++++++-- nni/tools/nnictl/nnictl.py | 8 ++++- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.rst b/docs/en_US/Tutorial/Nnictl.rst index 4b3d40f7b2..96d71bbad5 100644 --- a/docs/en_US/Tutorial/Nnictl.rst +++ b/docs/en_US/Tutorial/Nnictl.rst @@ -1061,6 +1061,60 @@ Manage experiment information nnictl experiment load --path [path] --codeDir [codeDir] + +* + **nnictl experiment view** + + + * + Description + + View an nni experiment from external folder. + + * + Usage + + .. code-block:: bash + + nnictl experiment view [OPTIONS] + + * + Options + +.. list-table:: + :header-rows: 1 + :widths: auto + + * - Name, shorthand + - Required + - Default + - Description + * - --experiment_dir, -d + - True + - + - The folder path of nni experiment + * - --port, -p + - False + - 8080 + - The port used to start an experiment. + * - --url_prefix, -u + - False + - + - The prefix url of an experiment + + +* + Examples + + .. + + view an external expeirment + + + .. code-block:: bash + + nnictl experiment view --experiment_dir [path] + :raw-html:`` Manage platform information diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index 9144bc0267..eb6cd71a3a 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -419,7 +419,6 @@ def launch_experiment(args, experiment_config, mode, experiment_id, config_versi experiment_config['searchSpace'] = search_space else: experiment_config['searchSpace'] = '' - # check rest server running, _ = check_rest_server(args.port) if running: @@ -436,7 +435,6 @@ def launch_experiment(args, experiment_config, mode, experiment_id, config_versi # set platform configuration set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ experiment_id, rest_process) - # start a new experiment print_normal('Starting experiment...') # set debug configuration @@ -574,3 +572,28 @@ def view_experiment(args): def resume_experiment(args): '''resume an experiment''' manage_stopped_experiment(args, 'resume') + +def view_external_experiment(args): + '''view a experiment from external path''' + # validate arguments + if not os.path.exists(args.experiment_dir): + print_error('Folder %s does not exist!' % args.experiment_dir) + exit(1) + if not os.path.isdir(args.experiment_dir): + print_error('Path %s is not folder directory!' % args.experiment_dir) + exit(1) + experiment_id = os.path.basename(args.experiment_dir) + log_dir = os.path.dirname(args.experiment_dir) + + experiment_config = Config(experiment_id, log_dir).get_config() + assert 'trainingService' in experiment_config or 'trainingServicePlatform' in experiment_config + try: + if 'trainingServicePlatform' in experiment_config: + experiment_config['logDir'] = log_dir + launch_experiment(args, experiment_config, 'view', experiment_id, 1) + else: + experiment_config['experimentWorkingDirectory'] = log_dir + launch_experiment(args, experiment_config, 'view', experiment_id, 2) + except Exception as exception: + print_error(exception) + exit(1) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index a5f02c8c9e..713aca8ff7 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -7,7 +7,7 @@ import pkg_resources from colorama import init from .common_utils import print_error -from .launcher import create_experiment, resume_experiment, view_experiment +from .launcher import create_experiment, resume_experiment, view_experiment, view_external_experiment from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ log_trial, experiment_clean, platform_clean, experiment_list, \ @@ -167,6 +167,12 @@ def parse_args(): parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ loaded experiment, this path contains file name. Default in $codeDir/search_space.json') parser_load_experiment.set_defaults(func=load_experiment) + #view an NNI experiment + parser_view_experiment = parser_experiment_subparsers.add_parser('view', help='load an experiment from a folder') + parser_view_experiment.add_argument('--experiment_dir', '-e', required=True, help='the path of nni experiment folder') + parser_view_experiment.add_argument('--url_prefix', '-u', dest='url_prefix', help=' set prefix url') + parser_view_experiment.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of experiment') + parser_view_experiment.set_defaults(func=view_external_experiment) #parse platform command parser_platform = subparsers.add_parser('platform', help='get platform information') From b7470a609e8e999ef3209241fef1f6e8f9634897 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 24 Jun 2021 15:18:33 +0800 Subject: [PATCH 2/9] remove blank line --- nni/tools/nnictl/launcher.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index eb6cd71a3a..58efe2ca4c 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -419,6 +419,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id, config_versi experiment_config['searchSpace'] = search_space else: experiment_config['searchSpace'] = '' + # check rest server running, _ = check_rest_server(args.port) if running: @@ -435,6 +436,7 @@ def launch_experiment(args, experiment_config, mode, experiment_id, config_versi # set platform configuration set_platform_config(experiment_config['trainingServicePlatform'], experiment_config, args.port,\ experiment_id, rest_process) + # start a new experiment print_normal('Starting experiment...') # set debug configuration From fad314994338b17981d4791c5d3075a7ade73b52 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 24 Jun 2021 15:21:37 +0800 Subject: [PATCH 3/9] fix annotation --- nni/tools/nnictl/nnictl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index 713aca8ff7..0fffa659d1 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -168,8 +168,8 @@ def parse_args(): loaded experiment, this path contains file name. Default in $codeDir/search_space.json') parser_load_experiment.set_defaults(func=load_experiment) #view an NNI experiment - parser_view_experiment = parser_experiment_subparsers.add_parser('view', help='load an experiment from a folder') - parser_view_experiment.add_argument('--experiment_dir', '-e', required=True, help='the path of nni experiment folder') + parser_view_experiment = parser_experiment_subparsers.add_parser('view', help='view an experiment from external folder') + parser_view_experiment.add_argument('--experiment_dir', '-e', required=True, help='the full path of nni experiment folder') parser_view_experiment.add_argument('--url_prefix', '-u', dest='url_prefix', help=' set prefix url') parser_view_experiment.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of experiment') parser_view_experiment.set_defaults(func=view_external_experiment) From 077938847ea3ea939db8ae3785a8c37de9818346 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 24 Jun 2021 15:24:29 +0800 Subject: [PATCH 4/9] fix typo --- docs/en_US/Tutorial/Nnictl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en_US/Tutorial/Nnictl.rst b/docs/en_US/Tutorial/Nnictl.rst index 96d71bbad5..7fb8c84f55 100644 --- a/docs/en_US/Tutorial/Nnictl.rst +++ b/docs/en_US/Tutorial/Nnictl.rst @@ -1089,7 +1089,7 @@ Manage experiment information - Required - Default - Description - * - --experiment_dir, -d + * - --experiment_dir, -e - True - - The folder path of nni experiment From c4bc4e2e94b45b3a484d8cb23b7eb91791b02f83 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Thu, 24 Jun 2021 15:31:22 +0800 Subject: [PATCH 5/9] update doc --- docs/en_US/Tutorial/Nnictl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en_US/Tutorial/Nnictl.rst b/docs/en_US/Tutorial/Nnictl.rst index 7fb8c84f55..e4b0ca5358 100644 --- a/docs/en_US/Tutorial/Nnictl.rst +++ b/docs/en_US/Tutorial/Nnictl.rst @@ -1092,7 +1092,7 @@ Manage experiment information * - --experiment_dir, -e - True - - - The folder path of nni experiment + - The folder path of nni experiment. Note: please make sure the folder name is consistent with the experiment id. * - --port, -p - False - 8080 From 83728202d3bcc0c59419dbff6b4ae52411f9471d Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 9 Jul 2021 17:45:43 +0800 Subject: [PATCH 6/9] fix comments --- docs/en_US/Tutorial/Nnictl.rst | 61 +++++--------------------------- nni/tools/nnictl/launcher.py | 35 +++++++++++++----- nni/tools/nnictl/nnictl.py | 10 ++---- nni/tools/nnictl/nnictl_utils.py | 2 +- 4 files changed, 38 insertions(+), 70 deletions(-) diff --git a/docs/en_US/Tutorial/Nnictl.rst b/docs/en_US/Tutorial/Nnictl.rst index e4b0ca5358..dbace6a691 100644 --- a/docs/en_US/Tutorial/Nnictl.rst +++ b/docs/en_US/Tutorial/Nnictl.rst @@ -166,6 +166,10 @@ nnictl resume - False - - set foreground mode, print log content to terminal + * - --experiment_dir, -e + - False + - + - Resume experiment from external folder, specify the full path of experiment folder @@ -218,6 +222,10 @@ nnictl view - False - - Rest port of the experiment you want to view + * - --experiment_dir, -e + - False + - + - View experiment from external folder, specify the full path of experiment folder @@ -1062,59 +1070,6 @@ Manage experiment information nnictl experiment load --path [path] --codeDir [codeDir] -* - **nnictl experiment view** - - - * - Description - - View an nni experiment from external folder. - - * - Usage - - .. code-block:: bash - - nnictl experiment view [OPTIONS] - - * - Options - -.. list-table:: - :header-rows: 1 - :widths: auto - - * - Name, shorthand - - Required - - Default - - Description - * - --experiment_dir, -e - - True - - - - The folder path of nni experiment. Note: please make sure the folder name is consistent with the experiment id. - * - --port, -p - - False - - 8080 - - The port used to start an experiment. - * - --url_prefix, -u - - False - - - - The prefix url of an experiment - - -* - Examples - - .. - - view an external expeirment - - - .. code-block:: bash - - nnictl experiment view --experiment_dir [path] - :raw-html:`` Manage platform information diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index 58efe2ca4c..09625cfdd5 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -538,7 +538,9 @@ def manage_stopped_experiment(args, mode): #find the latest stopped experiment if not args.id: print_error('Please set experiment id! \nYou could use \'nnictl {0} id\' to {0} a stopped experiment!\n' \ - 'You could use \'nnictl experiment list --all\' to show all experiments!'.format(mode)) + 'You could use \'nnictl experiment list --all\' to show all experiments!\n' \ + 'If your experiment is not started in current machine, you could specify experiment folder using ' \ + '--experiment_dir argument'.format(mode)) exit(1) else: if experiments_dict.get(args.id) is None: @@ -569,13 +571,20 @@ def manage_stopped_experiment(args, mode): def view_experiment(args): '''view a stopped experiment''' - manage_stopped_experiment(args, 'view') + if args.experiment_dir: + manage_external_experiment(args, 'view') + else: + manage_stopped_experiment(args, 'view') def resume_experiment(args): '''resume an experiment''' - manage_stopped_experiment(args, 'resume') + '''view a stopped experiment''' + if args.experiment_dir: + manage_external_experiment(args, 'resume') + else: + manage_stopped_experiment(args, 'resume') -def view_external_experiment(args): +def manage_external_experiment(args, mode): '''view a experiment from external path''' # validate arguments if not os.path.exists(args.experiment_dir): @@ -584,18 +593,26 @@ def view_external_experiment(args): if not os.path.isdir(args.experiment_dir): print_error('Path %s is not folder directory!' % args.experiment_dir) exit(1) - experiment_id = os.path.basename(args.experiment_dir) - log_dir = os.path.dirname(args.experiment_dir) - + if args.id: + experiment_id = args.id + log_dir = args.experiment_dir + else: + print_normal('NNI can not detect experiment id in argument, will use last folder name as experiment id in experiment_dir argument.') + experiment_id = os.path.basename(args.experiment_dir) + log_dir = os.path.dirname(args.experiment_dir) + if not experiment_id: + print_error("Please set experiment id argument, or add id as the last folder name in experiment_dir argument.") + exit(1) + args.url_prefix = None experiment_config = Config(experiment_id, log_dir).get_config() assert 'trainingService' in experiment_config or 'trainingServicePlatform' in experiment_config try: if 'trainingServicePlatform' in experiment_config: experiment_config['logDir'] = log_dir - launch_experiment(args, experiment_config, 'view', experiment_id, 1) + launch_experiment(args, experiment_config, mode, experiment_id, 1) else: experiment_config['experimentWorkingDirectory'] = log_dir - launch_experiment(args, experiment_config, 'view', experiment_id, 2) + launch_experiment(args, experiment_config, mode, experiment_id, 2) except Exception as exception: print_error(exception) exit(1) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index 0fffa659d1..8b8c4bfaf0 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -7,7 +7,7 @@ import pkg_resources from colorama import init from .common_utils import print_error -from .launcher import create_experiment, resume_experiment, view_experiment, view_external_experiment +from .launcher import create_experiment, resume_experiment, view_experiment from .updater import update_searchspace, update_concurrency, update_duration, update_trialnum, import_data from .nnictl_utils import stop_experiment, trial_ls, trial_kill, list_experiment, experiment_status,\ log_trial, experiment_clean, platform_clean, experiment_list, \ @@ -66,12 +66,14 @@ def parse_args(): parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') + parser_resume.add_argument('--experiment_dir', '-e', help='resume experiment from external folder, specify the full path of experiment folder') parser_resume.set_defaults(func=resume_experiment) # parse view command parser_view = subparsers.add_parser('view', help='view a stopped experiment') parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') + parser_view.add_argument('--experiment_dir', '-e', help='view experiment from external folder, specify the full path of experiment folder') parser_view.set_defaults(func=view_experiment) # parse update command @@ -167,12 +169,6 @@ def parse_args(): parser_load_experiment.add_argument('--searchSpacePath', '-s', required=False, help='the path of search space file for \ loaded experiment, this path contains file name. Default in $codeDir/search_space.json') parser_load_experiment.set_defaults(func=load_experiment) - #view an NNI experiment - parser_view_experiment = parser_experiment_subparsers.add_parser('view', help='view an experiment from external folder') - parser_view_experiment.add_argument('--experiment_dir', '-e', required=True, help='the full path of nni experiment folder') - parser_view_experiment.add_argument('--url_prefix', '-u', dest='url_prefix', help=' set prefix url') - parser_view_experiment.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', help='the port of experiment') - parser_view_experiment.set_defaults(func=view_external_experiment) #parse platform command parser_platform = subparsers.add_parser('platform', help='get platform information') diff --git a/nni/tools/nnictl/nnictl_utils.py b/nni/tools/nnictl/nnictl_utils.py index 7ef0fe7dd7..142e79d452 100644 --- a/nni/tools/nnictl/nnictl_utils.py +++ b/nni/tools/nnictl/nnictl_utils.py @@ -518,7 +518,7 @@ def experiment_clean(args): for experiment_id in experiment_id_list: experiment_id = get_config_filename(args) experiment_config = Config(experiment_id, Experiments().get_all_experiments()[experiment_id]['logDir']).get_config() - platform = experiment_config.get('trainingServicePlatform') + platform = experiment_config.get('trainingServicePlatform') or experiment_config.get('trainingService', {}).get('platform') if platform == 'remote': machine_list = experiment_config.get('machineList') remote_clean(machine_list, experiment_id) From fedf6e46c2e1f1e140913b65dce7753c7793cda9 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 9 Jul 2021 17:47:10 +0800 Subject: [PATCH 7/9] remvoe blank line --- docs/en_US/Tutorial/Nnictl.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en_US/Tutorial/Nnictl.rst b/docs/en_US/Tutorial/Nnictl.rst index dbace6a691..610c84d86a 100644 --- a/docs/en_US/Tutorial/Nnictl.rst +++ b/docs/en_US/Tutorial/Nnictl.rst @@ -1069,7 +1069,6 @@ Manage experiment information nnictl experiment load --path [path] --codeDir [codeDir] - :raw-html:`` Manage platform information From 15de16b7d1984a78cd5541388d1b46e9c90cb81f Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Fri, 9 Jul 2021 18:16:35 +0800 Subject: [PATCH 8/9] fix pylint --- nni/tools/nnictl/nnictl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index 8b8c4bfaf0..962698e7c1 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -66,14 +66,16 @@ def parse_args(): parser_resume.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') parser_resume.add_argument('--debug', '-d', action='store_true', help=' set debug mode') parser_resume.add_argument('--foreground', '-f', action='store_true', help=' set foreground mode, print log content to terminal') - parser_resume.add_argument('--experiment_dir', '-e', help='resume experiment from external folder, specify the full path of experiment folder') + parser_resume.add_argument('--experiment_dir', '-e', help='resume experiment from external folder, specify the full path of ' \ + 'experiment folder') parser_resume.set_defaults(func=resume_experiment) # parse view command parser_view = subparsers.add_parser('view', help='view a stopped experiment') parser_view.add_argument('id', nargs='?', help='The id of the experiment you want to view') parser_view.add_argument('--port', '-p', default=DEFAULT_REST_PORT, dest='port', type=int, help='the port of restful server') - parser_view.add_argument('--experiment_dir', '-e', help='view experiment from external folder, specify the full path of experiment folder') + parser_view.add_argument('--experiment_dir', '-e', help='view experiment from external folder, specify the full path of ' \ + 'experiment folder') parser_view.set_defaults(func=view_experiment) # parse update command From 5b123fc71e04cce48d579ab7e24d0aff0df4e3f9 Mon Sep 17 00:00:00 2001 From: SparkSnail Date: Mon, 12 Jul 2021 10:52:17 +0800 Subject: [PATCH 9/9] fix comments --- nni/tools/nnictl/launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index 09625cfdd5..0c24178889 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -598,7 +598,7 @@ def manage_external_experiment(args, mode): log_dir = args.experiment_dir else: print_normal('NNI can not detect experiment id in argument, will use last folder name as experiment id in experiment_dir argument.') - experiment_id = os.path.basename(args.experiment_dir) + experiment_id = Path(args.experiment_dir).name log_dir = os.path.dirname(args.experiment_dir) if not experiment_id: print_error("Please set experiment id argument, or add id as the last folder name in experiment_dir argument.")