From c9ac28d4469862a5ebe446cd65540e0bc98a86c6 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Sat, 28 Nov 2020 19:39:28 +0800 Subject: [PATCH 01/26] Refactor builtin tuner installation --- nni/tools/nnictl/algo_management.py | 98 +++++++++++++ nni/tools/nnictl/config_schema.py | 2 +- nni/tools/nnictl/constants.py | 21 --- nni/tools/nnictl/launcher.py | 8 +- nni/tools/nnictl/nnictl.py | 34 +++-- nni/tools/nnictl/package_management.py | 184 ------------------------- nni/tools/package_utils/__init__.py | 85 +++--------- nni/tools/package_utils/constants.py | 91 ------------ 8 files changed, 140 insertions(+), 383 deletions(-) create mode 100644 nni/tools/nnictl/algo_management.py delete mode 100644 nni/tools/nnictl/package_management.py delete mode 100644 nni/tools/package_utils/constants.py diff --git a/nni/tools/nnictl/algo_management.py b/nni/tools/nnictl/algo_management.py new file mode 100644 index 0000000000..e3cba0b9ff --- /dev/null +++ b/nni/tools/nnictl/algo_management.py @@ -0,0 +1,98 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import os +from collections import defaultdict +import json +import pkginfo +import nni +from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ + write_package_meta, ALGO_TYPES +from .common_utils import print_error, print_green, get_yml_content + +PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] + +def read_reg_meta_list(meta_path): + content = get_yml_content(meta_path) + if content.get('algorithms'): + meta_list = content.get('algorithms') + else: + meta_list = [content] + for meta in meta_list: + assert 'algoType' in meta + assert 'builtinName' in meta + assert 'className' in meta + return meta_list + +def algo_reg(args): + print(args) + meta_list = read_reg_meta_list(args.meta_path) + print(meta_list) + for meta in meta_list: + save_package_meta_data(meta) + +def algo_unreg(args): + '''uninstall packages''' + name = args.name[0] + meta = get_installed_package_meta(None, name) + print(meta) + if meta is None: + print_error('package {} not found!'.format(name)) + return + if remove_package_meta_data(name): + print_green('{} uninstalled sucessfully!'.format(name)) + else: + print_error('Failed to uninstall {}!'.format(name)) + +def algo_show(args): + '''show specified packages''' + builtin_name = args.name[0] + meta = get_installed_package_meta(None, builtin_name) + if meta: + print(json.dumps(meta, indent=4)) + else: + print_error('package {} not found'.format(builtin_name)) + +def algo_list(args): + meta = read_installed_package_meta() + print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') + print('| Name | Type | source | Class Name | Module Name |') + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + MAX_MODULE_NAME = 38 + for t in ['tuners', 'assessors', 'advisors']: + for p in meta[t]: + module_name = '.'.join(p['className'].split('.')[:-1]) + if len(module_name) > MAX_MODULE_NAME: + module_name = module_name[:MAX_MODULE_NAME-3] + '...' + class_name = p['className'].split('.')[-1] + print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['builtinName'], t, p['source'], class_name, module_name[:38])) + print('+-----------------+------------+-----------+----------------------+------------------------------------------+') + + +def save_package_meta_data(meta_data): + assert meta_data['algoType'] in PACKAGE_TYPES + assert 'builtinName' in meta_data + assert 'className' in meta_data + meta_data['source'] = 'user' + + config = read_installed_package_meta() + + if meta_data['builtinName'] in [x['builtinName'] for x in config[meta_data['algoType']+'s']]: + raise ValueError('builtinName %s already installed' % meta_data['builtinName']) + + config[meta_data['algoType']+'s'].append(meta_data) + write_package_meta(config) + +def remove_package_meta_data(name): + config = read_installed_package_meta() + + updated = False + for t in ALGO_TYPES: + for meta in config[t]: + if meta['builtinName'] == name: + config[t].remove(meta) + updated = True + if updated: + write_package_meta(config) + return True + return False diff --git a/nni/tools/nnictl/config_schema.py b/nni/tools/nnictl/config_schema.py index d320163595..9dd8282ba1 100644 --- a/nni/tools/nnictl/config_schema.py +++ b/nni/tools/nnictl/config_schema.py @@ -76,7 +76,7 @@ def validate_class_args(self, class_args, algo_type, builtin_name): if not builtin_name or not class_args: return meta = get_builtin_algo_meta(algo_type+'s', builtin_name) - if meta and 'accept_class_args' in meta and meta['accept_class_args'] == False: + if meta and 'acceptClassArgs' in meta and meta['acceptClassArgs'] == False: raise SchemaError('classArgs is not allowed.') logging.getLogger('nni.protocol').setLevel(logging.ERROR) # we know IPC is not there, don't complain diff --git a/nni/tools/nnictl/constants.py b/nni/tools/nnictl/constants.py index 0654473ed4..559c6caada 100644 --- a/nni/tools/nnictl/constants.py +++ b/nni/tools/nnictl/constants.py @@ -61,27 +61,6 @@ TRIAL_MONITOR_TAIL = '-------------------------------------------------------------------------------------\n\n\n' -INSTALLABLE_PACKAGE_META = { - 'SMAC': { - 'type': 'tuner', - 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', - 'code_sub_dir': 'smac_tuner', - 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator' - }, - 'BOHB': { - 'type': 'advisor', - 'class_name': 'nni.bohb_advisor.bohb_advisor.BOHB', - 'code_sub_dir': 'bohb_advisor', - 'class_args_validator': 'nni.bohb_advisor.bohb_advisor.BOHBClassArgsValidator' - }, - 'PPOTuner': { - 'type': 'tuner', - 'class_name': 'nni.ppo_tuner.ppo_tuner.PPOTuner', - 'code_sub_dir': 'ppo_tuner', - 'class_args_validator': 'nni.ppo_tuner.ppo_tuner.PPOClassArgsValidator' - } -} - TUNERS_SUPPORTING_IMPORT_DATA = { 'TPE', 'Anneal', diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index 576954e335..004f3c51a9 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -19,7 +19,7 @@ from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \ detect_port, get_user -from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, INSTALLABLE_PACKAGE_META +from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER from .command_utils import check_output_command, kill_command from .nnictl_utils import update_experiment @@ -441,9 +441,9 @@ def launch_experiment(args, experiment_config, mode, config_file_name, experimen except CalledProcessError: print_error('some errors happen when import package %s.' %(package_name)) print_log_content(config_file_name) - if package_name in INSTALLABLE_PACKAGE_META: - print_error('If %s is not installed, it should be installed through '\ - '\'nnictl package install --name %s\''%(package_name, package_name)) + #if package_name in INSTALLABLE_PACKAGE_META: + # print_error('If %s is not installed, it should be installed through '\ + # '\'nnictl package install --name %s\''%(package_name, package_name)) exit(1) log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index cc5d4cdd11..c2608a6c21 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -13,7 +13,7 @@ monitor_experiment, export_trials_data, trial_codegen, webui_url, \ get_config, log_stdout, log_stderr, search_space_auto_gen, webui_nas, \ save_experiment, load_experiment -from .package_management import package_install, package_uninstall, package_show, package_list +from .algo_management import algo_reg, algo_unreg, algo_show, algo_list from .constants import DEFAULT_REST_PORT from .tensorboard_utils import start_tensorboard, stop_tensorboard init(autoreset=True) @@ -212,26 +212,24 @@ def parse_args(): parser_log_trial.add_argument('--trial_id', '-T', dest='trial_id', help='find trial log path by id') parser_log_trial.set_defaults(func=log_trial) - #parse package command - parser_package = subparsers.add_parser('package', help='control nni tuner and assessor packages') - # add subparsers for parser_package - parser_package_subparsers = parser_package.add_subparsers() - parser_package_install = parser_package_subparsers.add_parser('install', help='install packages') - parser_package_install.add_argument('source', nargs='?', help='installation source, can be a directory or whl file') - parser_package_install.add_argument('--name', '-n', dest='name', help='package name to be installed', required=False) - parser_package_install.set_defaults(func=package_install) + #parse algo command + parser_algo = subparsers.add_parser('algo', help='control nni builtin tuner, assessor and advisor algorithms') + # add subparsers for parser_algo + parser_algo_subparsers = parser_algo.add_subparsers() + parser_algo_reg = parser_algo_subparsers.add_parser('register', help='register algorithms as nni builtin algorithm') + parser_algo_reg.add_argument('--meta_path', '-m', dest='meta_path', help='path to the meta file', required=True) + parser_algo_reg.set_defaults(func=algo_reg) - parser_package_uninstall = parser_package_subparsers.add_parser('uninstall', help='uninstall packages') - parser_package_uninstall.add_argument('name', nargs=1, help='package name to be uninstalled') - parser_package_uninstall.set_defaults(func=package_uninstall) + parser_algo_unreg = parser_algo_subparsers.add_parser('unregister', help='unregister algorithm') + parser_algo_unreg.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_unreg.set_defaults(func=algo_unreg) - parser_package_show = parser_package_subparsers.add_parser('show', help='show the information of packages') - parser_package_show.add_argument('name', nargs=1, help='builtin name of the package') - parser_package_show.set_defaults(func=package_show) + parser_algo_show = parser_algo_subparsers.add_parser('show', help='show the information of algorithm') + parser_algo_show.add_argument('name', nargs=1, help='builtin name of the algorithm') + parser_algo_show.set_defaults(func=algo_show) - parser_package_list = parser_package_subparsers.add_parser('list', help='list installed packages') - parser_package_list.add_argument('--all', action='store_true', help='list all builtin packages') - parser_package_list.set_defaults(func=package_list) + parser_algo_list = parser_algo_subparsers.add_parser('list', help='list registered algorithms') + parser_algo_list.set_defaults(func=algo_list) #parse tensorboard command parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard') diff --git a/nni/tools/nnictl/package_management.py b/nni/tools/nnictl/package_management.py deleted file mode 100644 index 5eef340752..0000000000 --- a/nni/tools/nnictl/package_management.py +++ /dev/null @@ -1,184 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -import os -from collections import defaultdict -import json -import pkginfo -import nni -from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ - write_package_meta, get_builtin_algo_meta, get_not_installable_builtin_names, ALGO_TYPES - -from .constants import INSTALLABLE_PACKAGE_META -from .common_utils import print_error, print_green -from .command_utils import install_requirements_command, call_pip_install, call_pip_uninstall - -PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] - -def install_by_name(package_name): - if package_name not in INSTALLABLE_PACKAGE_META: - raise RuntimeError('{} is not found in installable packages!'.format(package_name)) - - requirements_path = os.path.join(nni.__path__[0], 'algorithms/hpo', INSTALLABLE_PACKAGE_META[package_name]['code_sub_dir'], 'requirements.txt') - assert os.path.exists(requirements_path) - - return install_requirements_command(requirements_path) - -def package_install(args): - '''install packages''' - installed = False - try: - if args.name: - if install_by_name(args.name) == 0: - package_meta = {} - package_meta['type'] = INSTALLABLE_PACKAGE_META[args.name]['type'] - package_meta['name'] = args.name - package_meta['class_name'] = INSTALLABLE_PACKAGE_META[args.name]['class_name'] - package_meta['class_args_validator'] = INSTALLABLE_PACKAGE_META[args.name]['class_args_validator'] - save_package_meta_data(package_meta) - print_green('{} installed!'.format(args.name)) - installed = True - else: - package_meta = get_nni_meta(args.source) - if package_meta: - if call_pip_install(args.source) == 0: - save_package_meta_data(package_meta) - print_green('{} installed!'.format(package_meta['name'])) - installed = True - except Exception as e: - print_error(e) - if not installed: - print_error('installation failed!') - -def package_uninstall(args): - '''uninstall packages''' - name = args.name[0] - if name in get_not_installable_builtin_names(): - print_error('{} can not be uninstalled!'.format(name)) - exit(1) - meta = get_installed_package_meta(None, name) - if meta is None: - print_error('package {} not found!'.format(name)) - return - if 'installed_package' in meta: - call_pip_uninstall(meta['installed_package']) - if remove_package_meta_data(name): - print_green('{} uninstalled sucessfully!'.format(name)) - else: - print_error('Failed to uninstall {}!'.format(name)) - -def package_show(args): - '''show specified packages''' - builtin_name = args.name[0] - meta = get_builtin_algo_meta(builtin_name=builtin_name) - if meta: - print(json.dumps(meta, indent=4)) - else: - print_error('package {} not found'.format(builtin_name)) - -def print_package_list(meta): - print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') - print('| Name | Type | Installed | Class Name | Module Name |') - print('+-----------------+------------+-----------+----------------------+------------------------------------------+') - MAX_MODULE_NAME = 38 - for t in ['tuners', 'assessors', 'advisors']: - for p in meta[t]: - module_name = '.'.join(p['class_name'].split('.')[:-1]) - if len(module_name) > MAX_MODULE_NAME: - module_name = module_name[:MAX_MODULE_NAME-3] + '...' - class_name = p['class_name'].split('.')[-1] - print('| {:15s} | {:10s} | {:9s} | {:20s} | {:40s} |'.format(p['name'], t, p['installed'], class_name, module_name[:38])) - print('+-----------------+------------+-----------+----------------------+------------------------------------------+') - -def package_list(args): - '''list all packages''' - if args.all: - meta = get_builtin_algo_meta() - else: - meta = read_installed_package_meta() - - installed_names = defaultdict(list) - for t in ['tuners', 'assessors', 'advisors']: - for p in meta[t]: - p['installed'] = 'Yes' - installed_names[t].append(p['name']) - for k, v in INSTALLABLE_PACKAGE_META.items(): - t = v['type']+'s' - if k not in installed_names[t]: - meta[t].append({ - 'name': k, - 'class_name': v['class_name'], - 'class_args_validator': v['class_args_validator'], - 'installed': 'No' - }) - - print_package_list(meta) - -def save_package_meta_data(meta_data): - assert meta_data['type'] in PACKAGE_TYPES - assert 'name' in meta_data - assert 'class_name' in meta_data - - config = read_installed_package_meta() - - if meta_data['name'] in [x['name'] for x in config[meta_data['type']+'s']]: - raise ValueError('name %s already installed' % meta_data['name']) - - package_meta = {k: meta_data[k] for k in ['name', 'class_name', 'class_args_validator'] if k in meta_data} - if 'package_name' in meta_data: - package_meta['installed_package'] = meta_data['package_name'] - config[meta_data['type']+'s'].append(package_meta) - write_package_meta(config) - -def remove_package_meta_data(name): - config = read_installed_package_meta() - - updated = False - for t in ALGO_TYPES: - for meta in config[t]: - if meta['name'] == name: - config[t].remove(meta) - updated = True - if updated: - write_package_meta(config) - return True - return False - -def get_nni_meta(source): - if not os.path.exists(source): - print_error('{} does not exist'.format(source)) - return None - - if os.path.isdir(source): - if not os.path.exists(os.path.join(source, 'setup.py')): - print_error('setup.py not found') - return None - pkg = pkginfo.Develop(source) - else: - if not source.endswith('.whl'): - print_error('File name {} must ends with \'.whl\''.format(source)) - return False - pkg = pkginfo.Wheel(source) - - classifiers = pkg.classifiers - meta = parse_classifiers(classifiers) - meta['package_name'] = pkg.name - return meta - -def parse_classifiers(classifiers): - parts = [] - for c in classifiers: - if c.startswith('NNI Package'): - parts = [x.strip() for x in c.split('::')] - break - if len(parts) < 4 or not all(parts): - raise ValueError('Can not find correct NNI meta data in package classifiers.') - meta = { - 'type': parts[1], - 'name': parts[2], - 'class_name': parts[3] - } - if len(parts) >= 5: - meta['class_args_validator'] = parts[4] - - return meta diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 9a1c004053..be1a5f7138 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -6,18 +6,13 @@ import os from pathlib import Path import sys - import ruamel.yaml as yaml - import nni -from .constants import BuiltinAlgorithms ALGO_TYPES = ['tuners', 'assessors', 'advisors'] def get_all_builtin_names(algo_type): - """Get all valid builtin names, including: - 1. BuiltinAlgorithms which is pre-installed. - 2. User installed packages in /config/installed_packages.yml + """Get all valid builtin names registered in /config/installed_packages.yml Parameters ---------- @@ -30,38 +25,9 @@ def get_all_builtin_names(algo_type): all builtin tuner names. """ assert algo_type in ALGO_TYPES - merged_dict = _get_merged_builtin_dict() - - builtin_names = [x['name'] for x in merged_dict[algo_type]] - return builtin_names - -def get_not_installable_builtin_names(algo_type=None): - """Get builtin names in BuiltinAlgorithms which do not need to be installed - and can be used once NNI is installed. - Parameters - ---------- - algo_type: str | None - can be one of 'tuners', 'assessors', 'advisors' or None + return [x['builtinName'] for x in read_installed_package_meta()[algo_type]] - Returns: list of string - ------- - All builtin names of specified type, for example, if algo_type is 'tuners', returns - all builtin tuner names. - If algo_type is None, returns all builtin names of all types. - """ - if algo_type is None: - meta = BuiltinAlgorithms - else: - assert algo_type in ALGO_TYPES - meta = { - algo_type: BuiltinAlgorithms[algo_type] - } - names = [] - for t in ALGO_TYPES: - if t in meta: - names.extend([x['name'] for x in meta[t]]) - return names def get_builtin_algo_meta(algo_type=None, builtin_name=None): """ Get meta information of builtin algorithms from: @@ -81,28 +47,28 @@ def get_builtin_algo_meta(algo_type=None, builtin_name=None): alogorithms, for example: { 'name': 'Random', - 'class_name': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { + 'className': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', + 'classArgs': { 'algorithm_name': 'random_search' }, - 'accept_class_args': False, - 'class_args_validator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' + 'acceptClassArgs': False, + 'classArgsValidator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' } If builtin_name is None, returns multiple meta information in a list. """ - merged_dict = _get_merged_builtin_dict() + algo_meta = read_installed_package_meta() if algo_type is None and builtin_name is None: - return merged_dict + return algo_meta if algo_type: assert algo_type in ALGO_TYPES - metas = merged_dict[algo_type] + metas = algo_meta[algo_type] else: - metas = merged_dict['tuners'] + merged_dict['assessors'] + merged_dict['advisors'] + metas = algo_meta['tuners'] + algo_meta['assessors'] + algo_meta['advisors'] if builtin_name: for m in metas: - if m['name'] == builtin_name: + if m['builtinName'] == builtin_name: return m else: return metas @@ -124,9 +90,9 @@ def get_installed_package_meta(algo_type, builtin_name): ------- Returns meta information of speicified builtin alogorithms, for example: { - 'class_args_validator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator', - 'class_name': 'nni.smac_tuner.smac_tuner.SMACTuner', - 'name': 'SMAC' + 'classArgsValidator': 'nni.smac_tuner.smac_tuner.SMACClassArgsValidator', + 'className': 'nni.smac_tuner.smac_tuner.SMACTuner', + 'builtinName': 'SMAC' } """ assert builtin_name is not None @@ -141,7 +107,7 @@ def get_installed_package_meta(algo_type, builtin_name): for algo_type in ALGO_TYPES: candidates.extend(config[algo_type]) for meta in candidates: - if meta['name'] == builtin_name: + if meta['builtinName'] == builtin_name: return meta return None @@ -171,7 +137,7 @@ def get_builtin_module_class_name(algo_type, builtin_name): meta = get_builtin_algo_meta(algo_type, builtin_name) if not meta: return None, None - return _parse_full_class_name(meta['class_name']) + return _parse_full_class_name(meta['className']) def create_validator_instance(algo_type, builtin_name): """Create instance of validator class @@ -191,9 +157,9 @@ def create_validator_instance(algo_type, builtin_name): assert algo_type in ALGO_TYPES assert builtin_name is not None meta = get_builtin_algo_meta(algo_type, builtin_name) - if not meta or 'class_args_validator' not in meta: + if not meta or 'classArgsValidator' not in meta: return None - module_name, class_name = _parse_full_class_name(meta['class_args_validator']) + module_name, class_name = _parse_full_class_name(meta['classArgsValidator']) class_module = importlib.import_module(module_name) class_constructor = getattr(class_module, class_name) @@ -229,11 +195,11 @@ class name: HyperoptTuner 2. merge user specified class args together with builtin class args. """ assert algo_meta - module_name, class_name = _parse_full_class_name(algo_meta['class_name']) + module_name, class_name = _parse_full_class_name(algo_meta['className']) class_args = {} - if 'class_args' in algo_meta: - class_args = algo_meta['class_args'] + if 'classArgs' in algo_meta: + class_args = algo_meta['classArgs'] if input_class_args is not None: class_args.update(input_class_args) @@ -310,12 +276,3 @@ def write_package_meta(config): config_file = get_package_config_path() with open(config_file, 'w') as f: f.write(yaml.dump(dict(config), default_flow_style=False)) - -def _get_merged_builtin_dict(): - def merge_meta_dict(d1, d2): - res = defaultdict(list) - for t in ALGO_TYPES: - res[t] = d1[t] + d2[t] - return res - - return merge_meta_dict(BuiltinAlgorithms, read_installed_package_meta()) diff --git a/nni/tools/package_utils/constants.py b/nni/tools/package_utils/constants.py deleted file mode 100644 index 952310b5f9..0000000000 --- a/nni/tools/package_utils/constants.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT license. - -BuiltinAlgorithms = { - 'tuners': [ - { - 'name': 'TPE', - 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'tpe' - }, - 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Random', - 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'random_search' - }, - 'accept_class_args': False, - 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Anneal', - 'class_name': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'class_args': { - 'algorithm_name': 'anneal' - }, - 'class_args_validator': 'nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - }, - { - 'name': 'Evolution', - 'class_name': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner', - 'class_args_validator': 'nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator' - }, - { - 'name': 'BatchTuner', - 'class_name': 'nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner', - 'accept_class_args': False, - }, - { - 'name': 'GridSearch', - 'class_name': 'nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner', - 'accept_class_args': False, - }, - { - 'name': 'NetworkMorphism', - 'class_name': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner', - 'class_args_validator': 'nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator' - }, - { - 'name': 'MetisTuner', - 'class_name': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner', - 'class_args_validator': 'nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator' - }, - { - 'name': 'GPTuner', - 'class_name': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner', - 'class_args_validator': 'nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator' - }, - { - 'name': 'PBTTuner', - 'class_name': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner', - 'class_args_validator': 'nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator' - }, - { - 'name': 'RegularizedEvolutionTuner', - 'class_name': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner', - 'class_args_validator': 'nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator' - } - ], - 'assessors': [ - { - 'name': 'Medianstop', - 'class_name': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor', - 'class_args_validator': 'nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator' - }, - { - 'name': 'Curvefitting', - 'class_name': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor', - 'class_args_validator': 'nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator' - }, - ], - 'advisors': [ - { - 'name': 'Hyperband', - 'class_name': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband', - 'class_args_validator': 'nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator' - } - ] -} From 1a247a8a4119a56634de2dcf24c1642b2d968a15 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Sat, 28 Nov 2020 20:09:37 +0800 Subject: [PATCH 02/26] update setup.py --- deployment/registered_algorithms.yml | 78 ++++++++++++++++++++++++++++ nni/tools/package_utils/__init__.py | 5 +- setup.py | 7 +++ 3 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 deployment/registered_algorithms.yml diff --git a/deployment/registered_algorithms.yml b/deployment/registered_algorithms.yml new file mode 100644 index 0000000000..62bf8b612f --- /dev/null +++ b/deployment/registered_algorithms.yml @@ -0,0 +1,78 @@ +advisors: +- builtinName: Hyperband + classArgsValidator: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator + className: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband + source: nni +- builtinName: BOHB + classArgsValidator: nni.bohb_advisor.bohb_advisor.BOHBClassArgsValidator + className: nni.bohb_advisor.bohb_advisor.BOHB + source: nni +assessors: +- builtinName: Medianstop + classArgsValidator: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator + className: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor + source: nni +- builtinName: Curvefitting + classArgsValidator: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator + className: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor + source: nni +tuners: +- builtinName: PPOTuner + classArgsValidator: nni.ppo_tuner.ppo_tuner.PPOClassArgsValidator + className: nni.ppo_tuner.ppo_tuner.PPOTuner + source: nni +- builtinName: SMAC + classArgsValidator: nni.smac_tuner.smac_tuner.SMACClassArgsValidator + className: nni.smac_tuner.smac_tuner.SMACTuner + source: nni +- builtinName: TPE + classArgs: + algorithm_name: tpe + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner + source: nni +- acceptClassArgs: false + builtinName: Random + classArgs: + algorithm_name: random_search + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Anneal + classArgs: + algorithm_name: anneal + classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator + className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner + source: nni +- builtinName: Evolution + classArgsValidator: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner + source: nni +- acceptClassArgs: false + builtinName: BatchTuner + className: nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner + source: nni +- acceptClassArgs: false + builtinName: GridSearch + className: nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner + source: nni +- builtinName: NetworkMorphism + classArgsValidator: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator + className: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner + source: nni +- builtinName: MetisTuner + classArgsValidator: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator + className: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner + source: nni +- builtinName: GPTuner + classArgsValidator: nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator + className: nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner + source: nni +- builtinName: PBTTuner + classArgsValidator: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator + className: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner + source: nni +- builtinName: RegularizedEvolutionTuner + classArgsValidator: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator + className: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner + source: nni diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index be1a5f7138..c7dea83a1d 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -255,10 +255,11 @@ def create_customized_class_instance(class_params): def get_package_config_path(): # FIXME: this might not be the desired location - config_dir = Path(nni.__path__[0]).parent / 'nni_config' + #config_dir = Path(nni.__path__[0]).parent / 'nni_config' + config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(config_dir): os.makedirs(config_dir, exist_ok=True) - return os.path.join(config_dir, 'installed_packages.yml') + return os.path.join(config_dir, 'registered_algorithms.yml') def read_installed_package_meta(): config_file = get_package_config_path() diff --git a/setup.py b/setup.py index 0c22b67c04..e5fa8f0a1a 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,11 @@ def _find_node_files(): def _using_conda_or_virtual_environment(): return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) +def _copy_data_files(): + nni_config_dir = os.path.expanduser('~/.config/nni') + if not os.path.exists(nni_config_dir): + os.makedirs(nni_config_dir) + shutil.copyfile('./deployment/registered_algorithms.yml', os.path.join(nni_config_dir, 'registered_algorithms.yml')) class BuildTs(Command): description = 'build TypeScript modules' @@ -166,6 +171,7 @@ def run(self): assert release, 'Please set environment variable "NNI_RELEASE="' assert os.path.isfile('nni_node/main.js'), 'Please run "build_ts" before "build"' assert not os.path.islink('nni_node/main.js'), 'This is a development build' + _copy_data_files() super().run() class Develop(develop): @@ -188,6 +194,7 @@ def finalize_options(self): def run(self): setup_ts.build(release=None) + _copy_data_files() super().run() class Clean(clean): From 6b783dd9edc10e9dbb88fde490869136edcbbfb6 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 14:27:06 +0800 Subject: [PATCH 03/26] updates --- docs/en_US/Tuner/InstallCustomizedTuner.md | 21 +++++---- .../tuners/customized_tuner/meta_file.yml | 4 ++ examples/tuners/customized_tuner/setup.py | 7 --- nni/tools/nnictl/algo_management.py | 43 ++++++++++++------- nni/tools/package_utils/__init__.py | 24 +++++------ 5 files changed, 55 insertions(+), 44 deletions(-) create mode 100644 examples/tuners/customized_tuner/meta_file.yml diff --git a/docs/en_US/Tuner/InstallCustomizedTuner.md b/docs/en_US/Tuner/InstallCustomizedTuner.md index f24c0248b7..9142041245 100644 --- a/docs/en_US/Tuner/InstallCustomizedTuner.md +++ b/docs/en_US/Tuner/InstallCustomizedTuner.md @@ -2,21 +2,18 @@ You can following below steps to install a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner. -## Prepare installation source and install package +## Install the customized tuner package into python environment -There are 2 options to install this customized tuner: +There are 2 options to install the package into python environment: ### Option 1: install from directory -Step 1: From `nni/examples/tuners/customized_tuner` directory, run: +From `nni/examples/tuners/customized_tuner` directory, run: `python setup.py develop` This command will build the `nni/examples/tuners/customized_tuner` directory as a pip installation source. -Step 2: Run command: - -`nnictl package install ./` ### Option 2: install from whl file @@ -28,16 +25,22 @@ This command build a whl file which is a pip installation source. Step 2: Run command: -`nnictl package install dist/demo_tuner-0.1-py3-none-any.whl` +`pip install dist/demo_tuner-0.1-py3-none-any.whl` + +## Register the installed package as builtin tuner: + +Run following command: + +`nnictl algo register --meta meta_file.yml` ## Check the installed package Then run command `nnictl package list`, you should be able to see that demotuner is installed: ``` +-----------------+------------+-----------+--------=-------------+------------------------------------------+ -| Name | Type | Installed | Class Name | Module Name | +| Name | Type | source | Class Name | Module Name | +-----------------+------------+-----------+----------------------+------------------------------------------+ -| demotuner | tuners | Yes | DemoTuner | demo_tuner | +| demotuner | tuners | user | DemoTuner | demo_tuner | +-----------------+------------+-----------+----------------------+------------------------------------------+ ``` diff --git a/examples/tuners/customized_tuner/meta_file.yml b/examples/tuners/customized_tuner/meta_file.yml new file mode 100644 index 0000000000..64a79f6c9f --- /dev/null +++ b/examples/tuners/customized_tuner/meta_file.yml @@ -0,0 +1,4 @@ +algoType: tuner +builtinName: demotuner +className: demo_tuner.DemoTuner +classArgsValidator: demo_tuner.MyClassArgsValidator diff --git a/examples/tuners/customized_tuner/setup.py b/examples/tuners/customized_tuner/setup.py index 9dcd16c76e..2ea0e4748b 100644 --- a/examples/tuners/customized_tuner/setup.py +++ b/examples/tuners/customized_tuner/setup.py @@ -9,13 +9,6 @@ packages = setuptools.find_packages(exclude=['*test*']), python_requires = '>=3.6', - classifiers = [ - 'Programming Language :: Python :: 3', - 'License :: OSI Approved :: MIT License', - 'Operating System :: ', - 'NNI Package :: tuner :: demotuner :: demo_tuner.DemoTuner :: demo_tuner.MyClassArgsValidator' - ], - author = 'Microsoft NNI Team', author_email = 'nni@microsoft.com', description = 'NNI control for Neural Network Intelligence project', diff --git a/nni/tools/nnictl/algo_management.py b/nni/tools/nnictl/algo_management.py index e3cba0b9ff..8f36d50852 100644 --- a/nni/tools/nnictl/algo_management.py +++ b/nni/tools/nnictl/algo_management.py @@ -3,11 +3,12 @@ import os from collections import defaultdict +import importlib import json import pkginfo import nni -from nni.tools.package_utils import read_installed_package_meta, get_installed_package_meta, \ - write_package_meta, ALGO_TYPES +from nni.tools.package_utils import read_registerd_algo_meta, get_installed_package_meta, \ + write_registered_algo_meta, ALGO_TYPES, parse_full_class_name from .common_utils import print_error, print_green, get_yml_content PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] @@ -24,25 +25,35 @@ def read_reg_meta_list(meta_path): assert 'className' in meta return meta_list +def verify_algo_import(meta): + def _do_verify_import(fullName): + module_name, class_name = parse_full_class_name(fullName) + class_module = importlib.import_module(module_name) + getattr(class_module, class_name) + + _do_verify_import(meta['className']) + + if meta.get('classArgsValidator'): + _do_verify_import(meta['classArgsValidator']) + def algo_reg(args): - print(args) meta_list = read_reg_meta_list(args.meta_path) - print(meta_list) for meta in meta_list: - save_package_meta_data(meta) + verify_algo_import(meta) + save_algo_meta_data(meta) + print_green('{} registered sucessfully!'.format(meta['builtinName'])) def algo_unreg(args): '''uninstall packages''' name = args.name[0] meta = get_installed_package_meta(None, name) - print(meta) if meta is None: print_error('package {} not found!'.format(name)) return - if remove_package_meta_data(name): - print_green('{} uninstalled sucessfully!'.format(name)) + if remove_algo_meta_data(name): + print_green('{} unregistered sucessfully!'.format(name)) else: - print_error('Failed to uninstall {}!'.format(name)) + print_error('Failed to unregistered {}!'.format(name)) def algo_show(args): '''show specified packages''' @@ -54,7 +65,7 @@ def algo_show(args): print_error('package {} not found'.format(builtin_name)) def algo_list(args): - meta = read_installed_package_meta() + meta = read_registerd_algo_meta() print('+-----------------+------------+-----------+--------=-------------+------------------------------------------+') print('| Name | Type | source | Class Name | Module Name |') print('+-----------------+------------+-----------+----------------------+------------------------------------------+') @@ -69,22 +80,22 @@ def algo_list(args): print('+-----------------+------------+-----------+----------------------+------------------------------------------+') -def save_package_meta_data(meta_data): +def save_algo_meta_data(meta_data): assert meta_data['algoType'] in PACKAGE_TYPES assert 'builtinName' in meta_data assert 'className' in meta_data meta_data['source'] = 'user' - config = read_installed_package_meta() + config = read_registerd_algo_meta() if meta_data['builtinName'] in [x['builtinName'] for x in config[meta_data['algoType']+'s']]: raise ValueError('builtinName %s already installed' % meta_data['builtinName']) config[meta_data['algoType']+'s'].append(meta_data) - write_package_meta(config) + write_registered_algo_meta(config) -def remove_package_meta_data(name): - config = read_installed_package_meta() +def remove_algo_meta_data(name): + config = read_registerd_algo_meta() updated = False for t in ALGO_TYPES: @@ -93,6 +104,6 @@ def remove_package_meta_data(name): config[t].remove(meta) updated = True if updated: - write_package_meta(config) + write_registered_algo_meta(config) return True return False diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index c7dea83a1d..cc3aa1908b 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -26,7 +26,7 @@ def get_all_builtin_names(algo_type): """ assert algo_type in ALGO_TYPES - return [x['builtinName'] for x in read_installed_package_meta()[algo_type]] + return [x['builtinName'] for x in read_registerd_algo_meta()[algo_type]] def get_builtin_algo_meta(algo_type=None, builtin_name=None): @@ -56,7 +56,7 @@ def get_builtin_algo_meta(algo_type=None, builtin_name=None): } If builtin_name is None, returns multiple meta information in a list. """ - algo_meta = read_installed_package_meta() + algo_meta = read_registerd_algo_meta() if algo_type is None and builtin_name is None: return algo_meta @@ -98,7 +98,7 @@ def get_installed_package_meta(algo_type, builtin_name): assert builtin_name is not None if algo_type: assert algo_type in ALGO_TYPES - config = read_installed_package_meta() + config = read_registerd_algo_meta() candidates = [] if algo_type: @@ -111,7 +111,7 @@ def get_installed_package_meta(algo_type, builtin_name): return meta return None -def _parse_full_class_name(full_class_name): +def parse_full_class_name(full_class_name): if not full_class_name: return None, None parts = full_class_name.split('.') @@ -137,7 +137,7 @@ def get_builtin_module_class_name(algo_type, builtin_name): meta = get_builtin_algo_meta(algo_type, builtin_name) if not meta: return None, None - return _parse_full_class_name(meta['className']) + return parse_full_class_name(meta['className']) def create_validator_instance(algo_type, builtin_name): """Create instance of validator class @@ -159,7 +159,7 @@ def create_validator_instance(algo_type, builtin_name): meta = get_builtin_algo_meta(algo_type, builtin_name) if not meta or 'classArgsValidator' not in meta: return None - module_name, class_name = _parse_full_class_name(meta['classArgsValidator']) + module_name, class_name = parse_full_class_name(meta['classArgsValidator']) class_module = importlib.import_module(module_name) class_constructor = getattr(class_module, class_name) @@ -195,7 +195,7 @@ class name: HyperoptTuner 2. merge user specified class args together with builtin class args. """ assert algo_meta - module_name, class_name = _parse_full_class_name(algo_meta['className']) + module_name, class_name = parse_full_class_name(algo_meta['className']) class_args = {} if 'classArgs' in algo_meta: @@ -253,7 +253,7 @@ def create_customized_class_instance(class_params): return instance -def get_package_config_path(): +def get_registered_algo_config_path(): # FIXME: this might not be the desired location #config_dir = Path(nni.__path__[0]).parent / 'nni_config' config_dir = os.path.expanduser('~/.config/nni') @@ -261,8 +261,8 @@ def get_package_config_path(): os.makedirs(config_dir, exist_ok=True) return os.path.join(config_dir, 'registered_algorithms.yml') -def read_installed_package_meta(): - config_file = get_package_config_path() +def read_registerd_algo_meta(): + config_file = get_registered_algo_config_path() if os.path.exists(config_file): with open(config_file, 'r') as f: config = yaml.load(f, Loader=yaml.Loader) @@ -273,7 +273,7 @@ def read_installed_package_meta(): config[t] = [] return config -def write_package_meta(config): - config_file = get_package_config_path() +def write_registered_algo_meta(config): + config_file = get_registered_algo_config_path() with open(config_file, 'w') as f: f.write(yaml.dump(dict(config), default_flow_style=False)) From a13ea4a14859e53c6f8a9b4395ba645c5882aebc Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 14:53:49 +0800 Subject: [PATCH 04/26] updates --- deployment/registered_algorithms.yml | 12 ++++++------ examples/tuners/customized_tuner/setup.py | 5 +++++ setup.py | 8 ++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/deployment/registered_algorithms.yml b/deployment/registered_algorithms.yml index 62bf8b612f..3cf8a322d8 100644 --- a/deployment/registered_algorithms.yml +++ b/deployment/registered_algorithms.yml @@ -4,8 +4,8 @@ advisors: className: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband source: nni - builtinName: BOHB - classArgsValidator: nni.bohb_advisor.bohb_advisor.BOHBClassArgsValidator - className: nni.bohb_advisor.bohb_advisor.BOHB + classArgsValidator: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHBClassArgsValidator + className: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHB source: nni assessors: - builtinName: Medianstop @@ -18,12 +18,12 @@ assessors: source: nni tuners: - builtinName: PPOTuner - classArgsValidator: nni.ppo_tuner.ppo_tuner.PPOClassArgsValidator - className: nni.ppo_tuner.ppo_tuner.PPOTuner + classArgsValidator: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOClassArgsValidator + className: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOTuner source: nni - builtinName: SMAC - classArgsValidator: nni.smac_tuner.smac_tuner.SMACClassArgsValidator - className: nni.smac_tuner.smac_tuner.SMACTuner + classArgsValidator: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACClassArgsValidator + className: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACTuner source: nni - builtinName: TPE classArgs: diff --git a/examples/tuners/customized_tuner/setup.py b/examples/tuners/customized_tuner/setup.py index 2ea0e4748b..9c32114917 100644 --- a/examples/tuners/customized_tuner/setup.py +++ b/examples/tuners/customized_tuner/setup.py @@ -9,6 +9,11 @@ packages = setuptools.find_packages(exclude=['*test*']), python_requires = '>=3.6', + classifiers = [ + 'Programming Language :: Python :: 3', + 'License :: OSI Approved :: MIT License', + 'Operating System :: ' + ], author = 'Microsoft NNI Team', author_email = 'nni@microsoft.com', description = 'NNI control for Neural Network Intelligence project', diff --git a/setup.py b/setup.py index 4dfff41be6..8bfa07a12e 100644 --- a/setup.py +++ b/setup.py @@ -107,6 +107,14 @@ def _setup(): python_requires = '>=3.6', install_requires = dependencies, + extras_require = { + 'SMAC': [ + 'git+https://github.com/QuanluZhang/ConfigSpace.git', + 'git+https://github.com/QuanluZhang/SMAC3.git' + ], + 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], + 'PPOTuner': ['enum34', 'gym'] + } setup_requires = ['requests'], entry_points = { From a918f6ceaa983a3fa2517db7426b6e730cad2716 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 15:22:02 +0800 Subject: [PATCH 05/26] updates --- nni/tools/package_utils/__init__.py | 12 +++++++++--- setup.py | 5 ++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index cc3aa1908b..842dedd3c4 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -253,13 +253,19 @@ def create_customized_class_instance(class_params): return instance +def _using_conda_or_virtual_environment(): + return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) + def get_registered_algo_config_path(): # FIXME: this might not be the desired location #config_dir = Path(nni.__path__[0]).parent / 'nni_config' - config_dir = os.path.expanduser('~/.config/nni') - if not os.path.exists(config_dir): + if _using_conda_or_virtual_environment(): + nni_config_dir = os.path.join(sys.prefix, 'nni') + else: + nni_config_dir = os.path.expanduser('~/.config/nni') + if not os.path.exists(nni_config_dir): os.makedirs(config_dir, exist_ok=True) - return os.path.join(config_dir, 'registered_algorithms.yml') + return os.path.join(nni_config_dir, 'registered_algorithms.yml') def read_registerd_algo_meta(): config_file = get_registered_algo_config_path() diff --git a/setup.py b/setup.py index 8bfa07a12e..97c6b94a08 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,10 @@ def _using_conda_or_virtual_environment(): return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) def _copy_data_files(): - nni_config_dir = os.path.expanduser('~/.config/nni') + if _using_conda_or_virtual_environment(): + nni_config_dir = os.path.join(sys.prefix, 'nni') + else: + nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): os.makedirs(nni_config_dir) shutil.copyfile('./deployment/registered_algorithms.yml', os.path.join(nni_config_dir, 'registered_algorithms.yml')) From e2857625b1467bf6d8f4132703dc3379573a71ff Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 15:26:05 +0800 Subject: [PATCH 06/26] updates --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 97c6b94a08..f15596f849 100644 --- a/setup.py +++ b/setup.py @@ -114,7 +114,7 @@ def _setup(): ], 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], 'PPOTuner': ['enum34', 'gym'] - } + }, setup_requires = ['requests'], entry_points = { From ce7830f3eb526c7243155a09e56e2b444ff44e6d Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 15:30:33 +0800 Subject: [PATCH 07/26] updates --- setup.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index f15596f849..6b3e49d48f 100644 --- a/setup.py +++ b/setup.py @@ -108,10 +108,10 @@ def _setup(): python_requires = '>=3.6', install_requires = dependencies, extras_require = { - 'SMAC': [ - 'git+https://github.com/QuanluZhang/ConfigSpace.git', - 'git+https://github.com/QuanluZhang/SMAC3.git' - ], + #'SMAC': [ + # 'git+https://github.com/QuanluZhang/ConfigSpace.git', + # 'git+https://github.com/QuanluZhang/SMAC3.git' + #], 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], 'PPOTuner': ['enum34', 'gym'] }, From 77f8da71a743bcb98f33b2c8b76af33c6c235cc4 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 15:53:06 +0800 Subject: [PATCH 08/26] updates --- pipelines/fast-test.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 7e0fd53ece..30c0a23762 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 setup.py develop + python3 -m pip install -e .[BOHB,PPOTuner] displayName: 'Install NNI' - script: | @@ -40,8 +40,6 @@ jobs: python3 -m pip install -U gym onnx peewee thop python3 -m pip install -U sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx sudo apt-get install swig -y - nnictl package install --name=SMAC - nnictl package install --name=BOHB displayName: 'Install extra dependencies' - script: | From 796ec589138ccd935c0f352303ec6a0c613c8d76 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 15:55:59 +0800 Subject: [PATCH 09/26] updates --- pipelines/fast-test.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 30c0a23762..95e6bf4a75 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -89,7 +89,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 setup.py develop + python3 -m pip install -e .[BOHB,PPOTuner] displayName: 'Install NNI' - script: | @@ -99,8 +99,6 @@ jobs: python3 -m pip install -U keras==2.1.6 python3 -m pip install -U gym onnx peewee sudo apt-get install swig -y - nnictl package install --name=SMAC - nnictl package install --name=BOHB displayName: 'Install extra dependencies' - script: | @@ -142,7 +140,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 setup.py develop + python3 -m pip install -e .[BOHB,PPOTuner] displayName: 'Install NNI' - script: | @@ -160,7 +158,6 @@ jobs: brew install swig@3 rm -f /usr/local/bin/swig ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig - nnictl package install --name=SMAC displayName: 'Install extra dependencies' - script: | From 9f67a073819c93d53f0a5e00ea8ca39723ce2f15 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 16:01:27 +0800 Subject: [PATCH 10/26] updates --- pipelines/fast-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 95e6bf4a75..2aef72072f 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 -m pip install -e .[BOHB,PPOTuner] + python3 setup.py develop displayName: 'Install NNI' - script: | @@ -89,7 +89,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 -m pip install -e .[BOHB,PPOTuner] + python3 setup.py develop displayName: 'Install NNI' - script: | @@ -140,7 +140,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 -m pip install -e .[BOHB,PPOTuner] + python3 setup.py develop displayName: 'Install NNI' - script: | From 015966d9e8260823608fa817c0834345953be6b7 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Sun, 29 Nov 2020 16:25:13 +0800 Subject: [PATCH 11/26] updates --- nni/tools/package_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 842dedd3c4..4bc32da195 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -264,7 +264,7 @@ def get_registered_algo_config_path(): else: nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): - os.makedirs(config_dir, exist_ok=True) + os.makedirs(nni_config_dir, exist_ok=True) return os.path.join(nni_config_dir, 'registered_algorithms.yml') def read_registerd_algo_meta(): From 0f5d4bb356c42806abeec68c4e0cb15cd0db105e Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 00:17:08 +0800 Subject: [PATCH 12/26] update docs --- docs/en_US/Tutorial/InstallCustomizedAlgos.md | 135 +++++++----------- docs/en_US/Tutorial/Nnictl.md | 73 ++++------ nni/tools/nnictl/algo_management.py | 5 +- pipelines/fast-test.yml | 10 ++ 4 files changed, 93 insertions(+), 130 deletions(-) diff --git a/docs/en_US/Tutorial/InstallCustomizedAlgos.md b/docs/en_US/Tutorial/InstallCustomizedAlgos.md index ff5a3d3514..5b58105140 100644 --- a/docs/en_US/Tutorial/InstallCustomizedAlgos.md +++ b/docs/en_US/Tutorial/InstallCustomizedAlgos.md @@ -1,9 +1,9 @@ -**How to install customized algorithms as builtin tuners, assessors and advisors** +**How to register customized algorithms as builtin tuners, assessors and advisors** === ## Overview -NNI provides a lot of [builtin tuners](../Tuner/BuiltinTuner.md), [advisors](../Tuner/HyperbandAdvisor.md) and [assessors](../Assessor/BuiltinAssessor.md) can be used directly for Hyper Parameter Optimization, and some extra algorithms can be installed via `nnictl package install --name ` after NNI is installed. You can check these extra algorithms via `nnictl package list` command. +NNI provides a lot of [builtin tuners](../Tuner/BuiltinTuner.md), [advisors](../Tuner/HyperbandAdvisor.md) and [assessors](../Assessor/BuiltinAssessor.md) can be used directly for Hyper Parameter Optimization, and some extra algorithms can be registered via `nnictl algo register --meta ` after NNI is installed. You can check builtin algorithms via `nnictl algo list` command. NNI also provides the ability to build your own customized tuners, advisors and assessors. To use the customized algorithm, users can simply follow the spec in experiment config file to properly reference the algorithm, which has been illustrated in the tutorials of [customized tuners](../Tuner/CustomizeTuner.md)/[advisors](../Tuner/CustomizeAdvisor.md)/[assessors](../Assessor/CustomizeAssessor.md). @@ -13,8 +13,8 @@ tuner: builtinTunerName: mytuner ``` -## Install customized algorithms as builtin tuners, assessors and advisors -You can follow below steps to build a customized tuner/assessor/advisor, and install it into NNI as builtin algorithm. +## Register customized algorithms as builtin tuners, assessors and advisors +You can follow below steps to build a customized tuner/assessor/advisor, and register it into NNI as builtin algorithm. ### 1. Create a customized tuner/assessor/advisor Reference following instructions to create: @@ -48,56 +48,43 @@ class MedianstopClassArgsValidator(ClassArgsValidator): ``` The validator will be invoked before experiment is started to check whether the classArgs fields are valid for your customized algorithms. -### 3. Prepare package installation source -In order to be installed as builtin tuners, assessors and advisors, the customized algorithms need to be packaged as installable source which can be recognized by `pip` command, under the hood nni calls `pip` command to install the package. -Besides being a common pip source, the package needs to provide meta information in the `classifiers` field. -Format of classifiers field is a following: -``` -NNI Package :: :: :: :: -``` -* `type`: type of algorithms, could be one of `tuner`, `assessor`, `advisor` -* `builtin name`: builtin name used in experiment configuration file -* `full class name of tuner`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner` -* `full class name of class args validator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator` - -Following is an example of classfiers in package's `setup.py`: +### 3. Install your customized algorithms into python environment +Firstly, the customized algorithms need to be prepared as a python package. Then you can install the package into python environment via: +* Run command `python setup.py develop` from the package directory, this command will install the package in development mode, this is recommended if your algorithm is under development. +* Run command `python setup.py bdist_wheel` from the package directory, this command build a whl file which is a pip installation source. Then run `pip install ` to install it. -```python - classifiers = [ - 'Programming Language :: Python :: 3', - 'License :: OSI Approved :: MIT License', - 'Operating System :: ', - 'NNI Package :: tuner :: demotuner :: demo_tuner.DemoTuner :: demo_tuner.MyClassArgsValidator' - ], -``` -Once you have the meta info in `setup.py`, you can build your pip installation source via: -* Run command `python setup.py develop` from the package directory, this command will build the directory as a pip installation source. -* Run command `python setup.py bdist_wheel` from the package directory, this command build a whl file which is a pip installation source. +### 4. Prepare meta file -NNI will look for the classifier starts with `NNI Package` to retrieve the package meta information while the package being installed with `nnictl package install ` command. +Create a yaml file with following keys as meta file: +* `algoType`: type of algorithms, could be one of `tuner`, `assessor`, `advisor` +* `builtinName`: builtin name used in experiment configuration file +* `className`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner` +* `classArgsValidator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator` -Reference [customized tuner example](../Tuner/InstallCustomizedTuner.md) for a full example. +Following is an example of the yaml file: -### 4. Install customized algorithms package into NNI - -If your installation source is prepared as a directory with `python setup.py develop`, you can install the package by following command: - -`nnictl package install ` +```yaml +algoType: tuner +builtinName: demotuner +className: demo_tuner.DemoTuner +classArgsValidator: demo_tuner.MyClassArgsValidator -For example: +``` -`nnictl package install nni/examples/tuners/customized_tuner/` +### 5. Register customized algorithms into NNI +Run following command to register the customized algorithms as builtin algorithms in NNI: -If your installation source is prepared as a whl file with `python setup.py bdist_wheel`, you can install the package by following command: +```bash +nnictl algo register --meta +``` +The `` is the path to the yaml file your created in above section. -`nnictl package install ` -For example: +Reference [customized tuner example](../Tuner/InstallCustomizedTuner.md) for a full example. -`nnictl package install nni/examples/tuners/customized_tuner/dist/demo_tuner-0.1-py3-none-any.whl` -## 5. Use the installed builtin algorithms in experiment +## 6. Use the installed builtin algorithms in experiment Once your customized algorithms is installed, you can use it in experiment configuration file the same way as other builtin tuners/assessors/advisors, for example: ```yaml @@ -109,56 +96,42 @@ tuner: ``` -## Manage packages using `nnictl package` +## Manage builtin algorithms using `nnictl algo` -### List installed packages +### List builtin algorithms -Run following command to list the installed packages: +Run following command to list the registered builtin algorithms: -``` -nnictl package list -+-----------------+------------+-----------+--------=-------------+------------------------------------------+ -| Name | Type | Installed | Class Name | Module Name | -+-----------------+------------+-----------+----------------------+------------------------------------------+ -| demotuner | tuners | Yes | DemoTuner | demo_tuner | -| SMAC | tuners | No | SMACTuner | nni.smac_tuner.smac_tuner | -| PPOTuner | tuners | No | PPOTuner | nni.ppo_tuner.ppo_tuner | -| BOHB | advisors | Yes | BOHB | nni.bohb_advisor.bohb_advisor | -+-----------------+------------+-----------+----------------------+------------------------------------------+ -``` - -Run following command to list all packages, including the builtin packages can not be uninstalled. - -``` -nnictl package list --all +```bash +nnictl algo list +-----------------+------------+-----------+--------=-------------+------------------------------------------+ -| Name | Type | Installed | Class Name | Module Name | +| Name | Type | Source | Class Name | Module Name | +-----------------+------------+-----------+----------------------+------------------------------------------+ -| TPE | tuners | Yes | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | -| Random | tuners | Yes | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | -| Anneal | tuners | Yes | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | -| Evolution | tuners | Yes | EvolutionTuner | nni.evolution_tuner.evolution_tuner | -| BatchTuner | tuners | Yes | BatchTuner | nni.batch_tuner.batch_tuner | -| GridSearch | tuners | Yes | GridSearchTuner | nni.gridsearch_tuner.gridsearch_tuner | -| NetworkMorphism | tuners | Yes | NetworkMorphismTuner | nni.networkmorphism_tuner.networkmo... | -| MetisTuner | tuners | Yes | MetisTuner | nni.metis_tuner.metis_tuner | -| GPTuner | tuners | Yes | GPTuner | nni.gp_tuner.gp_tuner | -| PBTTuner | tuners | Yes | PBTTuner | nni.pbt_tuner.pbt_tuner | -| SMAC | tuners | No | SMACTuner | nni.smac_tuner.smac_tuner | -| PPOTuner | tuners | No | PPOTuner | nni.ppo_tuner.ppo_tuner | -| Medianstop | assessors | Yes | MedianstopAssessor | nni.medianstop_assessor.medianstop_... | -| Curvefitting | assessors | Yes | CurvefittingAssessor | nni.curvefitting_assessor.curvefitt... | -| Hyperband | advisors | Yes | Hyperband | nni.hyperband_advisor.hyperband_adv... | -| BOHB | advisors | Yes | BOHB | nni.bohb_advisor.bohb_advisor | +| TPE | tuners | nni | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | +| Random | tuners | nni | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | +| Anneal | tuners | nni | HyperoptTuner | nni.hyperopt_tuner.hyperopt_tuner | +| Evolution | tuners | nni | EvolutionTuner | nni.evolution_tuner.evolution_tuner | +| BatchTuner | tuners | nni | BatchTuner | nni.batch_tuner.batch_tuner | +| GridSearch | tuners | nni | GridSearchTuner | nni.gridsearch_tuner.gridsearch_tuner | +| NetworkMorphism | tuners | nni | NetworkMorphismTuner | nni.networkmorphism_tuner.networkmo... | +| MetisTuner | tuners | nni | MetisTuner | nni.metis_tuner.metis_tuner | +| GPTuner | tuners | nni | GPTuner | nni.gp_tuner.gp_tuner | +| PBTTuner | tuners | nni | PBTTuner | nni.pbt_tuner.pbt_tuner | +| SMAC | tuners | nni | SMACTuner | nni.smac_tuner.smac_tuner | +| PPOTuner | tuners | nni | PPOTuner | nni.ppo_tuner.ppo_tuner | +| Medianstop | assessors | nni | MedianstopAssessor | nni.medianstop_assessor.medianstop_... | +| Curvefitting | assessors | nni | CurvefittingAssessor | nni.curvefitting_assessor.curvefitt... | +| Hyperband | advisors | nni | Hyperband | nni.hyperband_advisor.hyperband_adv... | +| BOHB | advisors | nni | BOHB | nni.bohb_advisor.bohb_advisor | +-----------------+------------+-----------+----------------------+------------------------------------------+ ``` -### Uninstall package +### Unregister builtin algorithms Run following command to uninstall an installed package: -`nnictl package uninstall ` +`nnictl algo unregister ` For example: -`nnictl package uninstall demotuner` +`nnictl algo unregister demotuner` diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index edb25afacd..a8c7616d56 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -769,47 +769,39 @@ Debug mode will disable version check function in Trialkeeper. |------|------|------ |------| |id| False| |ID of the experiment you want to set| - + -### Manage package +### Manage builtin algorithms -* __nnictl package install__ +* __nnictl algo register__ * Description - Install a package (customized algorithms or nni provided algorithms) as builtin tuner/assessor/advisor. + Register customized algorithms as builtin tuner/assessor/advisor. * Usage ```bash - nnictl package install --name + nnictl algo register --meta ``` + `` is the path to the meta data file in yml format, which has following keys: + * `algoType`: type of algorithms, could be one of `tuner`, `assessor`, `advisor` + * `builtinName`: builtin name used in experiment configuration file + * `className`: tuner class name, including its module name, for example: `demo_tuner.DemoTuner` + * `classArgsValidator`: class args validator class name, including its module name, for example: `demo_tuner.MyClassArgsValidator` - The available `` can be checked via `nnictl package list` command. - - or - - ```bash - nnictl package install - ``` - - Reference [Install customized algorithms](InstallCustomizedAlgos.md) to prepare the installation source. * Example - > Install SMAC tuner + > Install a customized tuner in nni examples ```bash - nnictl package install --name SMAC + cd nni/examples/tuners/customized_tuner + python3 setup.py develop + nnictl algo register --meta meta_file.yml ``` - > Install a customized tuner - - ```bash - nnictl package install nni/examples/tuners/customized_tuner/dist/demo_tuner-0.1-py3-none-any.whl - ``` - -* __nnictl package show__ +* __nnictl algo show__ * Description @@ -818,63 +810,48 @@ Debug mode will disable version check function in Trialkeeper. * Usage ```bash - nnictl package show + nnictl algo show ``` * Example ```bash - nnictl package show SMAC + nnictl algo show SMAC ``` -* __nnictl package list__ +* __nnictl algo list__ * Description - List the installed/all packages. + List the registered builtin algorithms * Usage ```bash - nnictl package list [OPTIONS] + nnictl algo list ``` - * Options - - |Name, shorthand|Required|Default|Description| - |------|------|------ |------| - |--all| False| |List all packages| - * Example - > List installed packages - - ```bash - nnictl package list - ``` - - > List all packages - ```bash - nnictl package list --all + nnictl algo list ``` -* __nnictl package uninstall__ +* __nnictl algo unregister__ * Description - Uninstall a package. + Unregister a registered customized builtin algorithms. The NNI provided builtin algorithms can not be unregistered. * Usage ```bash - nnictl package uninstall + nnictl algo unregister ``` * Example - Uninstall SMAC package ```bash - nnictl package uninstall SMAC + nnictl algo unregister demotuner ``` diff --git a/nni/tools/nnictl/algo_management.py b/nni/tools/nnictl/algo_management.py index 8f36d50852..791bcc3b37 100644 --- a/nni/tools/nnictl/algo_management.py +++ b/nni/tools/nnictl/algo_management.py @@ -48,7 +48,10 @@ def algo_unreg(args): name = args.name[0] meta = get_installed_package_meta(None, name) if meta is None: - print_error('package {} not found!'.format(name)) + print_error('builtin algorithms {} not found!'.format(name)) + return + if meta['source'] == 'nni': + print_error('{} is provided by nni, can not be unregistered!'.format(name)) return if remove_algo_meta_data(name): print_green('{} unregistered sucessfully!'.format(name)) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 2aef72072f..bb715be393 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -40,6 +40,10 @@ jobs: python3 -m pip install -U gym onnx peewee thop python3 -m pip install -U sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx sudo apt-get install swig -y + python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git + python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git + python3 -m pip install -U ConfigSpace==0.4.7 + python3 -m pip install -U statsmodels==0.10.0 displayName: 'Install extra dependencies' - script: | @@ -99,6 +103,10 @@ jobs: python3 -m pip install -U keras==2.1.6 python3 -m pip install -U gym onnx peewee sudo apt-get install swig -y + python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git + python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git + python3 -m pip install -U ConfigSpace==0.4.7 + python3 -m pip install -U statsmodels==0.10.0 displayName: 'Install extra dependencies' - script: | @@ -158,6 +166,8 @@ jobs: brew install swig@3 rm -f /usr/local/bin/swig ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig + python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git + python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git displayName: 'Install extra dependencies' - script: | From 2ee49d9f35923149f6dc01467800e1e850577fda Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 00:22:28 +0800 Subject: [PATCH 13/26] updates --- docs/en_US/Tuner/InstallCustomizedTuner.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/en_US/Tuner/InstallCustomizedTuner.md b/docs/en_US/Tuner/InstallCustomizedTuner.md index 9142041245..f02565ffc1 100644 --- a/docs/en_US/Tuner/InstallCustomizedTuner.md +++ b/docs/en_US/Tuner/InstallCustomizedTuner.md @@ -1,6 +1,6 @@ -# How to install customized tuner as a builtin tuner +# How to register a customized tuner as a builtin tuner -You can following below steps to install a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner. +You can following below steps to register a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner. ## Install the customized tuner package into python environment @@ -27,15 +27,15 @@ Step 2: Run command: `pip install dist/demo_tuner-0.1-py3-none-any.whl` -## Register the installed package as builtin tuner: +## Register the customized tuner as builtin tuner: Run following command: `nnictl algo register --meta meta_file.yml` -## Check the installed package +## Check the registered builtin algorithms -Then run command `nnictl package list`, you should be able to see that demotuner is installed: +Then run command `nnictl algo list`, you should be able to see that demotuner is installed: ``` +-----------------+------------+-----------+--------=-------------+------------------------------------------+ | Name | Type | source | Class Name | Module Name | From 1e974713ce2645a6edd1deb2c29fc6196d95ff74 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 00:31:37 +0800 Subject: [PATCH 14/26] updates --- nni/tools/nnictl/algo_management.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nni/tools/nnictl/algo_management.py b/nni/tools/nnictl/algo_management.py index 791bcc3b37..5231f7874f 100644 --- a/nni/tools/nnictl/algo_management.py +++ b/nni/tools/nnictl/algo_management.py @@ -2,17 +2,12 @@ # Licensed under the MIT license. import os -from collections import defaultdict import importlib import json -import pkginfo -import nni from nni.tools.package_utils import read_registerd_algo_meta, get_installed_package_meta, \ write_registered_algo_meta, ALGO_TYPES, parse_full_class_name from .common_utils import print_error, print_green, get_yml_content -PACKAGE_TYPES = ['tuner', 'assessor', 'advisor'] - def read_reg_meta_list(meta_path): content = get_yml_content(meta_path) if content.get('algorithms'): @@ -21,6 +16,7 @@ def read_reg_meta_list(meta_path): meta_list = [content] for meta in meta_list: assert 'algoType' in meta + assert meta['algoType'] in ['tuner', 'assessor', 'advisor'] assert 'builtinName' in meta assert 'className' in meta return meta_list @@ -44,7 +40,6 @@ def algo_reg(args): print_green('{} registered sucessfully!'.format(meta['builtinName'])) def algo_unreg(args): - '''uninstall packages''' name = args.name[0] meta = get_installed_package_meta(None, name) if meta is None: @@ -59,7 +54,6 @@ def algo_unreg(args): print_error('Failed to unregistered {}!'.format(name)) def algo_show(args): - '''show specified packages''' builtin_name = args.name[0] meta = get_installed_package_meta(None, builtin_name) if meta: @@ -84,9 +78,6 @@ def algo_list(args): def save_algo_meta_data(meta_data): - assert meta_data['algoType'] in PACKAGE_TYPES - assert 'builtinName' in meta_data - assert 'className' in meta_data meta_data['source'] = 'user' config = read_registerd_algo_meta() From f15afb3c4a4e422ba08b1f2688cbc48f0c197b53 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 00:56:24 +0800 Subject: [PATCH 15/26] updates --- nni/tools/nnictl/algo_management.py | 14 ++++++-------- nni/tools/nnictl/nnictl.py | 4 ++-- nni/tools/package_utils/__init__.py | 6 +++--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/nni/tools/nnictl/algo_management.py b/nni/tools/nnictl/algo_management.py index 5231f7874f..e671f295c4 100644 --- a/nni/tools/nnictl/algo_management.py +++ b/nni/tools/nnictl/algo_management.py @@ -4,7 +4,7 @@ import os import importlib import json -from nni.tools.package_utils import read_registerd_algo_meta, get_installed_package_meta, \ +from nni.tools.package_utils import read_registerd_algo_meta, get_registered_algo_meta, \ write_registered_algo_meta, ALGO_TYPES, parse_full_class_name from .common_utils import print_error, print_green, get_yml_content @@ -35,13 +35,16 @@ def _do_verify_import(fullName): def algo_reg(args): meta_list = read_reg_meta_list(args.meta_path) for meta in meta_list: + if get_registered_algo_meta(meta['builtinName']) is not None: + print_error('builtinName {} already registered'.format(meta['builtinName'])) + return verify_algo_import(meta) save_algo_meta_data(meta) print_green('{} registered sucessfully!'.format(meta['builtinName'])) def algo_unreg(args): name = args.name[0] - meta = get_installed_package_meta(None, name) + meta = get_registered_algo_meta(name) if meta is None: print_error('builtin algorithms {} not found!'.format(name)) return @@ -55,7 +58,7 @@ def algo_unreg(args): def algo_show(args): builtin_name = args.name[0] - meta = get_installed_package_meta(None, builtin_name) + meta = get_registered_algo_meta(builtin_name) if meta: print(json.dumps(meta, indent=4)) else: @@ -79,12 +82,7 @@ def algo_list(args): def save_algo_meta_data(meta_data): meta_data['source'] = 'user' - config = read_registerd_algo_meta() - - if meta_data['builtinName'] in [x['builtinName'] for x in config[meta_data['algoType']+'s']]: - raise ValueError('builtinName %s already installed' % meta_data['builtinName']) - config[meta_data['algoType']+'s'].append(meta_data) write_registered_algo_meta(config) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index c2608a6c21..55a2de0a21 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -216,11 +216,11 @@ def parse_args(): parser_algo = subparsers.add_parser('algo', help='control nni builtin tuner, assessor and advisor algorithms') # add subparsers for parser_algo parser_algo_subparsers = parser_algo.add_subparsers() - parser_algo_reg = parser_algo_subparsers.add_parser('register', help='register algorithms as nni builtin algorithm') + parser_algo_reg = parser_algo_subparsers.add_parser('register', aliases=('reg',), help='register algorithms as nni builtin algorithm') parser_algo_reg.add_argument('--meta_path', '-m', dest='meta_path', help='path to the meta file', required=True) parser_algo_reg.set_defaults(func=algo_reg) - parser_algo_unreg = parser_algo_subparsers.add_parser('unregister', help='unregister algorithm') + parser_algo_unreg = parser_algo_subparsers.add_parser('unregister', aliases=('unreg',), help='unregister algorithm') parser_algo_unreg.add_argument('name', nargs=1, help='builtin name of the algorithm') parser_algo_unreg.set_defaults(func=algo_unreg) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 4bc32da195..57e6cf8ee9 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -75,16 +75,16 @@ def get_builtin_algo_meta(algo_type=None, builtin_name=None): return None -def get_installed_package_meta(algo_type, builtin_name): +def get_registered_algo_meta(builtin_name, algo_type=None): """ Get meta information of user installed algorithms from: /config/installed_packages.yml Parameters ---------- - algo_type: str | None - can be one of 'tuners', 'assessors', 'advisors' or None builtin_name: str builtin name. + algo_type: str | None + can be one of 'tuners', 'assessors', 'advisors' or None Returns: dict | None ------- From 1d8c2f33cb7ae84ca8c8739de90bbb1890cf4987 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 01:18:23 +0800 Subject: [PATCH 16/26] updates --- nni/tools/nnictl/config_schema.py | 4 +-- nni/tools/package_utils/__init__.py | 55 +++-------------------------- 2 files changed, 6 insertions(+), 53 deletions(-) diff --git a/nni/tools/nnictl/config_schema.py b/nni/tools/nnictl/config_schema.py index aa4478b90f..29685de6ea 100644 --- a/nni/tools/nnictl/config_schema.py +++ b/nni/tools/nnictl/config_schema.py @@ -6,7 +6,7 @@ import os import netifaces from schema import Schema, And, Optional, Regex, Or, SchemaError -from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_builtin_algo_meta +from nni.tools.package_utils import create_validator_instance, get_all_builtin_names, get_registered_algo_meta from .constants import SCHEMA_TYPE_ERROR, SCHEMA_RANGE_ERROR, SCHEMA_PATH_ERROR from .common_utils import get_yml_content, print_warning @@ -75,7 +75,7 @@ def __init__(self, algo_type): def validate_class_args(self, class_args, algo_type, builtin_name): if not builtin_name or not class_args: return - meta = get_builtin_algo_meta(algo_type+'s', builtin_name) + meta = get_registered_algo_meta(builtin_name, algo_type+'s') if meta and 'acceptClassArgs' in meta and meta['acceptClassArgs'] == False: raise SchemaError('classArgs is not allowed.') diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 57e6cf8ee9..ca4d7ab5f2 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -29,55 +29,8 @@ def get_all_builtin_names(algo_type): return [x['builtinName'] for x in read_registerd_algo_meta()[algo_type]] -def get_builtin_algo_meta(algo_type=None, builtin_name=None): - """ Get meta information of builtin algorithms from: - 1. Pre-installed BuiltinAlgorithms - 2. User installed packages in /config/installed_packages.yml - - Parameters - ---------- - algo_type: str | None - can be one of 'tuners', 'assessors', 'advisors' or None - builtin_name: str | None - builtin name. - - Returns: dict | list of dict | None - ------- - If builtin_name is specified, returns meta information of speicified builtin - alogorithms, for example: - { - 'name': 'Random', - 'className': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptTuner', - 'classArgs': { - 'algorithm_name': 'random_search' - }, - 'acceptClassArgs': False, - 'classArgsValidator': 'nni.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator' - } - If builtin_name is None, returns multiple meta information in a list. - """ - algo_meta = read_registerd_algo_meta() - - if algo_type is None and builtin_name is None: - return algo_meta - - if algo_type: - assert algo_type in ALGO_TYPES - metas = algo_meta[algo_type] - else: - metas = algo_meta['tuners'] + algo_meta['assessors'] + algo_meta['advisors'] - if builtin_name: - for m in metas: - if m['builtinName'] == builtin_name: - return m - else: - return metas - - return None - def get_registered_algo_meta(builtin_name, algo_type=None): - """ Get meta information of user installed algorithms from: - /config/installed_packages.yml + """ Get meta information of registered algorithms. Parameters ---------- @@ -134,7 +87,7 @@ def get_builtin_module_class_name(algo_type, builtin_name): """ assert algo_type in ALGO_TYPES assert builtin_name is not None - meta = get_builtin_algo_meta(algo_type, builtin_name) + meta = get_registered_algo_meta(builtin_name, algo_type) if not meta: return None, None return parse_full_class_name(meta['className']) @@ -156,7 +109,7 @@ def create_validator_instance(algo_type, builtin_name): """ assert algo_type in ALGO_TYPES assert builtin_name is not None - meta = get_builtin_algo_meta(algo_type, builtin_name) + meta = get_registered_algo_meta(builtin_name, algo_type) if not meta or 'classArgsValidator' not in meta: return None module_name, class_name = parse_full_class_name(meta['classArgsValidator']) @@ -205,7 +158,7 @@ class name: HyperoptTuner return module_name, class_name, class_args - algo_meta = get_builtin_algo_meta(algo_type, builtin_name) + algo_meta = get_registered_algo_meta(builtin_name, algo_type) module_name, class_name, class_args = parse_algo_meta(algo_meta, input_class_args) if importlib.util.find_spec(module_name) is None: From 91526017fc951292d4a0fb3f10a101e77772a495 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 01:26:55 +0800 Subject: [PATCH 17/26] updates --- nni/tools/package_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index ca4d7ab5f2..7ae0eae7b0 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -12,7 +12,7 @@ ALGO_TYPES = ['tuners', 'assessors', 'advisors'] def get_all_builtin_names(algo_type): - """Get all valid builtin names registered in /config/installed_packages.yml + """Get all builtin names of registered algorithms of specified type Parameters ---------- From b06051cde6863c547708464447587487bb9fed6e Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 01:46:17 +0800 Subject: [PATCH 18/26] updates --- nni/tools/package_utils/__init__.py | 2 ++ setup.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 7ae0eae7b0..713eaf80a7 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -214,6 +214,8 @@ def get_registered_algo_config_path(): #config_dir = Path(nni.__path__[0]).parent / 'nni_config' if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') + elif sys.platform == 'win32': + nni_config_dir = os.getenv('APPDATA') else: nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): diff --git a/setup.py b/setup.py index 6b3e49d48f..2185c339ac 100644 --- a/setup.py +++ b/setup.py @@ -158,6 +158,8 @@ def _using_conda_or_virtual_environment(): def _copy_data_files(): if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') + elif sys.platform == 'win32': + nni_config_dir = os.getenv('APPDATA') else: nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): From 9f5a3b0153c65b6a46c074c7d270f854dee8c206 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Mon, 30 Nov 2020 01:51:51 +0800 Subject: [PATCH 19/26] updates --- nni/tools/package_utils/__init__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 713eaf80a7..738b053bfc 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -215,7 +215,7 @@ def get_registered_algo_config_path(): if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') elif sys.platform == 'win32': - nni_config_dir = os.getenv('APPDATA') + nni_config_dir = os.path.join(os.getenv('APPDATA'), 'nni') else: nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): diff --git a/setup.py b/setup.py index 2185c339ac..3bed569ef4 100644 --- a/setup.py +++ b/setup.py @@ -159,7 +159,7 @@ def _copy_data_files(): if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') elif sys.platform == 'win32': - nni_config_dir = os.getenv('APPDATA') + nni_config_dir = os.path.join(os.getenv('APPDATA'), 'nni') else: nni_config_dir = os.path.expanduser('~/.config/nni') if not os.path.exists(nni_config_dir): From a3d7a107314f77ac95fcce811fa6564929455c1f Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Wed, 2 Dec 2020 15:41:38 +0800 Subject: [PATCH 20/26] updates --- pipelines/fast-test.yml | 4 ++-- setup.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index bb715be393..2b3465e91c 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 setup.py develop + python3 -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: 'Install NNI' - script: | @@ -93,7 +93,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 setup.py develop + python3 -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: 'Install NNI' - script: | diff --git a/setup.py b/setup.py index 3bed569ef4..b518004d1e 100644 --- a/setup.py +++ b/setup.py @@ -108,10 +108,10 @@ def _setup(): python_requires = '>=3.6', install_requires = dependencies, extras_require = { - #'SMAC': [ - # 'git+https://github.com/QuanluZhang/ConfigSpace.git', - # 'git+https://github.com/QuanluZhang/SMAC3.git' - #], + 'SMAC': [ + 'ConfigSpace @ git+https://github.com/QuanluZhang/ConfigSpace.git', + 'SMAC3 @ git+https://github.com/QuanluZhang/SMAC3.git' + ], 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], 'PPOTuner': ['enum34', 'gym'] }, From 9e7e8d1d86a7b0e236657ed0b8f7b7fa3528ff37 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Wed, 2 Dec 2020 16:10:44 +0800 Subject: [PATCH 21/26] updates --- pipelines/fast-test.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index 2b3465e91c..d7b85a1e89 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -19,7 +19,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 -m pip install -e .[SMAC,BOHB,PPOTuner] + python3 setup.py develop displayName: 'Install NNI' - script: | @@ -40,10 +40,7 @@ jobs: python3 -m pip install -U gym onnx peewee thop python3 -m pip install -U sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx sudo apt-get install swig -y - python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git - python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git - python3 -m pip install -U ConfigSpace==0.4.7 - python3 -m pip install -U statsmodels==0.10.0 + python3 -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: 'Install extra dependencies' - script: | @@ -93,7 +90,7 @@ jobs: displayName: 'Install Python tools' - script: | - python3 -m pip install -e .[SMAC,BOHB,PPOTuner] + python3 setup.py develop displayName: 'Install NNI' - script: | @@ -103,10 +100,7 @@ jobs: python3 -m pip install -U keras==2.1.6 python3 -m pip install -U gym onnx peewee sudo apt-get install swig -y - python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git - python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git - python3 -m pip install -U ConfigSpace==0.4.7 - python3 -m pip install -U statsmodels==0.10.0 + python3 -m pip install -e .[SMAC,BOHB,PPOTuner] displayName: 'Install extra dependencies' - script: | @@ -166,8 +160,7 @@ jobs: brew install swig@3 rm -f /usr/local/bin/swig ln -s /usr/local/opt/swig\@3/bin/swig /usr/local/bin/swig - python3 -m pip install -U git+https://github.com/QuanluZhang/ConfigSpace.git - python3 -m pip install -U git+https://github.com/QuanluZhang/SMAC3.git + python3 -m pip install -e .[SMAC] displayName: 'Install extra dependencies' - script: | From 296446536f04b9e8611d03ca6c133e7f6a57f327 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Wed, 2 Dec 2020 16:23:52 +0800 Subject: [PATCH 22/26] updates --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42bfcdad90..a6ffc7bd61 100644 --- a/setup.py +++ b/setup.py @@ -110,7 +110,7 @@ def _setup(): install_requires = dependencies, extras_require = { 'SMAC': [ - 'ConfigSpace @ git+https://github.com/QuanluZhang/ConfigSpace.git', + 'ConfigSpaceNNI @ git+https://github.com/QuanluZhang/ConfigSpace.git', 'SMAC3 @ git+https://github.com/QuanluZhang/SMAC3.git' ], 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], From 6c0d5e18ad41c792f143eb8a656f24245429b3ac Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Wed, 2 Dec 2020 16:50:13 +0800 Subject: [PATCH 23/26] updates --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a6ffc7bd61..68edcab157 100644 --- a/setup.py +++ b/setup.py @@ -111,7 +111,7 @@ def _setup(): extras_require = { 'SMAC': [ 'ConfigSpaceNNI @ git+https://github.com/QuanluZhang/ConfigSpace.git', - 'SMAC3 @ git+https://github.com/QuanluZhang/SMAC3.git' + 'smac @ git+https://github.com/QuanluZhang/SMAC3.git' ], 'BOHB': ['ConfigSpace==0.4.7', 'statsmodels==0.10.0'], 'PPOTuner': ['enum34', 'gym'] From f1e62b548a9120cfbbf046ac2e3c090153086a88 Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Wed, 2 Dec 2020 23:24:27 +0800 Subject: [PATCH 24/26] updates --- docs/en_US/Tuner/BohbAdvisor.md | 2 +- docs/en_US/Tuner/BuiltinTuner.md | 12 +++++++----- docs/en_US/Tutorial/InstallationWin.md | 2 +- docs/en_US/Tutorial/Nnictl.md | 4 ++-- nni/tools/nnictl/launcher.py | 6 ++---- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/en_US/Tuner/BohbAdvisor.md b/docs/en_US/Tuner/BohbAdvisor.md index d27a9958f5..26ee4446cd 100644 --- a/docs/en_US/Tuner/BohbAdvisor.md +++ b/docs/en_US/Tuner/BohbAdvisor.md @@ -46,7 +46,7 @@ The sampling procedure (using Multidimensional KDE to guide selection) is summar BOHB advisor requires the [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command. ```bash -nnictl package install --name=BOHB +pip install nni[BOHB] ``` To use BOHB, you should add the following spec in your experiment's YAML config file: diff --git a/docs/en_US/Tuner/BuiltinTuner.md b/docs/en_US/Tuner/BuiltinTuner.md index c8ad5af398..dd2bb1a6eb 100644 --- a/docs/en_US/Tuner/BuiltinTuner.md +++ b/docs/en_US/Tuner/BuiltinTuner.md @@ -12,7 +12,7 @@ Currently, we support the following algorithms: |[__Random Search__](#Random)|In Random Search for Hyper-Parameter Optimization show that Random Search might be surprisingly simple and effective. We suggest that we could use Random Search as the baseline when we have no knowledge about the prior distribution of hyper-parameters. [Reference Paper](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf)| |[__Anneal__](#Anneal)|This simple annealing algorithm begins by sampling from the prior, but tends over time to sample from points closer and closer to the best ones observed. This algorithm is a simple variation on the random search that leverages smoothness in the response surface. The annealing rate is not adaptive.| |[__Naïve Evolution__](#Evolution)|Naïve Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naïve Evolution requires many trials to work, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)| -|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)| +|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `pip install nni[SMAC]` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)| |[__Batch tuner__](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.| |[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, randint. | |[__Hyperband__](#Hyperband)|Hyperband tries to use limited resources to explore as many configurations as possible and returns the most promising ones as a final result. The basic idea is to generate many configurations and run them for a small number of trials. The half least-promising configurations are thrown out, the remaining are further trained along with a selection of new configurations. The size of these populations is sensitive to resource constraints (e.g. allotted search time). [Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)| @@ -27,7 +27,9 @@ Currently, we support the following algorithms: Using a built-in tuner provided by the NNI SDK requires one to declare the **builtinTunerName** and **classArgs** in the `config.yml` file. In this part, we will introduce each tuner along with information about usage and suggested scenarios, classArg requirements, and an example configuration. -Note: Please follow the format when you write your `config.yml` file. Some built-in tuners need to be installed using `nnictl package`, like SMAC. +Note: Please follow the format when you write your `config.yml` file. Some built-in tuners have +dependencies need to be installed using `pip install nni[]`, like SMAC's dependencies can +be installed using `pip install nni[SMAC]`. @@ -144,10 +146,10 @@ tuner: **Installation** -SMAC needs to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`. +SMAC has dependencies need to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`. ```bash -nnictl package install --name=SMAC +pip install nni[SMAC] ``` **Suggested scenario** @@ -340,7 +342,7 @@ tuner: BOHB advisor requires [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command. ```bash -nnictl package install --name=BOHB +pip install nni[BOHB] ``` **Suggested scenario** diff --git a/docs/en_US/Tutorial/InstallationWin.md b/docs/en_US/Tutorial/InstallationWin.md index 53daccba06..cea3449614 100644 --- a/docs/en_US/Tutorial/InstallationWin.md +++ b/docs/en_US/Tutorial/InstallationWin.md @@ -123,7 +123,7 @@ If there is a stderr file, please check it. Two possible cases are: ### Fail to use BOHB on Windows -Make sure a C++ 14.0 compiler is installed when trying to run `nnictl package install --name=BOHB` to install the dependencies. +Make sure a C++ 14.0 compiler is installed when trying to run `pip install nni[BOHB]` to install the dependencies. ### Not supported tuner on Windows diff --git a/docs/en_US/Tutorial/Nnictl.md b/docs/en_US/Tutorial/Nnictl.md index a8c7616d56..45ffca6786 100644 --- a/docs/en_US/Tutorial/Nnictl.md +++ b/docs/en_US/Tutorial/Nnictl.md @@ -21,7 +21,7 @@ nnictl support commands: * [nnictl log](#log) * [nnictl webui](#webui) * [nnictl tensorboard](#tensorboard) -* [nnictl package](#package) +* [nnictl algo](#algo) * [nnictl ss_gen](#ss_gen) * [nnictl --version](#version) @@ -805,7 +805,7 @@ Debug mode will disable version check function in Trialkeeper. * Description - Show the detailed information of specified packages. + Show the detailed information of specified registered algorithms. * Usage diff --git a/nni/tools/nnictl/launcher.py b/nni/tools/nnictl/launcher.py index 0dad4b17fc..a5ccbe244a 100644 --- a/nni/tools/nnictl/launcher.py +++ b/nni/tools/nnictl/launcher.py @@ -452,10 +452,8 @@ def launch_experiment(args, experiment_config, mode, experiment_id): except CalledProcessError: print_error('some errors happen when import package %s.' %(package_name)) print_log_content(experiment_id) - #if package_name in INSTALLABLE_PACKAGE_META: - # print_error('If %s is not installed, it should be installed through '\ - # '\'nnictl package install --name %s\'' % (package_name, package_name)) - #exit(1) + if package_name in ['SMAC', 'BOHB', 'PPOTuner']: + print_error(f'The dependencies for {package_name} can be installed through pip install nni[{package_name}]') raise log_dir = experiment_config['logDir'] if experiment_config.get('logDir') else None log_level = experiment_config['logLevel'] if experiment_config.get('logLevel') else None From a3960c20dd5c5dfa70931da6b4cf33d6339e0aed Mon Sep 17 00:00:00 2001 From: chicm-ms Date: Thu, 3 Dec 2020 00:04:14 +0800 Subject: [PATCH 25/26] updates --- azure-pipelines.yml | 6 ++---- pipelines/fast-test.yml | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8a20e51b74..6eab27e09a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -34,8 +34,7 @@ jobs: python3 -m pip install gym onnx peewee thop --user python3 -m pip install sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx --user sudo apt-get install swig -y - nnictl package install --name=SMAC - nnictl package install --name=BOHB + python3 -m pip install -e .[SMAC,BOHB] displayName: 'Install dependencies' - script: | cd test @@ -73,8 +72,7 @@ jobs: python3 -m pip install keras==2.1.6 --user python3 -m pip install gym onnx peewee --user sudo apt-get install swig -y - nnictl package install --name=SMAC - nnictl package install --name=BOHB + python3 -m pip install -e .[SMAC,BOHB] displayName: 'Install dependencies' - script: | set -e diff --git a/pipelines/fast-test.yml b/pipelines/fast-test.yml index d7b85a1e89..17b05129a0 100644 --- a/pipelines/fast-test.yml +++ b/pipelines/fast-test.yml @@ -40,7 +40,7 @@ jobs: python3 -m pip install -U gym onnx peewee thop python3 -m pip install -U sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx sudo apt-get install swig -y - python3 -m pip install -e .[SMAC,BOHB,PPOTuner] + python3 -m pip install -e .[SMAC,BOHB] displayName: 'Install extra dependencies' - script: | @@ -100,7 +100,7 @@ jobs: python3 -m pip install -U keras==2.1.6 python3 -m pip install -U gym onnx peewee sudo apt-get install swig -y - python3 -m pip install -e .[SMAC,BOHB,PPOTuner] + python3 -m pip install -e .[SMAC,BOHB] displayName: 'Install extra dependencies' - script: | From 2437a106bd46484c03c46cfd9c6c426ec7a08831 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Fri, 11 Dec 2020 16:16:43 +0800 Subject: [PATCH 26/26] add message for nnictl package --- nni/tools/nnictl/nnictl.py | 21 ++++++++++++++++++++- nni/tools/package_utils/__init__.py | 6 ++++-- setup.py | 3 +++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/nni/tools/nnictl/nnictl.py b/nni/tools/nnictl/nnictl.py index 3fca4c0e5e..82340c0543 100644 --- a/nni/tools/nnictl/nnictl.py +++ b/nni/tools/nnictl/nnictl.py @@ -216,7 +216,15 @@ def parse_args(): parser_algo = subparsers.add_parser('algo', help='control nni builtin tuner, assessor and advisor algorithms') # add subparsers for parser_algo parser_algo_subparsers = parser_algo.add_subparsers() - parser_algo_reg = parser_algo_subparsers.add_parser('register', aliases=('reg',), help='register algorithms as nni builtin algorithm') + parser_algo_reg = parser_algo_subparsers.add_parser( + 'register', + aliases=('reg',), + help='''register algorithms as nni builtin algorithm, for example: + nnictl reg --meta_path + where is the path to a meta data in yml format, + reference the nni document and examples/tuners/customized_tuner example + for the format of the yml file.''' + ) parser_algo_reg.add_argument('--meta_path', '-m', dest='meta_path', help='path to the meta file', required=True) parser_algo_reg.set_defaults(func=algo_reg) @@ -231,6 +239,17 @@ def parse_args(): parser_algo_list = parser_algo_subparsers.add_parser('list', help='list registered algorithms') parser_algo_list.set_defaults(func=algo_list) + # To show message that nnictl package command is replaced by nnictl algo, to be remove in the future release. + def show_messsage_for_nnictl_package(args): + print_error('nnictl package command is replaced by nnictl algo, please run nnictl algo -h to show the usage') + + parser_package_subparsers = subparsers.add_parser('package', help='control nni tuner and assessor packages').add_subparsers() + parser_package_subparsers.add_parser('install', help='install packages').set_defaults(func=show_messsage_for_nnictl_package) + parser_package_subparsers.add_parser('uninstall', help='uninstall packages').set_defaults(func=show_messsage_for_nnictl_package) + parser_package_subparsers.add_parser('show', help='show the information of packages').set_defaults( + func=show_messsage_for_nnictl_package) + parser_package_subparsers.add_parser('list', help='list installed packages').set_defaults(func=show_messsage_for_nnictl_package) + #parse tensorboard command parser_tensorboard = subparsers.add_parser('tensorboard', help='manage tensorboard') parser_tensorboard_subparsers = parser_tensorboard.add_subparsers() diff --git a/nni/tools/package_utils/__init__.py b/nni/tools/package_utils/__init__.py index 738b053bfc..64702b5d4d 100644 --- a/nni/tools/package_utils/__init__.py +++ b/nni/tools/package_utils/__init__.py @@ -210,8 +210,10 @@ def _using_conda_or_virtual_environment(): return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) def get_registered_algo_config_path(): - # FIXME: this might not be the desired location - #config_dir = Path(nni.__path__[0]).parent / 'nni_config' + # Find the path for registered_algorithms.yml for this nni installation, + # the registered_algorithms.yml is copied into this location in setup.py, + # so we need to ensure that we use the same logic as setup.py to find the location. + if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') elif sys.platform == 'win32': diff --git a/setup.py b/setup.py index fb1bbdd6ac..49b914d00b 100644 --- a/setup.py +++ b/setup.py @@ -166,6 +166,9 @@ def _using_conda_or_virtual_environment(): return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta')) def _copy_data_files(): + # after installation, nni needs to find this location in nni.tools.package_utils.get_registered_algo_config_path + # since we can not import nni here, we need to ensure get_registered_algo_config_path use the same + # logic here to retrieve registered_algorithms.yml if _using_conda_or_virtual_environment(): nni_config_dir = os.path.join(sys.prefix, 'nni') elif sys.platform == 'win32':