Refactor builtin/customized tuner installation (#3134)

azure-pipelines.yml

@@ -34,8 +34,7 @@ jobs:
       python3 -m pip install gym onnx peewee thop --user
       python3 -m pip install sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 nbsphinx --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
+      python3 -m pip install -e .[SMAC,BOHB]
     displayName: 'Install dependencies'
   - script: |
       cd test
@@ -73,8 +72,7 @@ jobs:
       python3 -m pip install keras==2.1.6 --user
       python3 -m pip install gym onnx peewee --user
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
+      python3 -m pip install -e .[SMAC,BOHB]
     displayName: 'Install dependencies'
   - script: |
       set -e

deployment/registered_algorithms.yml

@@ -0,0 +1,78 @@
+advisors:
+- builtinName: Hyperband
+  classArgsValidator: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.HyperbandClassArgsValidator
+  className: nni.algorithms.hpo.hyperband_advisor.hyperband_advisor.Hyperband
+  source: nni
+- builtinName: BOHB
+  classArgsValidator: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHBClassArgsValidator
+  className: nni.algorithms.hpo.bohb_advisor.bohb_advisor.BOHB
+  source: nni
+assessors:
+- builtinName: Medianstop
+  classArgsValidator: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopClassArgsValidator
+  className: nni.algorithms.hpo.medianstop_assessor.medianstop_assessor.MedianstopAssessor
+  source: nni
+- builtinName: Curvefitting
+  classArgsValidator: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingClassArgsValidator
+  className: nni.algorithms.hpo.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor
+  source: nni
+tuners:
+- builtinName: PPOTuner
+  classArgsValidator: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOClassArgsValidator
+  className: nni.algorithms.hpo.ppo_tuner.ppo_tuner.PPOTuner
+  source: nni
+- builtinName: SMAC
+  classArgsValidator: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACClassArgsValidator
+  className: nni.algorithms.hpo.smac_tuner.smac_tuner.SMACTuner
+  source: nni
+- builtinName: TPE
+  classArgs:
+    algorithm_name: tpe
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: Random
+  classArgs:
+    algorithm_name: random_search
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- builtinName: Anneal
+  classArgs:
+    algorithm_name: anneal
+  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptClassArgsValidator
+  className: nni.algorithms.hpo.hyperopt_tuner.hyperopt_tuner.HyperoptTuner
+  source: nni
+- builtinName: Evolution
+  classArgsValidator: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionClassArgsValidator
+  className: nni.algorithms.hpo.evolution_tuner.evolution_tuner.EvolutionTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: BatchTuner
+  className: nni.algorithms.hpo.batch_tuner.batch_tuner.BatchTuner
+  source: nni
+- acceptClassArgs: false
+  builtinName: GridSearch
+  className: nni.algorithms.hpo.gridsearch_tuner.gridsearch_tuner.GridSearchTuner
+  source: nni
+- builtinName: NetworkMorphism
+  classArgsValidator: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismClassArgsValidator
+  className: nni.algorithms.hpo.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner
+  source: nni
+- builtinName: MetisTuner
+  classArgsValidator: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisClassArgsValidator
+  className: nni.algorithms.hpo.metis_tuner.metis_tuner.MetisTuner
+  source: nni
+- builtinName: GPTuner
+  classArgsValidator: nni.algorithms.hpo.gp_tuner.gp_tuner.GPClassArgsValidator
+  className: nni.algorithms.hpo.gp_tuner.gp_tuner.GPTuner
+  source: nni
+- builtinName: PBTTuner
+  classArgsValidator: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTClassArgsValidator
+  className: nni.algorithms.hpo.pbt_tuner.pbt_tuner.PBTTuner
+  source: nni
+- builtinName: RegularizedEvolutionTuner
+  classArgsValidator: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.EvolutionClassArgsValidator
+  className: nni.algorithms.hpo.regularized_evolution_tuner.regularized_evolution_tuner.RegularizedEvolutionTuner
+  source: nni

docs/archive_en_US/Tuner/BohbAdvisor.md

@@ -46,7 +46,7 @@ The sampling procedure (using Multidimensional KDE to guide selection) is summar
 BOHB advisor requires the [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command.
 
 ```bash
-nnictl package install --name=BOHB
+pip install nni[BOHB]
 ```
 
 To use BOHB, you should add the following spec in your experiment's YAML config file:

docs/archive_en_US/Tuner/BuiltinTuner.md

@@ -12,7 +12,7 @@ Currently, we support the following algorithms:
 |[__Random Search__](#Random)|In Random Search for Hyper-Parameter Optimization show that Random Search might be surprisingly simple and effective. We suggest that we could use Random Search as the baseline when we have no knowledge about the prior distribution of hyper-parameters. [Reference Paper](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf)|
 |[__Anneal__](#Anneal)|This simple annealing algorithm begins by sampling from the prior, but tends over time to sample from points closer and closer to the best ones observed. This algorithm is a simple variation on the random search that leverages smoothness in the response surface. The annealing rate is not adaptive.|
 |[__Naïve Evolution__](#Evolution)|Naïve Evolution comes from Large-Scale Evolution of Image Classifiers. It randomly initializes a population-based on search space. For each generation, it chooses better ones and does some mutation (e.g., change a hyperparameter, add/remove one layer) on them to get the next generation. Naïve Evolution requires many trials to work, but it's very simple and easy to expand new features. [Reference paper](https://arxiv.org/pdf/1703.01041.pdf)|
-|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `nnictl package` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)|
+|[__SMAC__](#SMAC)|SMAC is based on Sequential Model-Based Optimization (SMBO). It adapts the most prominent previously used model class (Gaussian stochastic process models) and introduces the model class of random forests to SMBO, in order to handle categorical parameters. The SMAC supported by NNI is a wrapper on the SMAC3 GitHub repo. Notice, SMAC needs to be installed by `pip install nni[SMAC]` command. [Reference Paper,](https://www.cs.ubc.ca/~hutter/papers/10-TR-SMAC.pdf) [GitHub Repo](https://github.com/automl/SMAC3)|
 |[__Batch tuner__](#Batch)|Batch tuner allows users to simply provide several configurations (i.e., choices of hyper-parameters) for their trial code. After finishing all the configurations, the experiment is done. Batch tuner only supports the type choice in search space spec.|
 |[__Grid Search__](#GridSearch)|Grid Search performs an exhaustive searching through a manually specified subset of the hyperparameter space defined in the searchspace file. Note that the only acceptable types of search space are choice, quniform, randint. |
 |[__Hyperband__](#Hyperband)|Hyperband tries to use limited resources to explore as many configurations as possible and returns the most promising ones as a final result. The basic idea is to generate many configurations and run them for a small number of trials. The half least-promising configurations are thrown out, the remaining are further trained along with a selection of new configurations. The size of these populations is sensitive to resource constraints (e.g. allotted search time). [Reference Paper](https://arxiv.org/pdf/1603.06560.pdf)|
@@ -27,7 +27,9 @@ Currently, we support the following algorithms:
 
 Using a built-in tuner provided by the NNI SDK requires one to declare the  **builtinTunerName** and **classArgs** in the `config.yml` file. In this part, we will introduce each tuner along with information about usage and suggested scenarios, classArg requirements, and an example configuration.
 
-Note: Please follow the format when you write your `config.yml` file. Some built-in tuners need to be installed using `nnictl package`, like SMAC.
+Note: Please follow the format when you write your `config.yml` file. Some built-in tuners have 
+dependencies need to be installed using `pip install nni[<tuner>]`, like SMAC's dependencies can
+be installed using `pip install nni[SMAC]`.
 
 <a name="TPE"></a>
 
@@ -144,10 +146,10 @@ tuner:
 
 **Installation**
 
-SMAC needs to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`.
+SMAC has dependencies need to be installed by following command before the first usage. As a reminder, `swig` is required for SMAC: for Ubuntu `swig` can be installed with `apt`.
 
 ```bash
-nnictl package install --name=SMAC
+pip install nni[SMAC]
 ```
 
 **Suggested scenario**
@@ -340,7 +342,7 @@ tuner:
 BOHB advisor requires [ConfigSpace](https://github.com/automl/ConfigSpace) package. ConfigSpace can be installed using the following command.
 
 ```bash
-nnictl package install --name=BOHB
+pip install nni[BOHB]
 ```
 
 **Suggested scenario**

docs/archive_en_US/Tuner/InstallCustomizedTuner.md

@@ -1,22 +1,19 @@
-# How to install customized tuner as a builtin tuner
+# How to register a customized tuner as a builtin tuner
 
-You can following below steps to install a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner.
+You can following below steps to register a customized tuner in `nni/examples/tuners/customized_tuner` as a builtin tuner.
 
-## Prepare installation source and install package
+## Install the customized tuner package into python environment
 
-There are 2 options to install this customized tuner:
+There are 2 options to install the package into python environment:
 
 ### Option 1: install from directory
 
-Step 1: From `nni/examples/tuners/customized_tuner` directory, run:
+From `nni/examples/tuners/customized_tuner` directory, run:
 
 `python setup.py develop`
 
 This command will build the `nni/examples/tuners/customized_tuner` directory as a pip installation source.
 
-Step 2: Run command:
-
-`nnictl package install ./`
 
 ### Option 2: install from whl file
 
@@ -28,16 +25,22 @@ This command build a whl file which is a pip installation source.
 
 Step 2: Run command:
 
-`nnictl package install dist/demo_tuner-0.1-py3-none-any.whl`
+`pip install dist/demo_tuner-0.1-py3-none-any.whl`
+
+## Register the customized tuner as builtin tuner:
+
+Run following command:
+
+`nnictl algo register --meta meta_file.yml`
 
-## Check the installed package
+## Check the registered builtin algorithms
 
-Then run command `nnictl package list`, you should be able to see that demotuner is installed:
+Then run command `nnictl algo list`, you should be able to see that demotuner is installed:
 ```
 +-----------------+------------+-----------+--------=-------------+------------------------------------------+
-|      Name       |    Type    | Installed |      Class Name      |               Module Name                |
+|      Name       |    Type    |  source   |      Class Name      |               Module Name                |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
-| demotuner       | tuners     | Yes       | DemoTuner            | demo_tuner                               |
+| demotuner       | tuners     |  user     | DemoTuner            | demo_tuner                               |
 +-----------------+------------+-----------+----------------------+------------------------------------------+
 ```