Nested search space refinement

docs/en_US/SearchSpaceSpec.md

@@ -6,7 +6,7 @@ In NNI, tuner will sample parameters/architecture according to the search space,
 
 To define a search space, users should define the name of variable, the type of sampling strategy and its parameters.
 
-* A example of search space definition as follow:
+* An example of search space definition as follow:
 
 ```yaml
 {
@@ -26,9 +26,19 @@ Take the first line as an example. `dropout_rate` is defined as a variable whose
 All types of sampling strategies and their parameter are listed here:
 
 * {"_type":"choice","_value":options}
-  * Which means the variable value is one of the options, which should be a list. The elements of options can themselves be [nested] stochastic expressions. In this case, the stochastic choices that only appear in some of the options become conditional parameters.
+
+  * Which means the variable value is one of the options, which should be a list. The elements of options can themselves be **nested** stochastic expressions. In this case, the stochastic choices that only appear in some of the options become conditional parameters.
+
+  * An simple [example](../../examples/trials/mnist-cascading-search-space/search_space.json) of [nested] search space definition. The elements of options in the list must be dictionary with one key `_name` and its value pair. Here is a [sample]((../../examples/trials/mnist-cascading-search-space/sample.json) ) which users can get from nni with **nested** search space definition. Tuners which support this feature is as follows:
+
+    - Random Search 
+    - TPE
+
+    - Anneal
+    - Evolution
 
 * {"_type":"randint","_value":[upper]}
+
   * Which means the variable value is a random integer in the range [0, upper). The semantics of this distribution is that there is no more correlation in the loss function between nearby integer values, as compared with more distant integer values. This is an appropriate distribution for describing random seeds for example. If the loss function is probably more correlated for nearby integer values, then you should probably use one of the "quantized" continuous distributions, such as either quniform, qloguniform, qnormal or qlognormal. Note that if you want to change lower bound, you can use `quniform` for now.
 
 * {"_type":"uniform","_value":[low, high]}
@@ -48,13 +58,15 @@ All types of sampling strategies and their parameter are listed here:
   * Suitable for a discrete variable with respect to which the objective is "smooth" and gets smoother with the size of the value, but which should be bounded both above and below.
 
 * {"_type":"normal","_value":[mu, sigma]}
+
   * Which means the variable value is a real value that's normally-distributed with mean mu and standard deviation sigma. When optimizing, this is an unconstrained variable.
 
 * {"_type":"qnormal","_value":[mu, sigma, q]}
   * Which means the variable value is a value like round(normal(mu, sigma) / q) * q
   * Suitable for a discrete variable that probably takes a value around mu, but is fundamentally unbounded.
 
 * {"_type":"lognormal","_value":[mu, sigma]}
+
   * Which means the variable value is a value drawn according to exp(normal(mu, sigma)) so that the logarithm of the return value is normally distributed. When optimizing, this variable is constrained to be positive.
 
 * {"_type":"qlognormal","_value":[mu, sigma, q]}

examples/trials/mnist-cascading-search-space/mnist.py

@@ -131,21 +131,29 @@ def main(params):
 
         nni.report_final_result(test_acc)
 
-def generate_defualt_params():
-    params = {'data_dir': '/tmp/tensorflow/mnist/input_data',
-              'batch_num': 1000,
-              'batch_size': 200}
-    return params
-
+def get_params():
+    ''' Get parameters from command line '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--data_dir", type=str, default='/tmp/tensorflow/mnist/input_data', help="data directory")
+    parser.add_argument("--batch_num", type=int, default=1000)
+    parser.add_argument("--batch_size", type=int, default=200)
+    args, _ = parser.parse_known_args()
+    return args
 
 def parse_init_json(data):
     params = {}
     for key in data:
         value = data[key]
-        if value == 'Empty':
+        layer_name = value["_name"]
+        if layer_name == 'Empty':
+            # Empty Layer
             params[key] = ['Empty']
+        elif layer_name == 'Conv':
+            # Conv layer
+            params[key] = [layer_name, value['kernel_size'], value['kernel_size']]
         else:
-            params[key] = [value[0], value[1], value[1]]
+            # Pooling Layer
+            params[key] = [layer_name, value['pooling_size'], value['pooling_size']]
     return params
 
 
@@ -157,7 +165,7 @@ def parse_init_json(data):
 
         RCV_PARAMS = parse_init_json(data)
         logger.debug(RCV_PARAMS)
-        params = generate_defualt_params()
+        params = vars(get_params())
         params.update(RCV_PARAMS)
         print(RCV_PARAMS)
 

examples/trials/mnist-cascading-search-space/sample.json

@@ -1,12 +1,17 @@
 {
-    "layer2": "Empty", 
-    "layer8": ["Conv",  2], 
-    "layer3": ["Avg_pool", 5], 
-    "layer0": ["Max_pool", 5], 
-    "layer1": ["Conv", 2], 
-    "layer6": ["Max_pool", 3], 
-    "layer7": ["Max_pool", 5], 
-    "layer9": ["Conv", 2], 
-    "layer4": ["Avg_pool", 3], 
-    "layer5": ["Avg_pool", 5]
-}
+    "layer0": {
+        "_name": "Avg_pool",
+        "pooling_size": 3
+    },
+    "layer1": {
+        "_name": "Conv",
+        "kernel_size": 2
+    },
+    "layer2": {
+        "_name": "Empty"
+    },
+    "layer3": {
+        "_name": "Conv",
+        "kernel_size": 5
+    }
+}

examples/trials/mnist-cascading-search-space/search_space.json

@@ -1,62 +1,114 @@
 {
-    "layer0":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer1":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer2":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer3":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer4":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer5":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer6":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer7":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer8":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]},
-   "layer9":{"_type":"choice","_value":[
-		"Empty",
-		["Conv", {"_type":"choice","_value":[2,3,5]}],
-		["Max_pool", {"_type":"choice","_value":[2,3,5]}],
-		["Avg_pool", {"_type":"choice","_value":[2,3,5]}]
-	]}
+	"layer0": {
+		"_type": "choice",
+		"_value": [{
+				"_name": "Empty"
+			},
+			{
+				"_name": "Conv",
+				"kernel_size": {
+					"_type": "choice",
+					"_value": [1, 2, 3, 5]
+				}
+			},
+			{
+				"_name": "Max_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			},
+			{
+				"_name": "Avg_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			}
+		]
+	},
+	"layer1": {
+		"_type": "choice",
+		"_value": [{
+				"_name": "Empty"
+			},
+			{
+				"_name": "Conv",
+				"kernel_size": {
+					"_type": "choice",
+					"_value": [1, 2, 3, 5]
+				}
+			},
+			{
+				"_name": "Max_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			},
+			{
+				"_name": "Avg_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			}
+		]
+	},
+	"layer2": {
+		"_type": "choice",
+		"_value": [{
+				"_name": "Empty"
+			},
+			{
+				"_name": "Conv",
+				"kernel_size": {
+					"_type": "choice",
+					"_value": [1, 2, 3, 5]
+				}
+			},
+			{
+				"_name": "Max_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			},
+			{
+				"_name": "Avg_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			}
+		]
+	},
+	"layer3": {
+		"_type": "choice",
+		"_value": [{
+				"_name": "Empty"
+			},
+			{
+				"_name": "Conv",
+				"kernel_size": {
+					"_type": "choice",
+					"_value": [1, 2, 3, 5]
+				}
+			},
+			{
+				"_name": "Max_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			},
+			{
+				"_name": "Avg_pool",
+				"pooling_size": {
+					"_type": "choice",
+					"_value": [2, 3, 5]
+				}
+			}
+		]
+	}
 }

src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py

@@ -21,7 +21,6 @@
 bohb_advisor.py
 '''
 
-from enum import Enum, unique
 import sys
 import math
 import logging
@@ -32,7 +31,7 @@
 
 from nni.protocol import CommandType, send
 from nni.msg_dispatcher_base import MsgDispatcherBase
-from nni.utils import extract_scalar_reward
+from nni.utils import OptimizeMode, extract_scalar_reward
 
 from .config_generator import CG_BOHB
 
@@ -42,12 +41,6 @@
 _KEY = 'TRIAL_BUDGET'
 _epsilon = 1e-6
 
-@unique
-class OptimizeMode(Enum):
-    """Optimize Mode class"""
-    Minimize = 'minimize'
-    Maximize = 'maximize'
-
 
 def create_parameter_id():
     """Create an id