diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index f5a1d34d4f..ee14b4546f 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -72,6 +72,11 @@ All types of sampling strategies and their parameter are listed here: * Which means the variable value is a value like round(exp(normal(mu, sigma)) / q) * q * Suitable for a discrete variable with respect to which the objective is smooth and gets smoother with the size of the variable, which is bounded from one side. +* {"_type":"mutable_layer","_value":{mutable_layer_infomation}} + * Type for [Neural Architecture Search Space][1]. Value is also a dictionary, which contains key-value pairs representing respectively name and search space of each mutable_layer. + * For now, users can only use this type of search space with annotation, which means that there is no need to define a json file for search space since it will be automatically generated according to the annotation in trial code. + * For detailed usage, please refer to [General NAS Interfaces][1]. + ## Search Space Types Supported by Each Tuner | | choice | randint | uniform | quniform | loguniform | qloguniform | normal | qnormal | lognormal | qlognormal | @@ -103,3 +108,5 @@ Known Limitations: * Only Random Search/TPE/Anneal/Evolution tuner supports nested search space * We do not support nested search space "Hyper Parameter" in visualization now, the enhancement is being considered in #1110(https://github.com/microsoft/nni/issues/1110), any suggestions or discussions or contributions are warmly welcomed + +[1]: ../AdvancedFeature/GeneralNasInterfaces.md \ No newline at end of file diff --git a/examples/tuners/random_nas_tuner/random_nas_tuner.py b/examples/tuners/random_nas_tuner/random_nas_tuner.py index f5e500120c..d7f6214aa6 100644 --- a/examples/tuners/random_nas_tuner/random_nas_tuner.py +++ b/examples/tuners/random_nas_tuner/random_nas_tuner.py @@ -7,7 +7,9 @@ def random_archi_generator(nas_ss, random_state): ''' chosen_archi = {} print("zql: nas search space: ", nas_ss) - for block_name, block in nas_ss.items(): + for block_name, block_value in nas_ss.items(): + assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space whose _type is 'mutable_layer'" + block = block_value['_value'] tmp_block = {} for layer_name, layer in block.items(): tmp_layer = {} diff --git a/tools/nni_annotation/search_space_generator.py b/tools/nni_annotation/search_space_generator.py index ff2c32d203..04bca78584 100644 --- a/tools/nni_annotation/search_space_generator.py +++ b/tools/nni_annotation/search_space_generator.py @@ -57,8 +57,8 @@ def generate_mutable_layer_search_space(self, args): key = self.module_name + '/' + mutable_block args[0].s = key if key not in self.search_space: - self.search_space[key] = dict() - self.search_space[key][mutable_layer] = { + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { 'layer_choice': [k.s for k in args[2].keys], 'optional_inputs': [k.s for k in args[5].keys], 'optional_input_size': args[6].n if isinstance(args[6], ast.Num) else [args[6].elts[0].n, args[6].elts[1].n] diff --git a/tools/nni_annotation/test_annotation.py b/tools/nni_annotation/test_annotation.py index 0ddb53b7ab..7550ba2466 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/tools/nni_annotation/test_annotation.py @@ -44,8 +44,9 @@ def test_search_space_generator(self): self.assertEqual(search_space, json.load(f)) def test_code_generator(self): - code_dir = expand_annotations('testcase/usercode', '_generated') + code_dir = expand_annotations('testcase/usercode', '_generated', nas_mode='classic_mode') self.assertEqual(code_dir, '_generated') + self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py') self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py') self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py') with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst: diff --git a/tools/nni_annotation/testcase/annotated/nas.py b/tools/nni_annotation/testcase/annotated/nas.py new file mode 100644 index 0000000000..227f7960b2 --- /dev/null +++ b/tools/nni_annotation/testcase/annotated/nas.py @@ -0,0 +1,49 @@ +import nni +import time + + +def add_one(inputs): + return inputs + 1 + + +def add_two(inputs): + return inputs + 2 + + +def add_three(inputs): + return inputs + 3 + + +def add_four(inputs): + return inputs + 4 + + +def main(): + images = 5 + layer_1_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_0', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'images': images}, 1, + 'classic_mode') + layer_2_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_1', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': + layer_1_out}, 1, 'classic_mode') + layer_3_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_2', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': + layer_1_out, 'layer_2_out': layer_2_out}, 1, 'classic_mode') + nni.report_intermediate_result(layer_1_out) + time.sleep(2) + nni.report_intermediate_result(layer_2_out) + time.sleep(2) + nni.report_intermediate_result(layer_3_out) + time.sleep(2) + layer_3_out = layer_3_out + 10 + nni.report_final_result(layer_3_out) + + +if __name__ == '__main__': + main() diff --git a/tools/nni_annotation/testcase/searchspace.json b/tools/nni_annotation/testcase/searchspace.json index 0a7f4279c8..0ba6df6cfc 100644 --- a/tools/nni_annotation/testcase/searchspace.json +++ b/tools/nni_annotation/testcase/searchspace.json @@ -143,5 +143,47 @@ "(2 * 3 + 4)", "(lambda x: 1 + x)" ] + }, + "nas/mutable_block_39": { + "_type": "mutable_layer", + "_value": { + "mutable_layer_0": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "images" + ], + "optional_input_size": 1 + }, + "mutable_layer_1": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "layer_1_out" + ], + "optional_input_size": 1 + }, + "mutable_layer_2": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "layer_1_out", + "layer_2_out" + ], + "optional_input_size": 1 + } + } } } \ No newline at end of file diff --git a/tools/nni_annotation/testcase/mutable_layer_usercode/simple.py b/tools/nni_annotation/testcase/usercode/nas.py similarity index 100% rename from tools/nni_annotation/testcase/mutable_layer_usercode/simple.py rename to tools/nni_annotation/testcase/usercode/nas.py