From af312387c6b03d248515257f4016f8b852dbde59 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 11:59:07 +0800 Subject: [PATCH 1/9] unify NAS search space --- tools/nni_annotation/search_space_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nni_annotation/search_space_generator.py b/tools/nni_annotation/search_space_generator.py index ff2c32d203..04bca78584 100644 --- a/tools/nni_annotation/search_space_generator.py +++ b/tools/nni_annotation/search_space_generator.py @@ -57,8 +57,8 @@ def generate_mutable_layer_search_space(self, args): key = self.module_name + '/' + mutable_block args[0].s = key if key not in self.search_space: - self.search_space[key] = dict() - self.search_space[key][mutable_layer] = { + self.search_space[key] = {'_type': 'mutable_layer', '_value': {}} + self.search_space[key]['_value'][mutable_layer] = { 'layer_choice': [k.s for k in args[2].keys], 'optional_inputs': [k.s for k in args[5].keys], 'optional_input_size': args[6].n if isinstance(args[6], ast.Num) else [args[6].elts[0].n, args[6].elts[1].n] From b9ec5232d5a72fb8d9ce78e0dc06af37e3b8801d Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 15:18:38 +0800 Subject: [PATCH 2/9] update annotation ut --- .../mutable_layer_usercode/simple.py | 0 .../nni_annotation/testcase/searchspace.json | 42 +++++++++++++++++++ 2 files changed, 42 insertions(+) rename tools/nni_annotation/testcase/{ => annotated}/mutable_layer_usercode/simple.py (100%) diff --git a/tools/nni_annotation/testcase/mutable_layer_usercode/simple.py b/tools/nni_annotation/testcase/annotated/mutable_layer_usercode/simple.py similarity index 100% rename from tools/nni_annotation/testcase/mutable_layer_usercode/simple.py rename to tools/nni_annotation/testcase/annotated/mutable_layer_usercode/simple.py diff --git a/tools/nni_annotation/testcase/searchspace.json b/tools/nni_annotation/testcase/searchspace.json index 0a7f4279c8..8b9944d5b8 100644 --- a/tools/nni_annotation/testcase/searchspace.json +++ b/tools/nni_annotation/testcase/searchspace.json @@ -143,5 +143,47 @@ "(2 * 3 + 4)", "(lambda x: 1 + x)" ] + }, + "mutable_layer_usercode.simple/mutable_block_39": { + "_type": "mutable_layer", + "_value": { + "mutable_layer_0": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "images" + ], + "optional_input_size": 1 + }, + "mutable_layer_1": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "layer_1_out" + ], + "optional_input_size": 1 + }, + "mutable_layer_2": { + "layer_choice": [ + "add_one()", + "add_two()", + "add_three()", + "add_four()" + ], + "optional_inputs": [ + "layer_1_out", + "layer_2_out" + ], + "optional_input_size": 1 + } + } } } \ No newline at end of file From de04b44fcaf972ea9ef5f568c65186af38963858 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 15:37:14 +0800 Subject: [PATCH 3/9] update annotation ut --- tools/nni_annotation/test_annotation.py | 1 + .../nni_annotation/testcase/annotated/nas.py | 44 +++++++++++++++++++ .../simple.py => usercode/nas.py} | 0 3 files changed, 45 insertions(+) create mode 100644 tools/nni_annotation/testcase/annotated/nas.py rename tools/nni_annotation/testcase/{annotated/mutable_layer_usercode/simple.py => usercode/nas.py} (100%) diff --git a/tools/nni_annotation/test_annotation.py b/tools/nni_annotation/test_annotation.py index 0ddb53b7ab..ab15729813 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/tools/nni_annotation/test_annotation.py @@ -46,6 +46,7 @@ def test_search_space_generator(self): def test_code_generator(self): code_dir = expand_annotations('testcase/usercode', '_generated') self.assertEqual(code_dir, '_generated') + self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py') self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py') self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py') with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst: diff --git a/tools/nni_annotation/testcase/annotated/nas.py b/tools/nni_annotation/testcase/annotated/nas.py new file mode 100644 index 0000000000..4cacd33389 --- /dev/null +++ b/tools/nni_annotation/testcase/annotated/nas.py @@ -0,0 +1,44 @@ +import nni +import time + +def add_one(inputs): + return inputs + 1 + +def add_two(inputs): + return inputs + 2 + +def add_three(inputs): + return inputs + 3 + +def add_four(inputs): + return inputs + 4 + +def main(): + images = 5 + layer_1_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_0', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'images': images}, 1, + None) + layer_2_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_1', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': + layer_1_out}, 1, None) + layer_3_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_2', + {'add_one()': add_one, 'add_two()': add_two, 'add_three()': + add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': + {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': + layer_1_out, 'layer_2_out': layer_2_out}, 1, None) + nni.report_intermediate_result(layer_1_out) + time.sleep(2) + nni.report_intermediate_result(layer_2_out) + time.sleep(2) + nni.report_intermediate_result(layer_3_out) + time.sleep(2) + layer_3_out = layer_3_out + 10 + nni.report_final_result(layer_3_out) + + +if __name__ == '__main__': + main() diff --git a/tools/nni_annotation/testcase/annotated/mutable_layer_usercode/simple.py b/tools/nni_annotation/testcase/usercode/nas.py similarity index 100% rename from tools/nni_annotation/testcase/annotated/mutable_layer_usercode/simple.py rename to tools/nni_annotation/testcase/usercode/nas.py From b0389a166af4c8d87dc87d84c2c7644f80be71ec Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 15:50:36 +0800 Subject: [PATCH 4/9] update annotation ut --- tools/nni_annotation/testcase/searchspace.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nni_annotation/testcase/searchspace.json b/tools/nni_annotation/testcase/searchspace.json index 8b9944d5b8..0ba6df6cfc 100644 --- a/tools/nni_annotation/testcase/searchspace.json +++ b/tools/nni_annotation/testcase/searchspace.json @@ -144,7 +144,7 @@ "(lambda x: 1 + x)" ] }, - "mutable_layer_usercode.simple/mutable_block_39": { + "nas/mutable_block_39": { "_type": "mutable_layer", "_value": { "mutable_layer_0": { From a90a6f8a436f276eaf31f21b893042d1ac7d9900 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 15:56:18 +0800 Subject: [PATCH 5/9] update random nas tuner --- examples/tuners/random_nas_tuner/random_nas_tuner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/tuners/random_nas_tuner/random_nas_tuner.py b/examples/tuners/random_nas_tuner/random_nas_tuner.py index f5e500120c..1356720f59 100644 --- a/examples/tuners/random_nas_tuner/random_nas_tuner.py +++ b/examples/tuners/random_nas_tuner/random_nas_tuner.py @@ -7,7 +7,9 @@ def random_archi_generator(nas_ss, random_state): ''' chosen_archi = {} print("zql: nas search space: ", nas_ss) - for block_name, block in nas_ss.items(): + for block_name, block_value in nas_ss.items(): + assert block_value['_type'] is "mutable_layer", "Random NAS Tuner only receives NAS search space" + block = block_value['_value'] tmp_block = {} for layer_name, layer in block.items(): tmp_layer = {} From e2a5a4794e7c81638b834a22bd420ab1743b8114 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 16:15:52 +0800 Subject: [PATCH 6/9] update random nas tuner --- examples/tuners/random_nas_tuner/random_nas_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tuners/random_nas_tuner/random_nas_tuner.py b/examples/tuners/random_nas_tuner/random_nas_tuner.py index 1356720f59..a22f477c98 100644 --- a/examples/tuners/random_nas_tuner/random_nas_tuner.py +++ b/examples/tuners/random_nas_tuner/random_nas_tuner.py @@ -8,7 +8,7 @@ def random_archi_generator(nas_ss, random_state): chosen_archi = {} print("zql: nas search space: ", nas_ss) for block_name, block_value in nas_ss.items(): - assert block_value['_type'] is "mutable_layer", "Random NAS Tuner only receives NAS search space" + assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space" block = block_value['_value'] tmp_block = {} for layer_name, layer in block.items(): From 27aaaeb1dd999ca24636aa40a0bb203fe085afab Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 17:17:47 +0800 Subject: [PATCH 7/9] update random nas tuner --- tools/nni_annotation/test_annotation.py | 2 +- tools/nni_annotation/testcase/annotated/nas.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/nni_annotation/test_annotation.py b/tools/nni_annotation/test_annotation.py index ab15729813..6f531fd620 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/tools/nni_annotation/test_annotation.py @@ -46,7 +46,7 @@ def test_search_space_generator(self): def test_code_generator(self): code_dir = expand_annotations('testcase/usercode', '_generated') self.assertEqual(code_dir, '_generated') - self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py') + self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py', nas_mode='classic_mode') self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py') self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py') with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst: diff --git a/tools/nni_annotation/testcase/annotated/nas.py b/tools/nni_annotation/testcase/annotated/nas.py index 4cacd33389..227f7960b2 100644 --- a/tools/nni_annotation/testcase/annotated/nas.py +++ b/tools/nni_annotation/testcase/annotated/nas.py @@ -1,35 +1,40 @@ import nni import time + def add_one(inputs): return inputs + 1 + def add_two(inputs): return inputs + 2 + def add_three(inputs): return inputs + 3 + def add_four(inputs): return inputs + 4 + def main(): images = 5 layer_1_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_0', {'add_one()': add_one, 'add_two()': add_two, 'add_three()': add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': {}, 'add_three()': {}, 'add_four()': {}}, [], {'images': images}, 1, - None) + 'classic_mode') layer_2_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_1', {'add_one()': add_one, 'add_two()': add_two, 'add_three()': add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': - layer_1_out}, 1, None) + layer_1_out}, 1, 'classic_mode') layer_3_out = nni.mutable_layer('mutable_block_39', 'mutable_layer_2', {'add_one()': add_one, 'add_two()': add_two, 'add_three()': add_three, 'add_four()': add_four}, {'add_one()': {}, 'add_two()': {}, 'add_three()': {}, 'add_four()': {}}, [], {'layer_1_out': - layer_1_out, 'layer_2_out': layer_2_out}, 1, None) + layer_1_out, 'layer_2_out': layer_2_out}, 1, 'classic_mode') nni.report_intermediate_result(layer_1_out) time.sleep(2) nni.report_intermediate_result(layer_2_out) From 51a63155a92b8882dcf279449bef774957c61313 Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Wed, 31 Jul 2019 17:26:41 +0800 Subject: [PATCH 8/9] update annotation ut --- tools/nni_annotation/test_annotation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/nni_annotation/test_annotation.py b/tools/nni_annotation/test_annotation.py index 6f531fd620..7550ba2466 100644 --- a/tools/nni_annotation/test_annotation.py +++ b/tools/nni_annotation/test_annotation.py @@ -44,9 +44,9 @@ def test_search_space_generator(self): self.assertEqual(search_space, json.load(f)) def test_code_generator(self): - code_dir = expand_annotations('testcase/usercode', '_generated') + code_dir = expand_annotations('testcase/usercode', '_generated', nas_mode='classic_mode') self.assertEqual(code_dir, '_generated') - self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py', nas_mode='classic_mode') + self._assert_source_equal('testcase/annotated/nas.py', '_generated/nas.py') self._assert_source_equal('testcase/annotated/mnist.py', '_generated/mnist.py') self._assert_source_equal('testcase/annotated/dir/simple.py', '_generated/dir/simple.py') with open('testcase/usercode/nonpy.txt') as src, open('_generated/nonpy.txt') as dst: From 11d38a3ce406e87dcae084e9addf4d48ae61871f Mon Sep 17 00:00:00 2001 From: Crysple <871886504@qq.com> Date: Thu, 1 Aug 2019 17:10:46 +0800 Subject: [PATCH 9/9] update random nas tuner and docs --- docs/en_US/Tutorial/SearchSpaceSpec.md | 7 +++++++ examples/tuners/random_nas_tuner/random_nas_tuner.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/en_US/Tutorial/SearchSpaceSpec.md b/docs/en_US/Tutorial/SearchSpaceSpec.md index f5a1d34d4f..ee14b4546f 100644 --- a/docs/en_US/Tutorial/SearchSpaceSpec.md +++ b/docs/en_US/Tutorial/SearchSpaceSpec.md @@ -72,6 +72,11 @@ All types of sampling strategies and their parameter are listed here: * Which means the variable value is a value like round(exp(normal(mu, sigma)) / q) * q * Suitable for a discrete variable with respect to which the objective is smooth and gets smoother with the size of the variable, which is bounded from one side. +* {"_type":"mutable_layer","_value":{mutable_layer_infomation}} + * Type for [Neural Architecture Search Space][1]. Value is also a dictionary, which contains key-value pairs representing respectively name and search space of each mutable_layer. + * For now, users can only use this type of search space with annotation, which means that there is no need to define a json file for search space since it will be automatically generated according to the annotation in trial code. + * For detailed usage, please refer to [General NAS Interfaces][1]. + ## Search Space Types Supported by Each Tuner | | choice | randint | uniform | quniform | loguniform | qloguniform | normal | qnormal | lognormal | qlognormal | @@ -103,3 +108,5 @@ Known Limitations: * Only Random Search/TPE/Anneal/Evolution tuner supports nested search space * We do not support nested search space "Hyper Parameter" in visualization now, the enhancement is being considered in #1110(https://github.com/microsoft/nni/issues/1110), any suggestions or discussions or contributions are warmly welcomed + +[1]: ../AdvancedFeature/GeneralNasInterfaces.md \ No newline at end of file diff --git a/examples/tuners/random_nas_tuner/random_nas_tuner.py b/examples/tuners/random_nas_tuner/random_nas_tuner.py index a22f477c98..d7f6214aa6 100644 --- a/examples/tuners/random_nas_tuner/random_nas_tuner.py +++ b/examples/tuners/random_nas_tuner/random_nas_tuner.py @@ -8,7 +8,7 @@ def random_archi_generator(nas_ss, random_state): chosen_archi = {} print("zql: nas search space: ", nas_ss) for block_name, block_value in nas_ss.items(): - assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space" + assert block_value['_type'] == "mutable_layer", "Random NAS Tuner only receives NAS search space whose _type is 'mutable_layer'" block = block_value['_value'] tmp_block = {} for layer_name, layer in block.items():