Merge pull request #244 from microsoft/master

merge master

Makefile

@@ -173,12 +173,12 @@ install-python-modules:
 dev-install-python-modules:
 	#$(_INFO) Installing Python SDK $(_END)
 	mkdir -p build
-	ln -sf ../src/sdk/pynni/nni build/nni
-	ln -sf ../src/sdk/pynni/nnicli build/nnicli
-	ln -sf ../tools/nni_annotation build/nni_annotation
-	ln -sf ../tools/nni_cmd build/nni_cmd
-	ln -sf ../tools/nni_trial_tool build/nni_trial_tool
-	ln -sf ../tools/nni_gpu_tool build/nni_gpu_tool
+	ln -sf ../src/sdk/pynni/nni build
+	ln -sf ../src/sdk/pycli/nnicli build
+	ln -sf ../tools/nni_annotation build
+	ln -sf ../tools/nni_cmd build
+	ln -sf ../tools/nni_trial_tool build
+	ln -sf ../tools/nni_gpu_tool build
 	cp setup.py build/
 	cp README.md build/
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' build/setup.py
@@ -209,10 +209,12 @@ dev-install-node-modules:
 	ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
 	cp src/nni_manager/package.json $(NNI_PKG_FOLDER)
 	sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
-	ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
-	ln -sf ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static
-	ln -sf ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)/build
-	ln -sf ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)/server.js
+	ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)
+	ln -sf ${PWD}/src/webui/build -t $(NNI_PKG_FOLDER)
+	mv $(NNI_PKG_FOLDER)/build $(NNI_PKG_FOLDER)/static
+	mkdir -p $(NASUI_PKG_FOLDER)
+	ln -sf ${PWD}/src/nasui/build $(NASUI_PKG_FOLDER)
+	ln -sf ${PWD}/src/nasui/server.js $(NASUI_PKG_FOLDER)
 
 .PHONY: install-scripts
 install-scripts:

README.md

@@ -16,7 +16,7 @@
 
 **NNI (Neural Network Intelligence)** is a lightweight but powerful toolkit to help users **automate** <a href="docs/en_US/FeatureEngineering/Overview.md">Feature Engineering</a>, <a href="docs/en_US/NAS/Overview.md">Neural Architecture Search</a>, <a href="docs/en_US/Tuner/BuiltinTuner.md">Hyperparameter Tuning</a> and <a href="docs/en_US/Compressor/Overview.md">Model Compression</a>.
 
-The tool manages automated machine learning (AutoML) experiments, **dispatches and runs** experiments' trial jobs generated by tuning algorithms to search the best neural architecture and/or hyper-parameters in **different training environments** like <a href="docs/en_US/TrainingService/LocalMode.md">Local Machine</a>, <a href="docs/en_US/TrainingService/RemoteMachineMode.md">Remote Servers</a>, <a href="docs/en_US/TrainingService/PaiMode.md">OpenPAI</a>, <a href="docs/en_US/TrainingService/KubeflowMode.md">Kubeflow</a>, <a href="docs/en_US/TrainingService/FrameworkControllerMode.md">FrameworkController on K8S (AKS etc.)</a> and other cloud options.
+The tool manages automated machine learning (AutoML) experiments, **dispatches and runs** experiments' trial jobs generated by tuning algorithms to search the best neural architecture and/or hyper-parameters in **different training environments** like <a href="docs/en_US/TrainingService/LocalMode.md">Local Machine</a>, <a href="docs/en_US/TrainingService/RemoteMachineMode.md">Remote Servers</a>, <a href="docs/en_US/TrainingService/PaiMode.md">OpenPAI</a>, <a href="docs/en_US/TrainingService/KubeflowMode.md">Kubeflow</a>, <a href="docs/en_US/TrainingService/FrameworkControllerMode.md">FrameworkController on K8S (AKS etc.)</a>, <a href="docs/en_US/TrainingService/DLTSMode.md">DLWorkspace (aka. DLTS)</a> and other cloud options.
 
 ## **Who should consider using NNI**
 
@@ -25,7 +25,7 @@ The tool manages automated machine learning (AutoML) experiments, **dispatches a
 * Researchers and data scientists who want to easily **implement and experiment new AutoML algorithms**, may it be: hyperparameter tuning algorithm, neural architect search algorithm or model compression algorithm.
 * ML Platform owners who want to **support AutoML in their platform**.
 
-### **NNI v1.5 has been released! &nbsp;<a href="#nni-released-reminder"><img width="48" src="docs/img/release_icon.png"></a>**
+### **[NNI v1.5 has been released!](https://github.com/microsoft/nni/releases) &nbsp;<a href="#nni-released-reminder"><img width="48" src="docs/img/release_icon.png"></a>**
 
 ## **NNI capabilities in a glance**
 
@@ -170,6 +170,7 @@ Within the following table, we summarized the current NNI capabilities, we are g
             <li><a href="docs/en_US/TrainingService/KubeflowMode.md">Kubeflow</a></li>
             <li><a href="docs/en_US/TrainingService/FrameworkControllerMode.md">FrameworkController on K8S (AKS etc.)</a></li>
             </ul>
+            <ul><li><a href="docs/en_US/TrainingService/DLTSMode.md">DLWorkspace (aka. DLTS)</a></li>        
       </ul>
       </td>
     </tr>
@@ -334,10 +335,15 @@ With authors' permission, we listed a set of NNI usage examples and relevant art
   * **Blog (in Chinese)** - [A summary of NNI new capabilities in 2019](https://mp.weixin.qq.com/s/7_KRT-rRojQbNuJzkjFMuA) by @squirrelsc
 
 ## **Feedback**
-
-* Discuss on the NNI [Gitter](https://gitter.im/Microsoft/nni?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) in NNI.
 * [File an issue](https://github.com/microsoft/nni/issues/new/choose) on GitHub.
 * Ask a question with NNI tags on [Stack Overflow](https://stackoverflow.com/questions/tagged/nni?sort=Newest&edited=true).
+* Discuss on the NNI [Gitter](https://gitter.im/Microsoft/nni?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) in NNI.
+
+Join IM discussion groups:
+|Gitter||WeChat|
+|----|----|----|
+|![image](https://user-images.githubusercontent.com/39592018/80665738-e0574a80-8acc-11ea-91bc-0836dc4cbf89.png)| OR |![image](https://user-images.githubusercontent.com/39592018/80665762-f06f2a00-8acc-11ea-8d22-e461e68e2d9b.png)|
+
 
 ## Related Projects
 

README_zh_CN.md

@@ -10,7 +10,7 @@
 
 **NNI (Neural Network Intelligence)** 是一个轻量但强大的工具包，帮助用户**自动**的进行[特征工程](docs/zh_CN/FeatureEngineering/Overview.md)，[神经网络架构搜索](docs/zh_CN/NAS/Overview.md)，[超参调优](docs/zh_CN/Tuner/BuiltinTuner.md)以及[模型压缩](docs/zh_CN/Compressor/Overview.md)。
 
-NNI 管理自动机器学习 (AutoML) 的 Experiment，**调度运行**由调优算法生成的 Trial 任务来找到最好的神经网络架构和/或超参，支持**各种训练环境**，如[本机](docs/zh_CN/TrainingService/LocalMode.md)，[远程服务器](docs/zh_CN/TrainingService/RemoteMachineMode.md)，[OpenPAI](docs/zh_CN/TrainingService/PaiMode.md)，[Kubeflow](docs/zh_CN/TrainingService/KubeflowMode.md)，[基于 K8S 的 FrameworkController（如，AKS 等)](docs/zh_CN/TrainingService/FrameworkControllerMode.md)，以及其它云服务。
+NNI 管理自动机器学习 (AutoML) 的 Experiment，**调度运行**由调优算法生成的 Trial 任务来找到最好的神经网络架构和/或超参，支持**各种训练环境**，如[本机](docs/zh_CN/TrainingService/LocalMode.md)，[远程服务器](docs/zh_CN/TrainingService/RemoteMachineMode.md)，[OpenPAI](docs/zh_CN/TrainingService/PaiMode.md)，[Kubeflow](docs/zh_CN/TrainingService/KubeflowMode.md)，[基于 K8S 的 FrameworkController（如，AKS 等)](docs/zh_CN/TrainingService/FrameworkControllerMode.md)， [DLWorkspace (又称 DLTS)](docs/zh_CN/TrainingService/DLTSMode.md) 和其它云服务。
 
 ## **使用场景**
 
@@ -19,7 +19,7 @@ NNI 管理自动机器学习 (AutoML) 的 Experiment，**调度运行**由调优
 * 想要更容易**实现或试验新的自动机器学习算法**的研究员或数据科学家，包括：超参调优算法，神经网络搜索算法以及模型压缩算法。
 * 在机器学习平台中**支持自动机器学习**。
 
-### **NNI v1.5 已发布！ &nbsp;[<img width="48" src="docs/img/release_icon.png" />](#nni-released-reminder)**
+### **[NNI v1.5 已发布！](https://github.com/microsoft/nni/releases) &nbsp;[<img width="48" src="docs/img/release_icon.png" />](#nni-released-reminder)**
 
 ## **NNI 功能一览**
 
@@ -164,6 +164,7 @@ NNI 提供命令行工具以及友好的 WebUI 来管理训练的 Experiment。
             <li><a href="docs/zh_CN/TrainingService/KubeflowMode.md">Kubeflow</a></li>
             <li><a href="docs/zh_CN/TrainingService/FrameworkControllerMode.md">基于 Kubernetes（AKS 等）的 FrameworkController</a></li>
             </ul>
+            <ul><li><a href="docs/zh_CN/TrainingService/DLTSMode.md">DLWorkspace (又称 DLTS)</a></li>        
       </ul>
       </td>
     </tr>

docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md

@@ -6,7 +6,7 @@
 
 For debugging NNI source code, your development environment should be under Ubuntu 16.04 (or above) system with python 3 and pip 3 installed, then follow the below steps.
 
-**1. Clone the source code**
+### 1. Clone the source code
 
 Run the command
 
@@ -16,7 +16,7 @@ git clone https://github.com/Microsoft/nni.git
 
 to clone the source code
 
-**2. Prepare the debug environment and install dependencies**
+### 2. Prepare the debug environment and install dependencies
 
 Change directory to the source code folder, then run the command
 
@@ -26,7 +26,7 @@ make install-dependencies
 
 to install the dependent tools for the environment
 
-**3. Build source code**
+### 3. Build source code
 
 Run the command
 
@@ -36,7 +36,7 @@ make build
 
 to build the source code
 
-**4. Install NNI to development environment**
+### 4. Install NNI to development environment
 
 Run the command
 
@@ -46,7 +46,7 @@ make dev-install
 
 to install the distribution content to development environment, and create cli scripts
 
-**5. Check if the environment is ready**
+### 5. Check if the environment is ready
 
 Now, you can try to start an experiment to check if your environment is ready.
 For example, run the command
@@ -57,9 +57,21 @@ nnictl create --config ~/nni/examples/trials/mnist-tfv1/config.yml
 
 And open WebUI to check if everything is OK
 
-**6. Redeploy**
+### 6. Redeploy
+
+After the code changes, it may need to redeploy. It depends on what kind of code changed. 
+
+#### Python
+
+It doesn't need to redeploy, but the nnictl may need to be restarted.
+
+#### TypeScript
+
+* If `src/nni_manager` will be changed, run `yarn watch` continually under this folder. It will rebuild code instantly.
+* If `src/webui` or `src/nasui` is changed, use **step 3** to rebuild code.
+
+The nnictl may need to be restarted.
 
-After the code changes, use **step 3** to rebuild your codes, then the changes will take effect immediately.
 
 ---
 At last, wish you have a wonderful day.

docs/en_US/conf.py

@@ -45,6 +45,7 @@
     'sphinx_markdown_tables',
     'sphinxarg.ext',
     'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
 ]
 
 # Add mock modules

docs/zh_CN/Tuner/NetworkmorphismTuner.md

@@ -104,7 +104,7 @@ Tuner 有大量的文件、函数和类。 这里简单介绍最重要的文件
 
 - `networkmorphism_tuner.py` 是使用 network morphism 算法的 Tuner。
 
-- `bayesian.py` 是用来基于已经搜索道德模型来预测未知模型指标的贝叶斯算法。
+- `bayesian.py` 是用来基于已经搜索到的模型来预测未知模型指标的贝叶斯算法。
 
 - `graph.py` 是元图数据结构。 类 Graph 表示了模型的神经网络图。 
   - Graph 从模型中抽取神经网络。

docs/zh_CN/Tuner/PPOTuner.md

@@ -4,7 +4,7 @@
 
 这是一个用于 NNI 神经网络架构搜索（NAS）接口的 Tuner。 它使用了 [ppo 算法](https://arxiv.org/abs/1707.06347)。 此实现继承了 [OpenAI 的 ppo2 实现](https://github.com/openai/baselines/tree/master/baselines/ppo2)的主要逻辑，并为 NAS 场景做了适配。
 
-它能成功调优 [mnist-nas 示例](https://github.com/microsoft/nni/tree/master/examples/trials/mnist-nas)，结果如下：
+mnist-nas 示例已调优，并得到以下结果： **注意：此示例正在重构中，以支持最新的 NAS 接口，完成后会重新发布示例代码。**
 
 ![](../../img/ppo_mnist.png)
 

docs/zh_CN/Tutorial/SetupNniDeveloperEnvironment.md

@@ -6,7 +6,7 @@
 
 要调试 NNI 源代码，需要 Ubuntu 16.04 或更高版本系统的开发环境，并需要安装 Python 3 以及 pip 3，然后遵循以下步骤。
 
-**1. 克隆源代码**
+### 1. 克隆源代码
 
 运行命令
 
@@ -15,7 +15,7 @@
 
 来克隆源代码
 
-**2. 准备调试环境并安装依赖项**
+### 2. 准备调试环境并安装依赖项**
 
 将目录切换到源码目录，然后运行命令
 
@@ -24,7 +24,7 @@
 
 来安装环境的依赖项工具
 
-**3. 生成源代码**
+### 3. 生成源代码
 
 运行命令
 
@@ -33,7 +33,7 @@
 
 来生成源代码
 
-**4. 将 NNI 安装到开发环境中**
+### 4. 将 NNI 安装到开发环境中
 
 运行命令
 
@@ -42,7 +42,7 @@
 
 来安装分发内容到开发环境，并创建 cli 脚本
 
-**5. 检查环境是否正确**
+### 5. 检查环境是否正确
 
 Trial 启动 Experiment 来检查环境。 例如，运行命令
 
@@ -51,9 +51,20 @@ Trial 启动 Experiment 来检查环境。 例如，运行命令
 
 并打开网页界面查看
 
-**6. 重新部署**
+### 6. 重新部署
 
-代码改动后，用**第 3 步**来重新生成代码，改动会立即生效。
+代码更改后，可能需要重新部署。 这取决于更改了哪种代码。
+
+#### Python
+
+不需要重新部署，但可能需要重新启动 nnictl。
+
+#### TypeScript
+
+* 如果要更改 `src/nni_manager`，运行 `yarn watch` 可持续编译改动。 它将实时重建代码。
+* 如果更改了 `src/webui` 或 `src/nasui` ，请使用 **第 3 步** 来重建代码。
+
+可能需要重新启动 nnictl。
 
 * * *
 

examples/nas/classic_nas/mnist.py

@@ -8,6 +8,8 @@
 import os
 import argparse
 import logging
+from collections import OrderedDict
+
 import nni
 import torch
 import torch.nn as nn
@@ -26,13 +28,15 @@ class Net(nn.Module):
     def __init__(self, hidden_size):
         super(Net, self).__init__()
         # two options of conv1
-        self.conv1 = LayerChoice([nn.Conv2d(1, 20, 5, 1),
-                                  nn.Conv2d(1, 20, 3, 1)],
-                                 key='first_conv')
+        self.conv1 = LayerChoice(OrderedDict([
+            ("conv5x5", nn.Conv2d(1, 20, 5, 1)),
+            ("conv3x3", nn.Conv2d(1, 20, 3, 1))
+        ]), key='first_conv')
         # two options of mid_conv
-        self.mid_conv = LayerChoice([nn.Conv2d(20, 20, 3, 1, padding=1),
-                                     nn.Conv2d(20, 20, 5, 1, padding=2)],
-                                    key='mid_conv')
+        self.mid_conv = LayerChoice([
+            nn.Conv2d(20, 20, 3, 1, padding=1),
+            nn.Conv2d(20, 20, 5, 1, padding=2)
+        ], key='mid_conv')
         self.conv2 = nn.Conv2d(20, 50, 5, 1)
         self.fc1 = nn.Linear(4*4*50, hidden_size)
         self.fc2 = nn.Linear(hidden_size, 10)
@@ -167,7 +171,6 @@ def get_params():
     parser.add_argument('--log_interval', type=int, default=1000, metavar='N',
                         help='how many batches to wait before logging training status')
 
-
     args, _ = parser.parse_known_args()
     return args
 

examples/nas/darts/model.py

@@ -1,6 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+from collections import OrderedDict
+
 import torch
 import torch.nn as nn
 
@@ -43,17 +45,15 @@ def __init__(self, node_id, num_prev_nodes, channels, num_downsample_connect):
             stride = 2 if i < num_downsample_connect else 1
             choice_keys.append("{}_p{}".format(node_id, i))
             self.ops.append(
-                mutables.LayerChoice(
-                    [
-                        ops.PoolBN('max', channels, 3, stride, 1, affine=False),
-                        ops.PoolBN('avg', channels, 3, stride, 1, affine=False),
-                        nn.Identity() if stride == 1 else ops.FactorizedReduce(channels, channels, affine=False),
-                        ops.SepConv(channels, channels, 3, stride, 1, affine=False),
-                        ops.SepConv(channels, channels, 5, stride, 2, affine=False),
-                        ops.DilConv(channels, channels, 3, stride, 2, 2, affine=False),
-                        ops.DilConv(channels, channels, 5, stride, 4, 2, affine=False)
-                    ],
-                    key=choice_keys[-1]))
+                mutables.LayerChoice(OrderedDict([
+                    ("maxpool", ops.PoolBN('max', channels, 3, stride, 1, affine=False)),
+                    ("avgpool", ops.PoolBN('avg', channels, 3, stride, 1, affine=False)),
+                    ("skipconnect", nn.Identity() if stride == 1 else ops.FactorizedReduce(channels, channels, affine=False)),
+                    ("sepconv3x3", ops.SepConv(channels, channels, 3, stride, 1, affine=False)),
+                    ("sepconv5x5", ops.SepConv(channels, channels, 5, stride, 2, affine=False)),
+                    ("dilconv3x3", ops.DilConv(channels, channels, 3, stride, 2, 2, affine=False)),
+                    ("dilconv5x5", ops.DilConv(channels, channels, 5, stride, 4, 2, affine=False))
+                ]), key=choice_keys[-1]))
         self.drop_path = ops.DropPath()
         self.input_switch = mutables.InputChoice(choose_from=choice_keys, n_chosen=2, key="{}_switch".format(node_id))
 

examples/nas/spos/network.py

@@ -151,6 +151,5 @@ def load_and_parse_state_dict(filepath="./data/checkpoint-150000.pth.tar"):
     for k, v in checkpoint["state_dict"].items():
         if k.startswith("module."):
             k = k[len("module."):]
-        k = re.sub(r"^(features.\d+).(\d+)", "\\1.choices.\\2", k)
         result[k] = v
     return result

src/nni_manager/package.json

@@ -6,6 +6,7 @@
     "build": "tsc",
     "test": "nyc mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --colors",
     "start": "node dist/main.js",
+    "watch": "tsc --watch",
     "eslint": "npx eslint ./ --ext .ts"
   },
   "license": "MIT",