merge master

deployment/docker/Dockerfile → Dockerfile

@@ -3,12 +3,15 @@
 
 FROM nvidia/cuda:9.2-cudnn7-runtime-ubuntu18.04
 
+ARG NNI_RELEASE
+
 LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
 
 ENV DEBIAN_FRONTEND=noninteractive 
 
-RUN apt-get -y update && \
-    apt-get -y install sudo \
+RUN apt-get -y update
+RUN apt-get -y install \
+    sudo \
     apt-utils \
     git \
     curl \
@@ -26,28 +29,27 @@ RUN apt-get -y update && \
     python3-dev \
     python3-pip \
     python3-tk \
-    libcupti-dev && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+    libcupti-dev
+RUN apt-get clean
+RUN rm -rf /var/lib/apt/lists/*
 
 #
 # generate python script
 #
-RUN cp /usr/bin/python3 /usr/bin/python
+RUN ln -s python3 /usr/bin/python
 
 #
 # update pip
 #
-RUN python3 -m pip install --upgrade pip==20.0.2 setuptools==41.0.0
+RUN python3 -m pip install --upgrade pip==20.2.4 setuptools==50.3.2
 
 # numpy 1.14.3  scipy 1.1.0
-RUN python3 -m pip --no-cache-dir install \
-    numpy==1.14.3 scipy==1.1.0
+RUN python3 -m pip --no-cache-dir install numpy==1.14.3 scipy==1.1.0
 
 #
-# Tensorflow 1.15
+# TensorFlow
 #
-RUN python3 -m pip --no-cache-dir install tensorflow-gpu==1.15.0
+RUN python3 -m pip --no-cache-dir install tensorflow==2.3.1
 
 #
 # Keras 2.1.6
@@ -73,15 +75,15 @@ RUN python3 -m pip --no-cache-dir install pandas==0.23.4 lightgbm==2.2.2
 #
 # Install NNI
 #
-RUN python3 -m pip --no-cache-dir install nni
+COPY dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl .
+RUN python3 -m pip install nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
 
 #
 # install aml package
 #
 RUN python3 -m pip --no-cache-dir install azureml
 RUN python3 -m pip --no-cache-dir install azureml-sdk
 
-
 ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:/bin:/sbin
 
 WORKDIR /root

docs/en_US/ResearchPublications.md

@@ -0,0 +1,131 @@
+# Research and Publications
+
+We are intensively working on both tool chain and research to make automatic model design and tuning really practical and powerful. On the one hand, our main work is tool chain oriented development. On the other hand, our research works aim to improve this tool chain, rethink challenging problems in AutoML (on both system and algorithm) and propose elegant solutions. Below we list some of our research works, we encourage more research works on this topic and encourage collaboration with us.
+
+
+## System Research
+
+- [Retiarii: A Deep Learning Exploratory-Training Framework](https://www.usenix.org/system/files/osdi20-zhang_quanlu.pdf)
+
+```bibtex
+@inproceedings{zhang2020retiarii,
+  title={Retiarii: A Deep Learning Exploratory-Training Framework},
+  author={Zhang, Quanlu and Han, Zhenhua and Yang, Fan and Zhang, Yuge and Liu, Zhe and Yang, Mao and Zhou, Lidong},
+  booktitle={14th $\{$USENIX$\}$ Symposium on Operating Systems Design and Implementation ($\{$OSDI$\}$ 20)},
+  pages={919--936},
+  year={2020}
+}
+```
+
+- [AutoSys: The Design and Operation of Learning-Augmented Systems](https://www.usenix.org/system/files/atc20-liang-chieh-jan.pdf)
+
+```bibtex
+@inproceedings{liang2020autosys,
+  title={AutoSys: The Design and Operation of Learning-Augmented Systems},
+  author={Liang, Chieh-Jan Mike and Xue, Hui and Yang, Mao and Zhou, Lidong and Zhu, Lifei and Li, Zhao Lucis and Wang, Zibo and Chen, Qi and Zhang, Quanlu and Liu, Chuanjie and others},
+  booktitle={2020 $\{$USENIX$\}$ Annual Technical Conference ($\{$USENIX$\}$$\{$ATC$\}$ 20)},
+  pages={323--336},
+  year={2020}
+}
+```
+
+- [Gandiva: Introspective Cluster Scheduling for Deep Learning](https://www.usenix.org/system/files/osdi18-xiao.pdf)
+
+```bibtex
+@inproceedings{xiao2018gandiva,
+  title={Gandiva: Introspective cluster scheduling for deep learning},
+  author={Xiao, Wencong and Bhardwaj, Romil and Ramjee, Ramachandran and Sivathanu, Muthian and Kwatra, Nipun and Han, Zhenhua and Patel, Pratyush and Peng, Xuan and Zhao, Hanyu and Zhang, Quanlu and others},
+  booktitle={13th $\{$USENIX$\}$ Symposium on Operating Systems Design and Implementation ($\{$OSDI$\}$ 18)},
+  pages={595--610},
+  year={2018}
+}
+```
+
+## Algorithm Research
+
+### New Algorithms
+
+- [TextNAS: A Neural Architecture Search Space Tailored for Text Representation](https://arxiv.org/pdf/1912.10729.pdf)
+
+```bibtex
+@inproceedings{wang2020textnas,
+  title={TextNAS: A Neural Architecture Search Space Tailored for Text Representation.},
+  author={Wang, Yujing and Yang, Yaming and Chen, Yiren and Bai, Jing and Zhang, Ce and Su, Guinan and Kou, Xiaoyu and Tong, Yunhai and Yang, Mao and Zhou, Lidong},
+  booktitle={AAAI},
+  pages={9242--9249},
+  year={2020}
+}
+```
+
+- [Cream of the Crop: Distilling Prioritized Paths For One-Shot Neural Architecture Search](https://papers.nips.cc/paper/2020/file/d072677d210ac4c03ba046120f0802ec-Paper.pdf)
+
+```bibtex
+@article{peng2020cream,
+  title={Cream of the Crop: Distilling Prioritized Paths For One-Shot Neural Architecture Search},
+  author={Peng, Houwen and Du, Hao and Yu, Hongyuan and Li, Qi and Liao, Jing and Fu, Jianlong},
+  journal={Advances in Neural Information Processing Systems},
+  volume={33},
+  year={2020}
+}
+```
+
+- [Metis: Robustly tuning tail latencies of cloud systems](https://www.usenix.org/system/files/conference/atc18/atc18-li-zhao.pdf)
+
+```bibtex
+@inproceedings{li2018metis,
+  title={Metis: Robustly tuning tail latencies of cloud systems},
+  author={Li, Zhao Lucis and Liang, Chieh-Jan Mike and He, Wenjia and Zhu, Lianjie and Dai, Wenjun and Jiang, Jin and Sun, Guangzhong},
+  booktitle={2018 $\{$USENIX$\}$ Annual Technical Conference ($\{$USENIX$\}$$\{$ATC$\}$ 18)},
+  pages={981--992},
+  year={2018}
+}
+```
+
+- [OpEvo: An Evolutionary Method for Tensor Operator Optimization](https://arxiv.org/abs/2006.05664)
+
+```bibtex
+@article{gao2020opevo,
+  title={OpEvo: An Evolutionary Method for Tensor Operator Optimization},
+  author={Gao, Xiaotian and Wei, Cui and Zhang, Lintao and Yang, Mao},
+  journal={arXiv preprint arXiv:2006.05664},
+  year={2020}
+}
+```
+
+### Measurement and Understanding
+
+- [Deeper insights into weight sharing in neural architecture search](https://arxiv.org/pdf/2001.01431.pdf)
+
+```bibtex
+@article{zhang2020deeper,
+  title={Deeper insights into weight sharing in neural architecture search},
+  author={Zhang, Yuge and Lin, Zejun and Jiang, Junyang and Zhang, Quanlu and Wang, Yujing and Xue, Hui and Zhang, Chen and Yang, Yaming},
+  journal={arXiv preprint arXiv:2001.01431},
+  year={2020}
+}
+```
+
+- [How Does Supernet Help in Neural Architecture Search?](https://arxiv.org/abs/2010.08219)
+
+```bibtex
+@article{zhang2020does,
+  title={How Does Supernet Help in Neural Architecture Search?},
+  author={Zhang, Yuge and Zhang, Quanlu and Yang, Yaming},
+  journal={arXiv preprint arXiv:2010.08219},
+  year={2020}
+}
+```
+
+### Applications
+
+- [AutoADR: Automatic Model Design for Ad Relevance](https://arxiv.org/pdf/2010.07075.pdf)
+
+```bibtex
+@inproceedings{chen2020autoadr,
+  title={AutoADR: Automatic Model Design for Ad Relevance},
+  author={Chen, Yiren and Yang, Yaming and Sun, Hong and Wang, Yujing and Xu, Yu and Shen, Wei and Zhou, Rong and Tong, Yunhai and Bai, Jing and Zhang, Ruofei},
+  booktitle={Proceedings of the 29th ACM International Conference on Information \& Knowledge Management},
+  pages={2365--2372},
+  year={2020}
+}
+```

docs/en_US/contents.rst

@@ -17,6 +17,7 @@ Neural Network Intelligence
     Feature Engineering <feature_engineering>
     References <reference>
     Use Cases and Solutions <CommunitySharings/community_sharings>
+    Research and Publications <ResearchPublications.md>
     FAQ <Tutorial/FAQ>
     How to Contribute <contribution>
     Changelog <Release>

examples/model_compress/BNN_quantizer_cifar10.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import BNNQuantizer
+from nni.algorithms.compression.pytorch.quantization import BNNQuantizer
 
 
 class VGG_Cifar10(nn.Module):

examples/model_compress/DoReFaQuantizer_torch_mnist.py

@@ -1,7 +1,7 @@
 import torch
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import DoReFaQuantizer
+from nni.algorithms.compression.pytorch.quantization import DoReFaQuantizer
 
 
 class Mnist(torch.nn.Module):
@@ -86,4 +86,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()

examples/model_compress/L1_torch_cifar10.py

@@ -3,7 +3,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import L1FilterPruner
+from nni.algorithms.compression.pytorch.pruning import L1FilterPruner
 from models.cifar10.vgg import VGG
 
 

examples/model_compress/QAT_torch_quantizer.py

@@ -1,7 +1,7 @@
 import torch
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import QAT_Quantizer
+from nni.algorithms.compression.pytorch.quantization import QAT_Quantizer
 
 
 class Mnist(torch.nn.Module):

examples/model_compress/amc/amc_search.py

@@ -8,7 +8,7 @@
 import torch
 import torch.nn as nn
 from torchvision.models import resnet
-from nni.compression.torch import AMCPruner
+from nni.algorithms.compression.pytorch.pruning import AMCPruner
 from data import get_split_dataset
 from utils import AverageMeter, accuracy
 

examples/model_compress/amc/amc_train.py

@@ -15,9 +15,9 @@
 from tensorboardX import SummaryWriter
 from torchvision.models import resnet
 
-from nni.compression.torch.pruning.amc.lib.net_measure import measure_model
-from nni.compression.torch.pruning.amc.lib.utils import get_output_folder
-from nni.compression.torch import ModelSpeedup
+from nni.algorithms.compression.pytorch.pruning.amc.lib.net_measure import measure_model
+from nni.algorithms.compression.pytorch.pruning.amc.lib.utils import get_output_folder
+from nni.compression.pytorch import ModelSpeedup
 
 from data import get_dataset
 from utils import AverageMeter, accuracy, progress_bar

examples/model_compress/auto_pruners_torch.py

@@ -14,10 +14,10 @@
 from models.mnist.lenet import LeNet
 from models.cifar10.vgg import VGG
 from models.cifar10.resnet import ResNet18, ResNet50
-from nni.compression.torch import L1FilterPruner, L2FilterPruner, FPGMPruner
-from nni.compression.torch import SimulatedAnnealingPruner, ADMMPruner, NetAdaptPruner, AutoCompressPruner
-from nni.compression.torch import ModelSpeedup
-from nni.compression.torch.utils.counter import count_flops_params
+from nni.algorithms.compression.pytorch.pruning import L1FilterPruner, L2FilterPruner, FPGMPruner
+from nni.algorithms.compression.pytorch.pruning import SimulatedAnnealingPruner, ADMMPruner, NetAdaptPruner, AutoCompressPruner
+from nni.compression.pytorch import ModelSpeedup
+from nni.compression.pytorch.utils.counter import count_flops_params
 
 
 def get_data(dataset, data_dir, batch_size, test_batch_size):

examples/model_compress/lottery_torch_mnist_fc.py

@@ -6,7 +6,7 @@
 import torch.utils.data
 import torchvision.datasets as datasets
 import torchvision.transforms as transforms
-from nni.compression.torch import LotteryTicketPruner
+from nni.algorithms.compression.pytorch.pruning import LotteryTicketPruner
 
 class fc1(nn.Module):
 

examples/model_compress/model_prune_torch.py

@@ -9,8 +9,16 @@
 
 from models.cifar10.vgg import VGG
 import nni
-from nni.compression.torch import LevelPruner, SlimPruner, FPGMPruner, L1FilterPruner, \
-    L2FilterPruner, AGPPruner, ActivationMeanRankFilterPruner, ActivationAPoZRankFilterPruner
+from nni.algorithms.compression.pytorch.pruning import (
+    LevelPruner,
+    SlimPruner,
+    FPGMPruner,
+    L1FilterPruner,
+    L2FilterPruner,
+    AGPPruner,
+    ActivationMeanRankFilterPruner,
+    ActivationAPoZRankFilterPruner
+)
 
 prune_config = {
     'level': {

examples/model_compress/model_speedup.py

@@ -6,7 +6,7 @@
 import torch.nn.functional as F
 from torchvision import datasets, transforms
 from models.cifar10.vgg import VGG
-from nni.compression.torch import apply_compression_results, ModelSpeedup
+from nni.compression.pytorch import apply_compression_results, ModelSpeedup
 
 torch.manual_seed(0)
 use_mask = True

examples/model_compress/pruning_kd.py

@@ -3,7 +3,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import L1FilterPruner
+from nni.algorithms.compression.pytorch.pruning import L1FilterPruner
 from knowledge_distill.knowledge_distill import KnowledgeDistill
 from models.cifar10.vgg import VGG
 

examples/model_compress/slim_torch_cifar10.py

@@ -5,7 +5,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from torchvision import datasets, transforms
-from nni.compression.torch import SlimPruner
+from nni.algorithms.compression.pytorch.pruning import SlimPruner
 from models.cifar10.vgg import VGG
 
 def updateBN(model):

examples/nas/classic_nas/mnist.py

@@ -18,7 +18,7 @@
 from torchvision import datasets, transforms
 
 from nni.nas.pytorch.mutables import LayerChoice, InputChoice
-from nni.nas.pytorch.classic_nas import get_and_apply_next_architecture
+from nni.algorithms.nas.pytorch.classic_nas import get_and_apply_next_architecture
 
 
 logger = logging.getLogger('mnist_AutoML')

nni/algorithms/compression/pytorch/pruning/__init__.py

@@ -3,7 +3,6 @@
 
 from .finegrained_pruning import *
 from .structured_pruning import *
-from .apply_compression import apply_compression_results
 from .one_shot import *
 from .agp import *
 from .lottery_ticket import LotteryTicketPruner

nni/algorithms/compression/pytorch/pruning/structured_pruning.py

@@ -278,7 +278,8 @@ def _dependency_calc_mask(self, sparsities, wrappers, wrappers_idx, channel_dset
                 sparsity, _w, _w_idx)
             num_total = current_weight.size(0)
             if num_total < 2 or num_prune < 1:
-                return base_mask
+                masks[name] = base_mask
+                continue
             _tmp_mask = self.get_mask(
                 base_mask, current_weight, num_prune, _w, _w_idx, channel_masks)
 

nni/compression/pytorch/__init__.py

@@ -3,3 +3,4 @@
 
 from .speedup import ModelSpeedup
 from .compressor import Compressor, Pruner, Quantizer
+from .pruning import apply_compression_results