Fix bugs found in integration test

examples/nas/.gitignore

@@ -2,3 +2,4 @@ data
 checkpoints
 runs
 nni_auto_gen_search_space.json
+checkpoint.json

examples/nas/darts/search.py

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import json
 import logging
 import time
 from argparse import ArgumentParser
@@ -67,4 +68,6 @@
             unrolled=args.unrolled
         )
         trainer.fit()
+        final_architecture = trainer.export()
         print('Final architecture:', trainer.export())
+        json.dump(trainer.export(), open('checkpoint.json', 'w'))

nni/runtime/config.py

@@ -0,0 +1,34 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import os
+from pathlib import Path
+import shutil
+import sys
+
+import nni
+
+def get_config_directory() -> Path:
+    """
+    Get NNI config directory.
+    Create it if not exist.
+    """
+    if sys.prefix != sys.base_prefix or Path(sys.prefix, 'conda-meta').is_dir():
+        config_dir = Path(sys.prefix, 'nni')
+    elif sys.platform == 'win32':
+        config_dir = Path(os.environ['APPDATA'], 'nni')
+    else:
+        config_dir = Path.home() / '.config/nni'
+    config_dir.mkdir(parents=True, exist_ok=True)
+    return config_dir
+
+def get_config_file(name: str) -> Path:
+    """
+    Get an NNI config file.
+    Copy from `nni/runtime/default_config` if not exist.
+    """
+    config_file = get_config_directory() / name
+    if not config_file.exists():
+        default = Path(nni.__path__[0], 'runtime/default_config', name)
+        shutil.copyfile(default, config_file)
+    return config_file

nni/runtime/log.py

@@ -74,7 +74,8 @@ def _init_logger_trial() -> None:
     log_file = open(log_path, 'w')
     _setup_root_logger(StreamHandler(log_file), logging.INFO)
 
-    sys.stdout = _LogFileWrapper(log_file)
+    if trial_env_vars.NNI_PLATFORM == 'local':
+        sys.stdout = _LogFileWrapper(log_file)
 
 
 def _init_logger_standalone() -> None:

nni/tools/package_utils/__init__.py

@@ -7,7 +7,7 @@
 from pathlib import Path
 import sys
 import ruamel.yaml as yaml
-import nni
+from nni.runtime.config import get_config_file
 
 ALGO_TYPES = ['tuners', 'assessors', 'advisors']
 
@@ -210,19 +210,7 @@ def _using_conda_or_virtual_environment():
     return sys.prefix != sys.base_prefix or os.path.isdir(os.path.join(sys.prefix, 'conda-meta'))
 
 def get_registered_algo_config_path():
-    # Find the path for registered_algorithms.yml for this nni installation,
-    # the registered_algorithms.yml is copied into this location in setup.py,
-    # so we need to ensure that we use the same logic as setup.py to find the location.
-
-    if _using_conda_or_virtual_environment():
-        nni_config_dir = os.path.join(sys.prefix, 'nni')
-    elif sys.platform == 'win32':
-        nni_config_dir = os.path.join(os.getenv('APPDATA'), 'nni')
-    else:
-        nni_config_dir = os.path.expanduser('~/.config/nni')
-    if not os.path.exists(nni_config_dir):
-        os.makedirs(nni_config_dir, exist_ok=True)
-    return os.path.join(nni_config_dir, 'registered_algorithms.yml')
+    return str(get_config_file('registered_algorithms.yml'))
 
 def read_registerd_algo_meta():
     config_file = get_registered_algo_config_path()

nni/tools/trial_tool/trial_keeper.py

@@ -92,7 +92,10 @@ def main_loop(args):
 
     # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior
     log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader()
-    _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid)
+    if sys.platform == 'win32':
+        _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout)
+    else:
+        _trial_process = Popen(args.trial_command, shell=True, stdout=log_pipe_stdout, stderr=log_pipe_stdout, preexec_fn=os.setsid)
     nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(_trial_process.pid,
                                                                                                   shlex.split(
                                                                                                       args.trial_command)))

pipelines/full-test-linux.yml

@@ -15,36 +15,33 @@ jobs:
       echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
       echo "##vso[task.setvariable variable=NNI_RELEASE]999.$(date -u +%Y%m%d%H%M%S)"
 
-      python3 -m pip install -U --upgrade pip setuptools
-      python3 -m pip install -U pytest
+      python3 -m pip install --upgrade pip setuptools
+      python3 -m pip install pytest
     displayName: Prepare
 
   - script: |
       set -e
       python3 setup.py build_ts
       python3 setup.py bdist_wheel -p manylinux1_x86_64
-      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
+      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB,PPOTuner]
     displayName: Install NNI
 
   - script: |
       set -e
-      python3 -m pip install -U scikit-learn==0.23.2
-      python3 -m pip install -U torchvision==0.4.2
-      python3 -m pip install -U torch==1.3.1
-      python3 -m pip install -U keras==2.1.6
-      python3 -m pip install -U tensorflow==2.3.1 tensorflow-estimator==2.3.0
-      python3 -m pip install -U thop
+      python3 -m pip install scikit-learn==0.23.2
+      python3 -m pip install torchvision==0.4.2
+      python3 -m pip install torch==1.3.1
+      python3 -m pip install keras==2.1.6
+      python3 -m pip install tensorflow==2.3.1 tensorflow-estimator==2.3.0
+      python3 -m pip install thop
       sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
-      nnictl package install --name=PPOTuner
     displayName: Install extra dependencies
 
   - script: |
       set -e
       cd examples/tuners/customized_tuner
       python3 setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install customized tuner
 
   - script: |

pipelines/full-test-windows.yml

@@ -12,39 +12,38 @@ jobs:
 
   steps:
   - script: |
-      python -m pip install -U --upgrade pip setuptools
-      python -m pip install -U pytest
+      python -m pip install --upgrade pip setuptools
+      python -m pip install pytest
     displayName: Install Python tools
-  
+
   - script: |
       python -m pip uninstall nni --yes
       set NNI_RELEASE=999.0
       python setup.py build_ts
       python setup.py bdist_wheel -p win_amd64
-      python -m pip install dist/nni-999.0-py3-none-win_amd64.whl
+      python -m pip install dist/nni-999.0-py3-none-win_amd64.whl[PPOTuner]
     displayName: Install NNI
-  
+
   - script: |
-      python -m pip install -U scikit-learn==0.23.2
-      python -m pip install -U keras==2.1.6
-      python -m pip install -U torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html
-      python -m pip install -U tensorflow==2.3.1 tensorflow-estimator==2.3.0
-      nnictl package install --name=PPOTuner
+      python -m pip install scikit-learn==0.23.2
+      python -m pip install keras==2.1.6
+      python -m pip install torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html
+      python -m pip install tensorflow==2.3.1 tensorflow-estimator==2.3.0
     displayName: Install extra dependencies
-  
+
   - script: |
       cd examples/tuners/customized_tuner
       python setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install example customized tuner
-  
+
   - script: |
       cd test
       python -m pytest ut
       echo "TODO: TypeScript UT"
     displayName: Unit test
     continueOnError: true
-  
+
   - script: |
       cd test
       python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local

pipelines/integration-test-frameworkcontroller.yml

@@ -0,0 +1,52 @@
+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+# variables set on VSO: (mostly for security concern)
+#   manager_ip
+#   docker_hub_password
+
+jobs:
+- job: frameworkcontroller
+  pool: NNI CI KUBE CLI
+  timeoutInMinutes: 120
+
+  steps:
+  - script: |
+      echo "Working directory: ${PWD}"
+      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
+      python3 test/vso_tools/generate_nni_version.py
+      python3 -m pip install --upgrade pip setuptools
+    displayName: Prepare
+
+  - script: |
+      set -e
+      python3 test/vso_tools/install_nni.py $(NNI_RELEASE) SMAC,BOHB
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install NNI
+
+  - script: |
+      set -e
+      docker login -u nnidev -p $(docker_hub_password)
+      docker build --build-arg NNI_RELEASE=$(NNI_RELEASE) -t nnidev/nni-nightly .
+      docker push nnidev/nni-nightly
+    displayName: Build and upload docker image
+
+  - script: |
+      set -e
+      cd test
+      python3 nni_test/nnitest/generate_ts_config.py \
+          --ts frameworkcontroller \
+          --keyvault_vaultname NNIKeyVault \
+          --keyvault_name AzureStorageAccountKey \
+          --azs_account nniazurestorage \
+          --azs_share nni \
+          --nni_docker_image nnidev/nni-nightly \
+          --nni_manager_ip $(manager_ip)
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts frameworkcontroller --exclude multi-phase
+    displayName: Integration test

pipelines/integration-test-kubeflow.yml

@@ -0,0 +1,62 @@
+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+# variables set on VSO: (mostly for security concern)
+#   manager_ip
+#   docker_hub_password
+
+jobs:
+- job: kubeflow
+  pool: NNI CI KUBE CLI
+  timeoutInMinutes: 120
+
+  steps:
+  - script: |
+      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
+      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
+      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"
+
+      echo "Working directory: ${PWD}"
+      echo "NNI version: ${NNI_RELEASE}"
+
+      python3 -m pip install --upgrade pip setuptools
+    displayName: Prepare
+
+  - script: |
+      set -e
+      python3 setup.py build_ts
+      python3 setup.py bdist_wheel -p manylinux1_x86_64
+      python3 -m pip install dist/nni-$(NNI_RELEASE)-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
+    displayName: Build and install NNI
+
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+    displayName: Install customized tuner
+
+  - script: |
+      set -e
+      docker login -u nnidev -p $(docker_hub_password)
+      docker build --build-arg NNI_RELEASE=$(NNI_RELEASE) -t nnidev/nni-nightly .
+      docker push nnidev/nni-nightly
+    displayName: Build and upload docker image
+
+  - script: |
+      set -e
+      cd test
+      python3 nni_test/nnitest/generate_ts_config.py \
+          --ts kubeflow \
+          --keyvault_vaultname NNIKeyVault \
+          --keyvault_name AzureStorageAccountKey \
+          --azs_account nniazurestorage \
+          --azs_share nni \
+          --nni_docker_image nnidev/nni-nightly \
+          --nni_manager_ip $(manager_ip)
+      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts kubeflow --exclude multi-phase
+    displayName: Integration test

pipelines/integration-test-openpai-linux.yml

@@ -26,37 +26,30 @@ jobs:
       echo "NNI version: ${NNI_RELEASE}"
       echo "Build docker image: $(build_docker_image)"
 
-      python3 -m pip install -U --upgrade pip setuptools
+      python3 -m pip install --upgrade pip setuptools
     displayName: Prepare
 
   - script: |
       set -e
       python3 setup.py build_ts
       python3 setup.py bdist_wheel -p manylinux1_x86_64
-      python3 -m pip install -U dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
+      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
     displayName: Build and install NNI
 
-  - script: |
-      set -e
-      sudo apt-get install swig -y
-      nnictl package install --name=SMAC
-      nnictl package install --name=BOHB
-    displayName: Install extra tuners
-
   - script: |
       set -e
       cd examples/tuners/customized_tuner
       python3 setup.py develop --user
-      nnictl package install .
+      nnictl algo register --meta meta_file.yml
     displayName: Install customized tuner
 
   - script: |
       set -e
       docker login -u nnidev -p $(docker_hub_password)
       echo '## Build docker image ##'
-      docker build --build-arg NNI_RELEASE=${NNI_RELEASE} -t nnidev/nni-it-pai:latest .
+      docker build --build-arg NNI_RELEASE=${NNI_RELEASE} -t nnidev/nni-nightly .
       echo '## Upload docker image ##'
-      docker push nnidev/nni-it-pai:latest
+      docker push nnidev/nni-nightly
     condition: eq(variables['build_docker_image'], 'true')
     displayName: Build and upload docker image
 
@@ -68,7 +61,7 @@ jobs:
           --pai_reuse false \
           --pai_host https://ne.openpai.org \
           --pai_user $(pai_user) \
-          --nni_docker_image nnidev/nni-it-pai:latest \
+          --nni_docker_image nnidev/nni-nightly \
           --pai_storage_config_name confignfs-data \
           --pai_token $(pai_token) \
           --nni_manager_nfs_mount_path /home/quzha/mnt-pai-ne/shinyang3 \