Merge pull request #212 from microsoft/master

merge master

azure-pipelines.yml

@@ -13,8 +13,8 @@ jobs:
   - script: |
       python3 -m pip install torch==0.4.1 --user
       python3 -m pip install torchvision==0.2.1 --user
-      python3 -m pip install tensorflow==1.12.0 --user
-    displayName: 'Install dependencies for integration'
+      python3 -m pip install tensorflow==1.13.1 --user
+    displayName: 'Install dependencies'
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
@@ -59,7 +59,7 @@ jobs:
       python3 -m pip install torch==0.4.1 --user
       python3 -m pip install torchvision==0.2.1 --user
       python3 -m pip install tensorflow==1.13.1 --user
-    displayName: 'Install dependencies for integration'
+    displayName: 'Install dependencies'
   - script: |
       source install.sh
     displayName: 'Install nni toolkit via source code'
@@ -79,3 +79,43 @@ jobs:
       cd test
       PATH=$HOME/Library/Python/3.7/bin:$PATH python3 cli_test.py
     displayName: 'nnicli test'
+
+- job: 'basic_test_pr_Windows'
+  pool:
+    vmImage: 'vs2017-win2016'
+  strategy:
+    matrix:
+      Python36:
+        PYTHON_VERSION: '3.6'
+
+  steps:
+  - script: |
+      powershell.exe -file install.ps1
+    displayName: 'Install nni toolkit via source code'
+  - script: |
+      python -m pip install scikit-learn==0.20.0 --user
+      python -m pip install keras==2.1.6 --user
+      python -m pip install https://download.pytorch.org/whl/cu90/torch-0.4.1-cp36-cp36m-win_amd64.whl --user
+      python -m pip install torchvision --user
+      python -m pip install tensorflow==1.13.1 --user
+    displayName: 'Install dependencies'
+  - script: |
+      cd test
+      powershell.exe -file unittest.ps1
+    displayName: 'unit test'
+  - script: |
+      cd test
+      python naive_test.py
+    displayName: 'Naive test'
+  - script: |
+      cd test
+      python tuner_test.py
+    displayName: 'Built-in tuners / assessors tests'
+  - script: |
+      cd test
+      python metrics_test.py
+    displayName: 'Trial job metrics test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
+    displayName: 'nnicli test'

docs/en_US/Makefile

@@ -16,4 +16,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/en_US/Tuner/BuiltinTuner.md

@@ -122,7 +122,7 @@ Its requirement of computation resource is relatively high. Specifically, it req
 
 * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', the tuner will target to maximize metrics. If 'minimize', the tuner will target to minimize metrics.
 
-* **population_size** (*int value(should >0), optional, default = 20*) - the initial size of the population(trial num) in evolution tuner.
+* **population_size** (*int value (should > 0), optional, default = 20*) - the initial size of the population(trial num) in evolution tuner. Suggests `population_size` be much larger than `concurrency`, so users can get the most out of the algorithm (and at least `concurrency`, or the tuner will fail on their first generation of parameters).
 
 **Usage example**
 

docs/en_US/sdk_reference.rst

@@ -41,10 +41,13 @@ Assessor
 ..  autoclass:: nni.assessor.Assessor
     :members:
 
-..  autoclass:: nni.curvefitting_assessor.curvefitting_assessor.CurvefittingAssessor
+..  autoclass:: nni.assessor.AssessResult
     :members:
 
-..  autoclass:: nni.medianstop_assessor.medianstop_assessor.MedianstopAssessor
+..  autoclass:: nni.curvefitting_assessor.CurvefittingAssessor
+    :members:
+
+..  autoclass:: nni.medianstop_assessor.MedianstopAssessor
     :members:
 
 
@@ -57,4 +60,4 @@ Advisor
     :members:
 
 ..  autoclass:: nni.bohb_advisor.bohb_advisor.BOHB
-    :members:
+    :members:

pylintrc

@@ -28,7 +28,7 @@ enable= unused-wildcard-import,
         line-too-long,
         unused-variable,
         wildcard-import,
-        useless-super-delegation,
+#        useless-super-delegation,
         len-as-condition,
         logging-format-interpolation,
         redefined-builtin,
@@ -42,7 +42,7 @@ enable= unused-wildcard-import,
 #        too-many-branches,
 #        protected-access 
 
-ignore-patterns=test.py
+ignore-patterns=test*
 
 # List of members which are set dynamically and missed by pylint inference
 generated-members=numpy.*,torch.*

src/nni_manager/training_service/remote_machine/remoteMachineData.ts

@@ -209,7 +209,8 @@ export class SSHClientManager {
         const connectConfig: ConnectConfig = {
             host: this.rmMeta.ip,
             port: this.rmMeta.port,
-            username: this.rmMeta.username };
+            username: this.rmMeta.username,
+            tryKeyboard: true };
         if (this.rmMeta.passwd !== undefined) {
             connectConfig.password = this.rmMeta.passwd;
         } else if (this.rmMeta.sshKeyPath !== undefined) {
@@ -231,6 +232,8 @@ export class SSHClientManager {
           .on('error', (err: Error) => {
             // SSH connection error, reject with error message
             deferred.reject(new Error(err.message));
+        }).on("keyboard-interactive", (name, instructions, lang, prompts, finish) => {
+            finish([this.rmMeta.passwd]);
         })
           .connect(connectConfig);
 

src/sdk/pynni/nni/assessor.py

@@ -18,44 +18,118 @@
 # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 # ==================================================================================================
 
+"""
+Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset)
+to tell whether this trial can be early stopped or not.
+
+See :class:`Assessor`' specification and ``docs/en_US/assessors.rst`` for details.
+"""
 
-import logging
 from enum import Enum
+import logging
 
 from .recoverable import Recoverable
 
+__all__ = ['AssessResult', 'Assessor']
+
 _logger = logging.getLogger(__name__)
 
+
 class AssessResult(Enum):
+    """
+    Enum class for :meth:`Assessor.assess_trial` return value.
+    """
+
     Good = True
+    """The trial works well."""
+
     Bad = False
+    """The trial works poorly and should be early stopped."""
+
 
 class Assessor(Recoverable):
+    """
+    Assessor analyzes trial's intermediate results (e.g., periodically evaluated accuracy on test dataset)
+    to tell whether this trial can be early stopped or not.
+
+    This is the abstract base class for all assessors.
+    Early stopping algorithms should derive this class and override :meth:`assess_trial` method,
+    which receives intermediate results from trials and give an assessing result.
+
+    If :meth:`assess_trial` returns :obj:`AssessResult.Bad` for a trial,
+    it hints NNI framework that the trial is likely to result in a poor final accuracy,
+    and therefore should be killed to save resource.
+
+    If an accessor want's to get notified when a trial ends, it can also override :meth:`trial_end`.
+
+    To write a new assessor, you can reference :class:`~nni.medianstop_assessor.MedianstopAssessor`'s code as an example.
+
+    See Also
+    --------
+    Builtin assessors:
+    :class:`~nni.medianstop_assessor.MedianstopAssessor`
+    :class:`~nni.curvefitting_assessor.CurvefittingAssessor`
+    """
 
     def assess_trial(self, trial_job_id, trial_history):
-        """Determines whether a trial should be killed. Must override.
-        trial_job_id: identifier of the trial (str).
-        trial_history: a list of intermediate result objects.
-        Returns AssessResult.Good or AssessResult.Bad.
+        """
+        Abstract method for determining whether a trial should be killed. Must override.
+
+        The NNI framework has little guarantee on ``trial_history``.
+        This method is not guaranteed to be invoked for each time ``trial_history`` get updated.
+        It is also possible that a trial's history keeps updateing after receiving a bad result.
+        And if the trial failed and retried, ``trial_history`` may be inconsistent with its previous value.
+
+        The only guarantee is that ``trial_history`` is always growing.
+        It will not be empty and will always be longer than previous value.
+
+        This is an example of how :meth:`assess_trial` get invoked sequentially:
+
+        ::
+
+            trial_job_id | trial_history   | return value
+            ------------ | --------------- | ------------
+            Trial_A      | [1.0, 2.0]      | Good
+            Trial_B      | [1.5, 1.3]      | Bad
+            Trial_B      | [1.5, 1.3, 1.9] | Good
+            Trial_A      | [0.9, 1.8, 2.3] | Good
+
+        Parameters
+        ----------
+        trial_job_id: str
+            Unique identifier of the trial.
+        trial_history: list
+            Intermediate results of this trial. The element type is decided by trial code.
+
+        Returns
+        -------
+        AssessResult
+            :obj:`AssessResult.Good` or :obj:`AssessResult.Bad`.
         """
         raise NotImplementedError('Assessor: assess_trial not implemented')
 
     def trial_end(self, trial_job_id, success):
-        """Invoked when a trial is completed or terminated. Do nothing by default.
-        trial_job_id: identifier of the trial (str).
-        success: True if the trial successfully completed; False if failed or terminated.
+        """
+        Abstract method invoked when a trial is completed or terminated. Do nothing by default.
+
+        Parameters
+        ----------
+        trial_job_id: str
+            Unique identifier of the trial.
+        success: bool
+            True if the trial successfully completed; False if failed or terminated.
         """
 
     def load_checkpoint(self):
-        """Load the checkpoint of assessr.
-        path: checkpoint directory for assessor
+        """
+        Internal API under revising, not recommended for end users.
         """
         checkpoin_path = self.get_checkpoint_path()
         _logger.info('Load checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path)
 
     def save_checkpoint(self):
-        """Save the checkpoint of assessor.
-        path: checkpoint directory for assessor
+        """
+        Internal API under revising, not recommended for end users.
         """
         checkpoin_path = self.get_checkpoint_path()
         _logger.info('Save checkpoint ignored by assessor, checkpoint path: %s', checkpoin_path)

src/sdk/pynni/nni/curvefitting_assessor/__init__.py

@@ -0,0 +1 @@
+from .curvefitting_assessor import CurvefittingAssessor

src/sdk/pynni/nni/medianstop_assessor/__init__.py

@@ -0,0 +1 @@
+from .medianstop_assessor import MedianstopAssessor