Add built-in tuner test in CI

.travis.yml

@@ -14,6 +14,6 @@ install:
   - make install
   - export PATH=$HOME/.nni/bin:$PATH
 before_script:
-  - cd test/naive
+  - cd test
 script:
-  - python3 run.py
+  - source run.sh

src/sdk/pynni/nni/tuner.py

@@ -25,7 +25,7 @@
 from .recoverable import Recoverable
 
 _logger = logging.getLogger(__name__)
-
+_result = open('/tmp/nni_tuner_result.txt', 'w')
 
 class Tuner(Recoverable):
     # pylint: disable=no-self-use,unused-argument
@@ -89,7 +89,11 @@ def save_checkpoint(self):
         _logger.info('Save checkpoint ignored by tuner, checkpoint path: %s' % checkpoin_path)
 
     def _on_exit(self):
-        pass
+        """For debug's sake"""
+        _result.write('DONE\n')
+        _result.close()
 
     def _on_error(self):
-        pass
+        """For debug's sake"""
+        _result.write('ERROR\n')
+        _result.close()

test/naive/run.py

@@ -19,12 +19,12 @@ def read_last_line(file_name):
         return None
 
 def run():
-    os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD']
+    os.environ['PATH'] = os.environ['PATH'] + ':' + os.path.join(os.environ['PWD'], '..')
 
     with contextlib.suppress(FileNotFoundError):
-        os.remove('tuner_search_space.txt')
+        os.remove('/tmp/nni_tuner_search_space.txt')
     with contextlib.suppress(FileNotFoundError):
-        os.remove('tuner_result.txt')
+        os.remove('/tmp/nni_tuner_result.txt')
     with contextlib.suppress(FileNotFoundError):
         os.remove('/tmp/nni_assessor_result.txt')
 

test/run.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Integration Test
+cd naive && python3 run.py
+# Built-in Tuner Test 
+cd ../test_builtin_tuner/ && python3 run.py

test/test_builtin_tuner/local.yml

@@ -0,0 +1,16 @@
+authorName: nni
+experimentName: test_builtin_tuner
+maxExecDuration: 1h
+maxTrialNum: 2
+searchSpacePath: search_space.json
+trainingServicePlatform: local
+trial:
+  codeDir: .
+  command: python3 naive_trial.py
+  gpuNum: 0
+trialConcurrency: 2
+tuner:
+  builtinTunerName: Evolution
+  classArgs:
+    optimize_mode: maximize
+useAnnotation: false

test/test_builtin_tuner/naive_trial.py

@@ -0,0 +1,7 @@
+import nni
+
+params = nni.get_parameters()
+print('params:', params)
+x = params['x']
+
+nni.report_final_result(x)

test/test_builtin_tuner/run.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import contextlib
+import json
+import os
+import subprocess
+import time
+import traceback
+import yaml
+
+GREEN = '\33[32m'
+RED = '\33[31m'
+CLEAR = '\33[0m'
+
+TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution']
+
+def read_last_line(file_name):
+    try:
+        *_, last_line = open(file_name)
+        return last_line.strip()
+    except (FileNotFoundError, ValueError):
+        return None
+
+def get_yml_content(file_path):
+    '''Load yaml file content'''
+    with open(file_path, 'r') as file:
+        return yaml.load(file)
+
+def dump_yml_content(file_path, content):
+    '''Dump yaml file content'''
+    with open(file_path, 'w') as file:
+        file.write(yaml.dump(content, default_flow_style=False))
+
+def switch_tuner(tuner_name):
+    '''Change tuner in config.yml'''
+    config_path = 'local.yml'
+    experiment_config = get_yml_content(config_path)
+    experiment_config['tuner'] = {
+        'builtinTunerName': tuner_name,
+        'classArgs': {
+            'optimize_mode': 'maximize'
+        }
+    }
+    dump_yml_content(config_path, experiment_config)
+
+def test_builtin_tuner(tuner_name):
+    with contextlib.suppress(FileNotFoundError):
+        os.remove('/tmp/nni_tuner_result.txt')
+    switch_tuner(tuner_name)
+
+    print('Testing %s...'%tuner_name)
+    proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml'])
+    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+
+    time.sleep(16)
+    tuner_status = read_last_line('/tmp/nni_tuner_result.txt')
+
+    assert tuner_status is not None, 'Failed to finish in 16 sec'
+    assert tuner_status == 'DONE', 'Tuner exited with error'
+
+if __name__ == '__main__':
+
+    os.environ['PATH'] = os.environ['PATH'] + ':' + os.path.join(os.environ['PWD'],'..')
+    # Test each built-in tuner
+    for tuner_name in TUNER_LIST:
+        try:
+            test_builtin_tuner(tuner_name)
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR)
+        except Exception as error:
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR)
+            print('%r' % error)
+            traceback.print_exc()
+            raise error
+        finally:
+            subprocess.run(['nnictl', 'stop'])

test/test_builtin_tuner/search_space.json

@@ -0,0 +1,7 @@
+{
+    "x":
+    {
+        "_type" : "choice",
+        "_value" : [1, 100]
+    }
+}