Add builtin tuner to CI (#247)

* update Makefile

* update Makefile

* add builtin-tuner test

* add builtin-tuner test

* refractor ci

* update azure.yml

* add built-in tuner test

* fix bugs

azure-pipelines.yml

@@ -12,7 +12,12 @@ steps:
       source install.sh
     displayName: 'Install dependencies'
   - script: |
-      cd test/naive
+      cd test
       export PATH=$HOME/.local/bin:$PATH
-      python3 run.py
-    displayName: 'Run tests'
+      python3 naive_test.py
+    displayName: 'Integration tests'
+  - script: |
+      cd test
+      export PATH=$HOME/.local/bin:$PATH
+      python3 sdk_tuner_test.py
+    displayName: 'Built-in tuner tests'

test/naive_test.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+
+import json
+import subprocess
+import sys
+import time
+import traceback
+
+from utils import check_experiment_status, fetch_experiment_config, read_last_line, remove_files, setup_experiment
+
+GREEN = '\33[32m'
+RED = '\33[31m'
+CLEAR = '\33[0m'
+
+EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
+
+def run(installed = True):
+
+    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
+    to_remove = list(map(lambda file: 'naive_test/' + file, to_remove))
+    remove_files(to_remove)
+
+    proc = subprocess.run(['nnictl', 'create', '--config', 'naive_test/local.yml'])
+    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+
+    print('Spawning trials...')
+
+    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+    current_trial = 0
+
+    for _ in range(60):
+        time.sleep(1)
+
+        tuner_status = read_last_line('naive_test/tuner_result.txt')
+        assessor_status = read_last_line('naive_test/assessor_result.txt')
+        experiment_status = check_experiment_status(nnimanager_log_path)
+
+        assert tuner_status != 'ERROR', 'Tuner exited with error'
+        assert assessor_status != 'ERROR', 'Assessor exited with error'
+
+        if experiment_status:
+            break
+
+        if tuner_status is not None:
+            for line in open('naive_test/tuner_result.txt'):
+                if line.strip() == 'ERROR':
+                    break
+                trial = int(line.split(' ')[0])
+                if trial > current_trial:
+                    current_trial = trial
+                    print('Trial #%d done' % trial)
+
+    assert experiment_status, 'Failed to finish in 1 min'
+
+    ss1 = json.load(open('naive_test/search_space.json'))
+    ss2 = json.load(open('naive_test/tuner_search_space.json'))
+    assert ss1 == ss2, 'Tuner got wrong search space'
+
+    tuner_result = set(open('naive_test/tuner_result.txt'))
+    expected = set(open('naive_test/expected_tuner_result.txt'))
+    # Trials may complete before NNI gets assessor's result,
+    # so it is possible to have more final result than expected
+    assert tuner_result.issuperset(expected), 'Bad tuner result'
+
+    assessor_result = set(open('naive_test/assessor_result.txt'))
+    expected = set(open('naive_test/expected_assessor_result.txt'))
+    assert assessor_result == expected, 'Bad assessor result'
+
+if __name__ == '__main__':
+    installed = (sys.argv[-1] != '--preinstall')
+    setup_experiment(installed)
+    try:
+        run()
+        # TODO: check the output of rest server
+        print(GREEN + 'PASS' + CLEAR)
+    except Exception as error:
+        print(RED + 'FAIL' + CLEAR)
+        print('%r' % error)
+        traceback.print_exc()
+        sys.exit(1)
+    finally:
+        subprocess.run(['nnictl', 'stop'])

test/naive/README.md → test/naive_test/README.md

@@ -1,9 +1,9 @@
 ## Usage
 
 * To test before installing:
-`./run.py --preinstall`
+`python3 run.py --preinstall`
 * To test the integrity of installation:
-`./run.py`
+`python3 run.py`
 * It will print `PASS` in green eventually if everything works well.
 
 ## Details

test/sdk_tuner_test.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+import time
+import traceback
+
+from utils import *
+
+GREEN = '\33[32m'
+RED = '\33[31m'
+CLEAR = '\33[0m'
+
+TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution']
+EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment'
+
+
+def switch_tuner(tuner_name):
+    '''Change tuner in config.yml'''
+    config_path = 'sdk_tuner_test/local.yml'
+    experiment_config = get_yml_content(config_path)
+    experiment_config['tuner'] = {
+        'builtinTunerName': tuner_name,
+        'classArgs': {
+            'optimize_mode': 'maximize'
+        }
+    }
+    dump_yml_content(config_path, experiment_config)
+
+def test_builtin_tuner(tuner_name):
+    remove_files(['sdk_tuner_test/nni_tuner_result.txt'])
+    switch_tuner(tuner_name)
+
+    print('Testing %s...'%tuner_name)
+    proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_tuner_test/local.yml'])
+    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
+
+    nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL)
+
+    for _ in range(10):
+        time.sleep(3)
+
+        # check if tuner exists with error
+        tuner_status = read_last_line('tuner_result.txt')
+        assert tuner_status != 'ERROR', 'Tuner exited with error'
+
+        # check if experiment is done
+        experiment_status = check_experiment_status(nnimanager_log_path)
+        if experiment_status:
+            break
+
+    assert experiment_status, 'Failed to finish in 30 sec'
+
+def run():
+    to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt']
+    remove_files(to_remove)
+
+    for tuner_name in TUNER_LIST:
+        try:
+            test_builtin_tuner(tuner_name)
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR)
+        except Exception as error:
+            print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR)
+            print('%r' % error)
+            traceback.print_exc()
+            raise error
+        finally:
+            subprocess.run(['nnictl', 'stop'])
+
+
+if __name__ == '__main__':
+    installed = (sys.argv[-1] != '--preinstall')
+    setup_experiment(installed)
+
+    run()

test/sdk_tuner_test/local.yml

@@ -0,0 +1,16 @@
+authorName: nni
+experimentName: test_builtin_tuner
+maxExecDuration: 1h
+maxTrialNum: 2
+searchSpacePath: search_space.json
+trainingServicePlatform: local
+trial:
+  codeDir: .
+  command: python3 naive_trial.py
+  gpuNum: 0
+trialConcurrency: 2
+tuner:
+  builtinTunerName: Evolution
+  classArgs:
+    optimize_mode: maximize
+useAnnotation: false

test/sdk_tuner_test/naive_trial.py

@@ -0,0 +1,7 @@
+import nni
+
+params = nni.get_parameters()
+print('params:', params)
+x = params['x']
+
+nni.report_final_result(x)

test/sdk_tuner_test/search_space.json

@@ -0,0 +1,7 @@
+{
+    "x":
+    {
+        "_type" : "choice",
+        "_value" : [1, 100]
+    }
+}