From 07d51dd06932c1c220122351ddef6dbd98e68c0f Mon Sep 17 00:00:00 2001 From: Zejun Lin <871886504@qq.com> Date: Tue, 23 Oct 2018 16:18:37 +0800 Subject: [PATCH] Add builtin tuner to CI (#247) * update Makefile * update Makefile * add builtin-tuner test * add builtin-tuner test * refractor ci * update azure.yml * add built-in tuner test * fix bugs --- azure-pipelines.yml | 11 +- test/{naive => }/.gitignore | 0 test/naive/run.py | 121 ------------------ test/naive_test.py | 82 ++++++++++++ test/{naive => naive_test}/README.md | 4 +- .../expected_assessor_result.txt | 0 .../expected_tuner_result.txt | 0 test/{naive => naive_test}/local.yml | 0 test/{naive => naive_test}/naive_assessor.py | 0 test/{naive => naive_test}/naive_trial.py | 0 test/{naive => naive_test}/naive_tuner.py | 0 test/{naive => naive_test}/search_space.json | 0 test/{naive => }/nnictl | 0 test/{naive => }/nnimanager | 0 test/sdk_tuner_test.py | 75 +++++++++++ test/sdk_tuner_test/local.yml | 16 +++ test/sdk_tuner_test/naive_trial.py | 7 + test/sdk_tuner_test/search_space.json | 7 + test/utils.py | 58 +++++++++ 19 files changed, 255 insertions(+), 126 deletions(-) rename test/{naive => }/.gitignore (100%) delete mode 100644 test/naive/run.py create mode 100644 test/naive_test.py rename test/{naive => naive_test}/README.md (95%) rename test/{naive => naive_test}/expected_assessor_result.txt (100%) rename test/{naive => naive_test}/expected_tuner_result.txt (100%) rename test/{naive => naive_test}/local.yml (100%) rename test/{naive => naive_test}/naive_assessor.py (100%) rename test/{naive => naive_test}/naive_trial.py (100%) rename test/{naive => naive_test}/naive_tuner.py (100%) rename test/{naive => naive_test}/search_space.json (100%) rename test/{naive => }/nnictl (100%) mode change 100755 => 100644 rename test/{naive => }/nnimanager (100%) mode change 100755 => 100644 create mode 100644 test/sdk_tuner_test.py create mode 100644 test/sdk_tuner_test/local.yml create mode 100644 test/sdk_tuner_test/naive_trial.py create mode 100644 test/sdk_tuner_test/search_space.json create mode 100644 test/utils.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b8c27a7232..fb4f53e8b6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,7 +12,12 @@ steps: source install.sh displayName: 'Install dependencies' - script: | - cd test/naive + cd test export PATH=$HOME/.local/bin:$PATH - python3 run.py - displayName: 'Run tests' + python3 naive_test.py + displayName: 'Integration tests' + - script: | + cd test + export PATH=$HOME/.local/bin:$PATH + python3 sdk_tuner_test.py + displayName: 'Built-in tuner tests' diff --git a/test/naive/.gitignore b/test/.gitignore similarity index 100% rename from test/naive/.gitignore rename to test/.gitignore diff --git a/test/naive/run.py b/test/naive/run.py deleted file mode 100644 index cbdf7e8b60..0000000000 --- a/test/naive/run.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 - -import contextlib -import json -import os -import subprocess -import requests -import sys -import time -import traceback - -GREEN = '\33[32m' -RED = '\33[31m' -CLEAR = '\33[0m' - -class Integration_test(): - def __init__(self): - self.experiment_url = 'http://localhost:8080/api/v1/nni/experiment' - self.experiment_id = None - self.experiment_done_signal = '"Experiment done"' - - def read_last_line(self, file_name): - try: - *_, last_line = open(file_name) - return last_line.strip() - except (FileNotFoundError, ValueError): - return None - - def fetch_experiment_config(self): - experiment_profile = requests.get(self.experiment_url) - self.experiment_id = json.loads(experiment_profile.text)['id'] - self.experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', self.experiment_id) - self.nnimanager_log_path = os.path.join(self.experiment_path, 'log', 'nnimanager.log') - - def check_experiment_status(self): - assert os.path.exists(self.nnimanager_log_path), 'Experiment starts failed' - cmds = ['cat', self.nnimanager_log_path, '|', 'grep', self.experiment_done_signal] - completed_process = subprocess.run(' '.join(cmds), shell = True) - - return completed_process.returncode == 0 - - def remove_files(self, file_list): - for file_path in file_list: - with contextlib.suppress(FileNotFoundError): - os.remove(file_path) - - def run(self, installed = True): - if not installed: - os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] - sdk_path = os.path.abspath('../../src/sdk/pynni') - cmd_path = os.path.abspath('../../tools') - pypath = os.environ.get('PYTHONPATH') - if pypath: - pypath = ':'.join([pypath, sdk_path, cmd_path]) - else: - pypath = ':'.join([sdk_path, cmd_path]) - os.environ['PYTHONPATH'] = pypath - - to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] - self.remove_files(to_remove) - - proc = subprocess.run(['nnictl', 'create', '--config', 'local.yml']) - assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode - - print('Spawning trials...') - time.sleep(1) - self.fetch_experiment_config() - current_trial = 0 - - for _ in range(100): - time.sleep(1) - - tuner_status = self.read_last_line('tuner_result.txt') - assessor_status = self.read_last_line('assessor_result.txt') - experiment_status = self.check_experiment_status() - - assert tuner_status != 'ERROR', 'Tuner exited with error' - assert assessor_status != 'ERROR', 'Assessor exited with error' - - if experiment_status: - break - - if tuner_status is not None: - for line in open('tuner_result.txt'): - if line.strip() == 'ERROR': - break - trial = int(line.split(' ')[0]) - if trial > current_trial: - current_trial = trial - print('Trial #%d done' % trial) - - assert experiment_status, 'Failed to finish in 100 sec' - - ss1 = json.load(open('search_space.json')) - ss2 = json.load(open('tuner_search_space.json')) - assert ss1 == ss2, 'Tuner got wrong search space' - - tuner_result = set(open('tuner_result.txt')) - expected = set(open('expected_tuner_result.txt')) - # Trials may complete before NNI gets assessor's result, - # so it is possible to have more final result than expected - assert tuner_result.issuperset(expected), 'Bad tuner result' - - assessor_result = set(open('assessor_result.txt')) - expected = set(open('expected_assessor_result.txt')) - assert assessor_result == expected, 'Bad assessor result' - -if __name__ == '__main__': - installed = (sys.argv[-1] != '--preinstall') - ci = Integration_test() - try: - ci.run(installed) - # TODO: check the output of rest server - print(GREEN + 'PASS' + CLEAR) - except Exception as error: - print(RED + 'FAIL' + CLEAR) - print('%r' % error) - traceback.print_exc() - sys.exit(1) - finally: - subprocess.run(['nnictl', 'stop']) diff --git a/test/naive_test.py b/test/naive_test.py new file mode 100644 index 0000000000..7381072f7c --- /dev/null +++ b/test/naive_test.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +import json +import subprocess +import sys +import time +import traceback + +from utils import check_experiment_status, fetch_experiment_config, read_last_line, remove_files, setup_experiment + +GREEN = '\33[32m' +RED = '\33[31m' +CLEAR = '\33[0m' + +EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment' + +def run(installed = True): + + to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] + to_remove = list(map(lambda file: 'naive_test/' + file, to_remove)) + remove_files(to_remove) + + proc = subprocess.run(['nnictl', 'create', '--config', 'naive_test/local.yml']) + assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode + + print('Spawning trials...') + + nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL) + current_trial = 0 + + for _ in range(60): + time.sleep(1) + + tuner_status = read_last_line('naive_test/tuner_result.txt') + assessor_status = read_last_line('naive_test/assessor_result.txt') + experiment_status = check_experiment_status(nnimanager_log_path) + + assert tuner_status != 'ERROR', 'Tuner exited with error' + assert assessor_status != 'ERROR', 'Assessor exited with error' + + if experiment_status: + break + + if tuner_status is not None: + for line in open('naive_test/tuner_result.txt'): + if line.strip() == 'ERROR': + break + trial = int(line.split(' ')[0]) + if trial > current_trial: + current_trial = trial + print('Trial #%d done' % trial) + + assert experiment_status, 'Failed to finish in 1 min' + + ss1 = json.load(open('naive_test/search_space.json')) + ss2 = json.load(open('naive_test/tuner_search_space.json')) + assert ss1 == ss2, 'Tuner got wrong search space' + + tuner_result = set(open('naive_test/tuner_result.txt')) + expected = set(open('naive_test/expected_tuner_result.txt')) + # Trials may complete before NNI gets assessor's result, + # so it is possible to have more final result than expected + assert tuner_result.issuperset(expected), 'Bad tuner result' + + assessor_result = set(open('naive_test/assessor_result.txt')) + expected = set(open('naive_test/expected_assessor_result.txt')) + assert assessor_result == expected, 'Bad assessor result' + +if __name__ == '__main__': + installed = (sys.argv[-1] != '--preinstall') + setup_experiment(installed) + try: + run() + # TODO: check the output of rest server + print(GREEN + 'PASS' + CLEAR) + except Exception as error: + print(RED + 'FAIL' + CLEAR) + print('%r' % error) + traceback.print_exc() + sys.exit(1) + finally: + subprocess.run(['nnictl', 'stop']) diff --git a/test/naive/README.md b/test/naive_test/README.md similarity index 95% rename from test/naive/README.md rename to test/naive_test/README.md index 9c9fbc9222..eefc56655e 100644 --- a/test/naive/README.md +++ b/test/naive_test/README.md @@ -1,9 +1,9 @@ ## Usage * To test before installing: -`./run.py --preinstall` +`python3 run.py --preinstall` * To test the integrity of installation: -`./run.py` +`python3 run.py` * It will print `PASS` in green eventually if everything works well. ## Details diff --git a/test/naive/expected_assessor_result.txt b/test/naive_test/expected_assessor_result.txt similarity index 100% rename from test/naive/expected_assessor_result.txt rename to test/naive_test/expected_assessor_result.txt diff --git a/test/naive/expected_tuner_result.txt b/test/naive_test/expected_tuner_result.txt similarity index 100% rename from test/naive/expected_tuner_result.txt rename to test/naive_test/expected_tuner_result.txt diff --git a/test/naive/local.yml b/test/naive_test/local.yml similarity index 100% rename from test/naive/local.yml rename to test/naive_test/local.yml diff --git a/test/naive/naive_assessor.py b/test/naive_test/naive_assessor.py similarity index 100% rename from test/naive/naive_assessor.py rename to test/naive_test/naive_assessor.py diff --git a/test/naive/naive_trial.py b/test/naive_test/naive_trial.py similarity index 100% rename from test/naive/naive_trial.py rename to test/naive_test/naive_trial.py diff --git a/test/naive/naive_tuner.py b/test/naive_test/naive_tuner.py similarity index 100% rename from test/naive/naive_tuner.py rename to test/naive_test/naive_tuner.py diff --git a/test/naive/search_space.json b/test/naive_test/search_space.json similarity index 100% rename from test/naive/search_space.json rename to test/naive_test/search_space.json diff --git a/test/naive/nnictl b/test/nnictl old mode 100755 new mode 100644 similarity index 100% rename from test/naive/nnictl rename to test/nnictl diff --git a/test/naive/nnimanager b/test/nnimanager old mode 100755 new mode 100644 similarity index 100% rename from test/naive/nnimanager rename to test/nnimanager diff --git a/test/sdk_tuner_test.py b/test/sdk_tuner_test.py new file mode 100644 index 0000000000..f08822dc61 --- /dev/null +++ b/test/sdk_tuner_test.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import time +import traceback + +from utils import * + +GREEN = '\33[32m' +RED = '\33[31m' +CLEAR = '\33[0m' + +TUNER_LIST = ['TPE', 'Random', 'Anneal', 'Evolution'] +EXPERIMENT_URL = 'http://localhost:8080/api/v1/nni/experiment' + + +def switch_tuner(tuner_name): + '''Change tuner in config.yml''' + config_path = 'sdk_tuner_test/local.yml' + experiment_config = get_yml_content(config_path) + experiment_config['tuner'] = { + 'builtinTunerName': tuner_name, + 'classArgs': { + 'optimize_mode': 'maximize' + } + } + dump_yml_content(config_path, experiment_config) + +def test_builtin_tuner(tuner_name): + remove_files(['sdk_tuner_test/nni_tuner_result.txt']) + switch_tuner(tuner_name) + + print('Testing %s...'%tuner_name) + proc = subprocess.run(['nnictl', 'create', '--config', 'sdk_tuner_test/local.yml']) + assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode + + nnimanager_log_path = fetch_experiment_config(EXPERIMENT_URL) + + for _ in range(10): + time.sleep(3) + + # check if tuner exists with error + tuner_status = read_last_line('tuner_result.txt') + assert tuner_status != 'ERROR', 'Tuner exited with error' + + # check if experiment is done + experiment_status = check_experiment_status(nnimanager_log_path) + if experiment_status: + break + + assert experiment_status, 'Failed to finish in 30 sec' + +def run(): + to_remove = ['tuner_search_space.json', 'tuner_result.txt', 'assessor_result.txt'] + remove_files(to_remove) + + for tuner_name in TUNER_LIST: + try: + test_builtin_tuner(tuner_name) + print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST PASS' + CLEAR) + except Exception as error: + print(GREEN + 'Test ' +tuner_name+ ' tuner: TEST FAIL' + CLEAR) + print('%r' % error) + traceback.print_exc() + raise error + finally: + subprocess.run(['nnictl', 'stop']) + + +if __name__ == '__main__': + installed = (sys.argv[-1] != '--preinstall') + setup_experiment(installed) + + run() diff --git a/test/sdk_tuner_test/local.yml b/test/sdk_tuner_test/local.yml new file mode 100644 index 0000000000..798957a2c8 --- /dev/null +++ b/test/sdk_tuner_test/local.yml @@ -0,0 +1,16 @@ +authorName: nni +experimentName: test_builtin_tuner +maxExecDuration: 1h +maxTrialNum: 2 +searchSpacePath: search_space.json +trainingServicePlatform: local +trial: + codeDir: . + command: python3 naive_trial.py + gpuNum: 0 +trialConcurrency: 2 +tuner: + builtinTunerName: Evolution + classArgs: + optimize_mode: maximize +useAnnotation: false diff --git a/test/sdk_tuner_test/naive_trial.py b/test/sdk_tuner_test/naive_trial.py new file mode 100644 index 0000000000..d8cfbc2682 --- /dev/null +++ b/test/sdk_tuner_test/naive_trial.py @@ -0,0 +1,7 @@ +import nni + +params = nni.get_parameters() +print('params:', params) +x = params['x'] + +nni.report_final_result(x) diff --git a/test/sdk_tuner_test/search_space.json b/test/sdk_tuner_test/search_space.json new file mode 100644 index 0000000000..f20e76e0c5 --- /dev/null +++ b/test/sdk_tuner_test/search_space.json @@ -0,0 +1,7 @@ +{ + "x": + { + "_type" : "choice", + "_value" : [1, 100] + } +} diff --git a/test/utils.py b/test/utils.py new file mode 100644 index 0000000000..802a23dd30 --- /dev/null +++ b/test/utils.py @@ -0,0 +1,58 @@ +import contextlib +import json +import os +import subprocess +import requests +import traceback +import yaml + +EXPERIMENT_DONE_SIGNAL = '"Experiment done"' + +def read_last_line(file_name): + try: + *_, last_line = open(file_name) + return last_line.strip() + except (FileNotFoundError, ValueError): + return None + +def remove_files(file_list): + for file_path in file_list: + with contextlib.suppress(FileNotFoundError): + os.remove(file_path) + +def get_yml_content(file_path): + '''Load yaml file content''' + with open(file_path, 'r') as file: + return yaml.load(file) + +def dump_yml_content(file_path, content): + '''Dump yaml file content''' + with open(file_path, 'w') as file: + file.write(yaml.dump(content, default_flow_style=False)) + +def setup_experiment(installed = True): + if not installed: + os.environ['PATH'] = os.environ['PATH'] + ':' + os.environ['PWD'] + sdk_path = os.path.abspath('../src/sdk/pynni') + cmd_path = os.path.abspath('../tools') + pypath = os.environ.get('PYTHONPATH') + if pypath: + pypath = ':'.join([pypath, sdk_path, cmd_path]) + else: + pypath = ':'.join([sdk_path, cmd_path]) + os.environ['PYTHONPATH'] = pypath + +def fetch_experiment_config(experiment_url): + experiment_profile = requests.get(experiment_url) + experiment_id = json.loads(experiment_profile.text)['id'] + experiment_path = os.path.join(os.environ['HOME'], 'nni/experiments', experiment_id) + nnimanager_log_path = os.path.join(experiment_path, 'log', 'nnimanager.log') + + return nnimanager_log_path + +def check_experiment_status(nnimanager_log_path): + assert os.path.exists(nnimanager_log_path), 'Experiment starts failed' + cmds = ['cat', nnimanager_log_path, '|', 'grep', EXPERIMENT_DONE_SIGNAL] + completed_process = subprocess.run(' '.join(cmds), shell = True) + + return completed_process.returncode == 0 \ No newline at end of file