From 42e25d8810e582f857ba2c5dd88fb4d9be93c8e6 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Tue, 23 Oct 2018 16:35:26 +0800 Subject: [PATCH 1/7] Rename get_parameters to get_next_parameter --- docs/CustomizedTuner.md | 2 +- docs/WriteYourTrial.md | 2 +- examples/trials/README.md | 566 +++++++++--------- examples/trials/auto-gbdt/main.py | 2 +- examples/trials/ga_squad/trial.py | 2 +- .../mnist-batch-tune-keras/mnist-keras.py | 2 +- .../mnist-cascading-search-space/mnist.py | 2 +- examples/trials/mnist-keras/mnist-keras.py | 2 +- examples/trials/mnist/mnist.py | 2 +- examples/trials/pytorch_cifar10/main.py | 2 +- .../trials/sklearn/classification/main.py | 2 +- examples/trials/sklearn/regression/main.py | 2 +- src/sdk/pynni/nni/platform/local.py | 7 +- src/sdk/pynni/nni/platform/standalone.py | 2 +- src/sdk/pynni/nni/platform/test.py | 2 +- src/sdk/pynni/nni/smartparam.py | 2 +- src/sdk/pynni/nni/trial.py | 27 +- src/sdk/pynni/tests/test_trial.py | 4 +- test/naive/naive_trial.py | 2 +- 19 files changed, 322 insertions(+), 312 deletions(-) diff --git a/docs/CustomizedTuner.md b/docs/CustomizedTuner.md index 7994a82cad..b89e261705 100644 --- a/docs/CustomizedTuner.md +++ b/docs/CustomizedTuner.md @@ -61,7 +61,7 @@ If the you implement the ```generate_parameters``` like this: # your code implements here. return {"dropout": 0.3, "learning_rate": 0.4} ``` -It's means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` this object will using ```nni.get_parameters()``` API from NNI SDK. After training of Trial, it will send result to Tuner by calling ```nni.report_final_result(0.93)```. Then ```receive_trial_result``` will function will receied these parameters like： + It's means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` by calling API ```nni.get_next_parameter()```. Once the trial ends with a result (normally some kind of metrics), it can send the result to Tuner by calling API ```nni.report_final_result()```, such as ```nni.report_final_result(0.93)```. Then your Tuner's ```receive_trial_result``` function will receied the result like： ``` parameter_id = 82347 parameters = {"dropout": 0.3, "learning_rate": 0.4} diff --git a/docs/WriteYourTrial.md b/docs/WriteYourTrial.md index 58e513c9e3..907ff5b72e 100644 --- a/docs/WriteYourTrial.md +++ b/docs/WriteYourTrial.md @@ -27,7 +27,7 @@ Refer to [SearchSpaceSpec.md](SearchSpaceSpec.md) to learn more about search spa 2.2 Get predefined parameters Use the following code snippet: - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() to get hyper-parameters' values assigned by tuner. `RECEIVED_PARAMS` is an object, for example: diff --git a/examples/trials/README.md b/examples/trials/README.md index cd636e74f9..e78715120c 100644 --- a/examples/trials/README.md +++ b/examples/trials/README.md @@ -1,284 +1,284 @@ -# How to write a Trial running on NNI? - -*Trial receive the hyper-parameter/architecture configure from Tuner, and send intermediate result to Assessor and final result to Tuner.* - -So when user want to write a Trial running on NNI, she/he should: - -**1)Have an original Trial could run**, - -Trial's code could be any machine learning code that could run in local. Here we use ```mnist-keras.py``` as example: - -```python -import argparse -import logging -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -K.set_image_data_format('channels_last') - -H, W = 28, 28 -NUM_CLASSES = 10 - -def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): - layers = [ - Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), - Conv2D(64, (3, 3), activation='relu'), - MaxPooling2D(pool_size=(2, 2)), - Flatten(), - Dense(100, activation='relu'), - Dense(num_classes, activation='softmax') - ] - - model = Sequential(layers) - - if hyper_params['optimizer'] == 'Adam': - optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) - else: - optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) - model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) - - return model - -def load_mnist_data(args): - (x_train, y_train), (x_test, y_test) = mnist.load_data() - - x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] - x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] - y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] - y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] - - return x_train, y_train, x_test, y_test - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - pass - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - -def generate_default_params(): - return { - 'optimizer': 'Adam', - 'learning_rate': 0.001 - } - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - PARAMS = generate_default_params() - train(ARGS, PARAMS) -``` - -**2)Get configure from Tuner** - -User import ```nni``` and use ```nni.get_parameters()``` to recive configure. Please noted **10**, **24** and **25** line in the following code. - - -```python -import argparse -import logging -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -import nni - -... - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - - PARAMS = generate_default_params() - RECEIVED_PARAMS = nni.get_parameters() - PARAMS.update(RECEIVED_PARAMS) - train(ARGS, PARAMS) -``` - - -**3) Send intermediate result** - -Use ```nni.report_intermediate_result``` to send intermediate result to Assessor. Please noted **5** line in the following code. - - -```python -... - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - nni.report_intermediate_result(logs) - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - -... -``` -**4) Send final result** - -Use ```nni.report_final_result``` to send final result to Trial. Please noted **15** line in the following code. - -```python -... - -class SendMetrics(keras.callbacks.Callback): - def on_epoch_end(self, epoch, logs={}): - nni.report_intermediate_result(logs) - -def train(args, params): - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - nni.report_final_result(acc) -... -``` - -Here is the complete exampe: - - -```python -import argparse -import logging - -import keras -import numpy as np -from keras import backend as K -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D -from keras.models import Sequential - -import nni - -LOG = logging.getLogger('mnist_keras') -K.set_image_data_format('channels_last') - -H, W = 28, 28 -NUM_CLASSES = 10 - -def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): - ''' - Create simple convolutional model - ''' - layers = [ - Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), - Conv2D(64, (3, 3), activation='relu'), - MaxPooling2D(pool_size=(2, 2)), - Flatten(), - Dense(100, activation='relu'), - Dense(num_classes, activation='softmax') - ] - - model = Sequential(layers) - - if hyper_params['optimizer'] == 'Adam': - optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) - else: - optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) - model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) - - return model - -def load_mnist_data(args): - ''' - Load MNIST dataset - ''' - (x_train, y_train), (x_test, y_test) = mnist.load_data() - - x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] - x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] - y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] - y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] - - LOG.debug('x_train shape: %s', (x_train.shape,)) - LOG.debug('x_test shape: %s', (x_test.shape,)) - - return x_train, y_train, x_test, y_test - -class SendMetrics(keras.callbacks.Callback): - ''' - Keras callback to send metrics to NNI framework - ''' - def on_epoch_end(self, epoch, logs={}): - ''' - Run on end of each epoch - ''' - LOG.debug(logs) - nni.report_intermediate_result(logs) - -def train(args, params): - ''' - Train model - ''' - x_train, y_train, x_test, y_test = load_mnist_data(args) - model = create_mnist_model(params) - - model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, - validation_data=(x_test, y_test), callbacks=[SendMetrics()]) - - _, acc = model.evaluate(x_test, y_test, verbose=0) - LOG.debug('Final result is: %d', acc) - nni.report_final_result(acc) - -def generate_default_params(): - ''' - Generate default hyper parameters - ''' - return { - 'optimizer': 'Adam', - 'learning_rate': 0.001 - } - -if __name__ == '__main__': - PARSER = argparse.ArgumentParser() - PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) - PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) - PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) - PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) - - ARGS, UNKNOWN = PARSER.parse_known_args() - - try: - # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() - LOG.debug(RECEIVED_PARAMS) - PARAMS = generate_default_params() - PARAMS.update(RECEIVED_PARAMS) - # train - train(ARGS, PARAMS) - except Exception as e: - LOG.exception(e) - raise - +# How to write a Trial running on NNI? + +*Trial receive the hyper-parameter/architecture configure from Tuner, and send intermediate result to Assessor and final result to Tuner.* + +So when user want to write a Trial running on NNI, she/he should: + +**1)Have an original Trial could run**, + +Trial's code could be any machine learning code that could run in local. Here we use ```mnist-keras.py``` as example: + +```python +import argparse +import logging +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +K.set_image_data_format('channels_last') + +H, W = 28, 28 +NUM_CLASSES = 10 + +def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): + layers = [ + Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), + Conv2D(64, (3, 3), activation='relu'), + MaxPooling2D(pool_size=(2, 2)), + Flatten(), + Dense(100, activation='relu'), + Dense(num_classes, activation='softmax') + ] + + model = Sequential(layers) + + if hyper_params['optimizer'] == 'Adam': + optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) + else: + optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) + model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) + + return model + +def load_mnist_data(args): + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] + x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] + y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] + y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] + + return x_train, y_train, x_test, y_test + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + pass + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + +def generate_default_params(): + return { + 'optimizer': 'Adam', + 'learning_rate': 0.001 + } + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + PARAMS = generate_default_params() + train(ARGS, PARAMS) +``` + +**2)Get configure from Tuner** + +User import ```nni``` and use ```nni.get_next_parameter()``` to recive configure. Please noted **10**, **24** and **25** line in the following code. + + +```python +import argparse +import logging +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +import nni + +... + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + + PARAMS = generate_default_params() + RECEIVED_PARAMS = nni.get_next_parameter() + PARAMS.update(RECEIVED_PARAMS) + train(ARGS, PARAMS) +``` + + +**3) Send intermediate result** + +Use ```nni.report_intermediate_result``` to send intermediate result to Assessor. Please noted **5** line in the following code. + + +```python +... + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + nni.report_intermediate_result(logs) + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + +... +``` +**4) Send final result** + +Use ```nni.report_final_result``` to send final result to Trial. Please noted **15** line in the following code. + +```python +... + +class SendMetrics(keras.callbacks.Callback): + def on_epoch_end(self, epoch, logs={}): + nni.report_intermediate_result(logs) + +def train(args, params): + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + nni.report_final_result(acc) +... +``` + +Here is the complete exampe: + + +```python +import argparse +import logging + +import keras +import numpy as np +from keras import backend as K +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Flatten, MaxPooling2D +from keras.models import Sequential + +import nni + +LOG = logging.getLogger('mnist_keras') +K.set_image_data_format('channels_last') + +H, W = 28, 28 +NUM_CLASSES = 10 + +def create_mnist_model(hyper_params, input_shape=(H, W, 1), num_classes=NUM_CLASSES): + ''' + Create simple convolutional model + ''' + layers = [ + Conv2D(32, kernel_size=(3, 3), activation='relu', input_shape=input_shape), + Conv2D(64, (3, 3), activation='relu'), + MaxPooling2D(pool_size=(2, 2)), + Flatten(), + Dense(100, activation='relu'), + Dense(num_classes, activation='softmax') + ] + + model = Sequential(layers) + + if hyper_params['optimizer'] == 'Adam': + optimizer = keras.optimizers.Adam(lr=hyper_params['learning_rate']) + else: + optimizer = keras.optimizers.SGD(lr=hyper_params['learning_rate'], momentum=0.9) + model.compile(loss=keras.losses.categorical_crossentropy, optimizer=optimizer, metrics=['accuracy']) + + return model + +def load_mnist_data(args): + ''' + Load MNIST dataset + ''' + (x_train, y_train), (x_test, y_test) = mnist.load_data() + + x_train = (np.expand_dims(x_train, -1).astype(np.float) / 255.)[:args.num_train] + x_test = (np.expand_dims(x_test, -1).astype(np.float) / 255.)[:args.num_test] + y_train = keras.utils.to_categorical(y_train, NUM_CLASSES)[:args.num_train] + y_test = keras.utils.to_categorical(y_test, NUM_CLASSES)[:args.num_test] + + LOG.debug('x_train shape: %s', (x_train.shape,)) + LOG.debug('x_test shape: %s', (x_test.shape,)) + + return x_train, y_train, x_test, y_test + +class SendMetrics(keras.callbacks.Callback): + ''' + Keras callback to send metrics to NNI framework + ''' + def on_epoch_end(self, epoch, logs={}): + ''' + Run on end of each epoch + ''' + LOG.debug(logs) + nni.report_intermediate_result(logs) + +def train(args, params): + ''' + Train model + ''' + x_train, y_train, x_test, y_test = load_mnist_data(args) + model = create_mnist_model(params) + + model.fit(x_train, y_train, batch_size=args.batch_size, epochs=args.epochs, verbose=1, + validation_data=(x_test, y_test), callbacks=[SendMetrics()]) + + _, acc = model.evaluate(x_test, y_test, verbose=0) + LOG.debug('Final result is: %d', acc) + nni.report_final_result(acc) + +def generate_default_params(): + ''' + Generate default hyper parameters + ''' + return { + 'optimizer': 'Adam', + 'learning_rate': 0.001 + } + +if __name__ == '__main__': + PARSER = argparse.ArgumentParser() + PARSER.add_argument("--batch_size", type=int, default=200, help="batch size", required=False) + PARSER.add_argument("--epochs", type=int, default=10, help="Train epochs", required=False) + PARSER.add_argument("--num_train", type=int, default=1000, help="Number of train samples to be used, maximum 60000", required=False) + PARSER.add_argument("--num_test", type=int, default=1000, help="Number of test samples to be used, maximum 10000", required=False) + + ARGS, UNKNOWN = PARSER.parse_known_args() + + try: + # get parameters from tuner + RECEIVED_PARAMS = nni.get_next_parameter() + LOG.debug(RECEIVED_PARAMS) + PARAMS = generate_default_params() + PARAMS.update(RECEIVED_PARAMS) + # train + train(ARGS, PARAMS) + except Exception as e: + LOG.exception(e) + raise + ``` \ No newline at end of file diff --git a/examples/trials/auto-gbdt/main.py b/examples/trials/auto-gbdt/main.py index 85489a312b..ce8abe4e27 100644 --- a/examples/trials/auto-gbdt/main.py +++ b/examples/trials/auto-gbdt/main.py @@ -97,7 +97,7 @@ def run(lgb_train, lgb_eval, params, X_test, y_test): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/ga_squad/trial.py b/examples/trials/ga_squad/trial.py index b96805c9c7..4dbfdc6b30 100644 --- a/examples/trials/ga_squad/trial.py +++ b/examples/trials/ga_squad/trial.py @@ -436,7 +436,7 @@ def load_data(): qp_pairs, dev_qp_pairs = load_data() logger.debug('Init finish.') - original_params = nni.get_parameters() + original_params = nni.get_next_parameter() ''' with open('data.json') as f: original_params = json.load(f) diff --git a/examples/trials/mnist-batch-tune-keras/mnist-keras.py b/examples/trials/mnist-batch-tune-keras/mnist-keras.py index 87c2114991..133a52b25a 100644 --- a/examples/trials/mnist-batch-tune-keras/mnist-keras.py +++ b/examples/trials/mnist-batch-tune-keras/mnist-keras.py @@ -122,7 +122,7 @@ def generate_default_params(): try: # get parameters from tuner # RECEIVED_PARAMS = {"optimizer": "Adam", "learning_rate": 0.00001} - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/mnist-cascading-search-space/mnist.py b/examples/trials/mnist-cascading-search-space/mnist.py index bd6dd35a5c..8b4aacd9b9 100644 --- a/examples/trials/mnist-cascading-search-space/mnist.py +++ b/examples/trials/mnist-cascading-search-space/mnist.py @@ -149,7 +149,7 @@ def parse_init_json(data): if __name__ == '__main__': try: # get parameters form tuner - data = nni.get_parameters() + data = nni.get_next_parameter() logger.debug(data) RCV_PARAMS = parse_init_json(data) diff --git a/examples/trials/mnist-keras/mnist-keras.py b/examples/trials/mnist-keras/mnist-keras.py index a21d002841..27e26e152b 100644 --- a/examples/trials/mnist-keras/mnist-keras.py +++ b/examples/trials/mnist-keras/mnist-keras.py @@ -120,7 +120,7 @@ def generate_default_params(): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = generate_default_params() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/mnist/mnist.py b/examples/trials/mnist/mnist.py index 36f4bfe910..d5c6347b5a 100644 --- a/examples/trials/mnist/mnist.py +++ b/examples/trials/mnist/mnist.py @@ -219,7 +219,7 @@ def generate_default_params(): if __name__ == '__main__': try: # get parameters form tuner - RCV_PARAMS = nni.get_parameters() + RCV_PARAMS = nni.get_next_parameter() logger.debug(RCV_PARAMS) # run params = generate_default_params() diff --git a/examples/trials/pytorch_cifar10/main.py b/examples/trials/pytorch_cifar10/main.py index 1b1ec7b8e1..42e836fb8e 100644 --- a/examples/trials/pytorch_cifar10/main.py +++ b/examples/trials/pytorch_cifar10/main.py @@ -175,7 +175,7 @@ def test(epoch): if __name__ == '__main__': try: - RCV_CONFIG = nni.get_parameters() + RCV_CONFIG = nni.get_next_parameter() #RCV_CONFIG = {'lr': 0.1, 'optimizer': 'Adam', 'model':'senet18'} _logger.debug(RCV_CONFIG) diff --git a/examples/trials/sklearn/classification/main.py b/examples/trials/sklearn/classification/main.py index 537849d5bf..92bdd8219d 100644 --- a/examples/trials/sklearn/classification/main.py +++ b/examples/trials/sklearn/classification/main.py @@ -71,7 +71,7 @@ def run(X_train, X_test, y_train, y_test, PARAMS): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/examples/trials/sklearn/regression/main.py b/examples/trials/sklearn/regression/main.py index 0a8876887f..1e290f21df 100644 --- a/examples/trials/sklearn/regression/main.py +++ b/examples/trials/sklearn/regression/main.py @@ -90,7 +90,7 @@ def run(X_train, X_test, y_train, y_test, PARAMS): try: # get parameters from tuner - RECEIVED_PARAMS = nni.get_parameters() + RECEIVED_PARAMS = nni.get_next_parameter() LOG.debug(RECEIVED_PARAMS) PARAMS = get_default_parameters() PARAMS.update(RECEIVED_PARAMS) diff --git a/src/sdk/pynni/nni/platform/local.py b/src/sdk/pynni/nni/platform/local.py index e6da1d0126..2bfec144d1 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/src/sdk/pynni/nni/platform/local.py @@ -49,13 +49,16 @@ def request_next_parameter(): }) send_metric(metric) -def get_parameters(): +def get_next_parameter(): global _param_index params_file_name = '' if _multiphase and (_multiphase == 'true' or _multiphase == 'True'): params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] else: - params_file_name = 'parameter.cfg' + if _param_index > 0: + return None + else: + params_file_name = 'parameter.cfg' params_filepath = os.path.join(_sysdir, params_file_name) if not os.path.isfile(params_filepath): diff --git a/src/sdk/pynni/nni/platform/standalone.py b/src/sdk/pynni/nni/platform/standalone.py index 9fa1e947e5..f1236f61ea 100644 --- a/src/sdk/pynni/nni/platform/standalone.py +++ b/src/sdk/pynni/nni/platform/standalone.py @@ -22,7 +22,7 @@ import json_tricks -def get_parameters(): +def get_next_parameter(): pass def get_sequence_id(): diff --git a/src/sdk/pynni/nni/platform/test.py b/src/sdk/pynni/nni/platform/test.py index 8f896e09cf..1a87de5e2c 100644 --- a/src/sdk/pynni/nni/platform/test.py +++ b/src/sdk/pynni/nni/platform/test.py @@ -29,7 +29,7 @@ _last_metric = None -def get_parameters(): +def get_next_parameter(): return _params def send_metric(string): diff --git a/src/sdk/pynni/nni/smartparam.py b/src/sdk/pynni/nni/smartparam.py index ca035be575..87ca91b8f8 100644 --- a/src/sdk/pynni/nni/smartparam.py +++ b/src/sdk/pynni/nni/smartparam.py @@ -126,4 +126,4 @@ def _get_param(func, name): if name is None: name = '__line{:d}'.format(lineno) key = '{}/{}/{}'.format(module, name, func) - return trial.get_parameter(key) + return trial.get_current_parameter(key) diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index cbfd85e85a..555fc429e0 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -26,7 +26,8 @@ __all__ = [ - 'get_parameters', + 'get_next_parameter', + 'get_current_parameter', 'report_intermediate_result', 'report_final_result', 'get_sequence_id' @@ -37,15 +38,21 @@ _sequence_id = platform.get_sequence_id() -def get_parameters(): +def get_next_parameter(tag=None): """Returns a set of (hyper-)paremeters generated by Tuner.""" global _params - _params = platform.get_parameters() - return _params['parameters'] - - -def get_parameter(tag): - return get_parameters()[tag] + _params = platform.get_next_parameter() + if _params is None: + return None + if tag is None: + return _params['parameters'] + else: + return _params['parameters'][tag] + +def get_current_parameter(tag): + if _params is None: + return None + return _params['parameters'][tag] def get_sequence_id(): return _sequence_id @@ -57,7 +64,7 @@ def report_intermediate_result(metric): metric: serializable object. """ global _intermediate_seq - assert _params is not None, 'nni.get_parameters() needs to be called before report_intermediate_result' + assert _params is not None, 'nni.get_next_parameter() needs to be called before report_intermediate_result' metric = json_tricks.dumps({ 'parameter_id': _params['parameter_id'], 'trial_job_id': env_args.trial_job_id, @@ -73,7 +80,7 @@ def report_final_result(metric): """Reports final result to tuner. metric: serializable object. """ - assert _params is not None, 'nni.get_parameters() needs to be called before report_final_result' + assert _params is not None, 'nni.get_next_parameter() needs to be called before report_final_result' metric = json_tricks.dumps({ 'parameter_id': _params['parameter_id'], 'trial_job_id': env_args.trial_job_id, diff --git a/src/sdk/pynni/tests/test_trial.py b/src/sdk/pynni/tests/test_trial.py index de3bb2b77a..f7f854123b 100644 --- a/src/sdk/pynni/tests/test_trial.py +++ b/src/sdk/pynni/tests/test_trial.py @@ -32,8 +32,8 @@ def setUp(self): self._trial_params = { 'msg': 'hi', 'x': 123, 'dict': { 'key': 'value', 'y': None } } nni.trial._params = { 'parameter_id': 'test_param', 'parameters': self._trial_params } - def test_get_parameters(self): - self.assertEqual(nni.get_parameters(), self._trial_params) + def test_get_next_parameter(self): + self.assertEqual(nni.get_next_parameter(), self._trial_params) def test_report_intermediate_result(self): nni.report_intermediate_result(123) diff --git a/test/naive/naive_trial.py b/test/naive/naive_trial.py index 1512e9c72c..ce8b14fafe 100644 --- a/test/naive/naive_trial.py +++ b/test/naive/naive_trial.py @@ -2,7 +2,7 @@ import nni -params = nni.get_parameters() +params = nni.get_next_parameter() print('params:', params) x = params['x'] From 14deef9efcb4610d9582ae6d45fb18efc6b39d83 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Thu, 25 Oct 2018 16:38:13 +0800 Subject: [PATCH 2/7] annotations add get_next_parameter --- examples/trials/mnist-annotation/mnist.py | 2 ++ tools/nni_annotation/code_generator.py | 5 +++-- tools/nni_annotation/examples/mnist_with_annotation.py | 1 + tools/nni_annotation/testcase/annotated/mnist.py | 1 + tools/nni_annotation/testcase/usercode/mnist.py | 1 + 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/trials/mnist-annotation/mnist.py b/examples/trials/mnist-annotation/mnist.py index f99a7cd323..d7c97fafa5 100644 --- a/examples/trials/mnist-annotation/mnist.py +++ b/examples/trials/mnist-annotation/mnist.py @@ -4,6 +4,7 @@ import math import tempfile import tensorflow as tf +import nni from tensorflow.examples.tutorials.mnist import input_data @@ -229,6 +230,7 @@ def generate_defualt_params(): if __name__ == '__main__': + '''@nni.get_next_parameter()''' try: main(generate_defualt_params()) except Exception as exception: diff --git a/tools/nni_annotation/code_generator.py b/tools/nni_annotation/code_generator.py index b1ca3fc87b..215bbf4cde 100644 --- a/tools/nni_annotation/code_generator.py +++ b/tools/nni_annotation/code_generator.py @@ -196,8 +196,9 @@ def _visit_string(self, node): else: return node # not an annotation, ignore it - if string.startswith('@nni.report_intermediate_result(') \ - or string.startswith('@nni.report_final_result('): + if string.startswith('@nni.report_intermediate_result(') \ + or string.startswith('@nni.report_final_result(') \ + or string.startswith('@nni.get_next_parameter('): return parse_annotation(string[1:]) # expand annotation string to code if string.startswith('@nni.variable(') \ diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/tools/nni_annotation/examples/mnist_with_annotation.py index f1dea8e051..871c998af5 100644 --- a/tools/nni_annotation/examples/mnist_with_annotation.py +++ b/tools/nni_annotation/examples/mnist_with_annotation.py @@ -247,6 +247,7 @@ def generate_defualt_params(): if __name__ == '__main__': + """@nni.next_parameter""" try: main(generate_defualt_params()) except Exception as exception: diff --git a/tools/nni_annotation/testcase/annotated/mnist.py b/tools/nni_annotation/testcase/annotated/mnist.py index edcf118023..c8303f1a2c 100644 --- a/tools/nni_annotation/testcase/annotated/mnist.py +++ b/tools/nni_annotation/testcase/annotated/mnist.py @@ -161,6 +161,7 @@ def generate_default_params(): if __name__ == '__main__': + nni.get_next_parameter() try: params = generate_default_params() logger.debug('params') diff --git a/tools/nni_annotation/testcase/usercode/mnist.py b/tools/nni_annotation/testcase/usercode/mnist.py index 55a51db116..d640ae8a19 100644 --- a/tools/nni_annotation/testcase/usercode/mnist.py +++ b/tools/nni_annotation/testcase/usercode/mnist.py @@ -198,6 +198,7 @@ def generate_default_params(): #original_params = parse_init_json(FLAGS.init_file_path, {}) #pipe_interface.set_params_to_env() + '''@nni.get_next_parameter()''' try: params = generate_default_params() logger.debug('params') From 434e0f7f4e2f16e9f49fddcf8b96163d8984823a Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Thu, 25 Oct 2018 16:43:36 +0800 Subject: [PATCH 3/7] updates --- examples/trials/mnist-annotation/mnist.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/trials/mnist-annotation/mnist.py b/examples/trials/mnist-annotation/mnist.py index d7c97fafa5..69ef283336 100644 --- a/examples/trials/mnist-annotation/mnist.py +++ b/examples/trials/mnist-annotation/mnist.py @@ -4,7 +4,6 @@ import math import tempfile import tensorflow as tf -import nni from tensorflow.examples.tutorials.mnist import input_data From cec6a6199c42c24014790bb3f246c7d022128c7c Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Thu, 25 Oct 2018 16:53:10 +0800 Subject: [PATCH 4/7] updates --- src/sdk/pynni/nni/trial.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/sdk/pynni/nni/trial.py b/src/sdk/pynni/nni/trial.py index 555fc429e0..35d0397795 100644 --- a/src/sdk/pynni/nni/trial.py +++ b/src/sdk/pynni/nni/trial.py @@ -38,16 +38,13 @@ _sequence_id = platform.get_sequence_id() -def get_next_parameter(tag=None): +def get_next_parameter(): """Returns a set of (hyper-)paremeters generated by Tuner.""" global _params _params = platform.get_next_parameter() if _params is None: return None - if tag is None: - return _params['parameters'] - else: - return _params['parameters'][tag] + return _params['parameters'] def get_current_parameter(tag): if _params is None: From bc6c6785d23d8c1745c5ce87a75a19f4a30613a5 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Fri, 26 Oct 2018 14:35:20 +0800 Subject: [PATCH 5/7] updates --- docs/AnnotationSpec.md | 13 ++++++++----- docs/howto_2_CustomizedTuner.md | 2 +- src/sdk/pynni/nni/platform/local.py | 4 +++- .../examples/mnist_with_annotation.py | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/AnnotationSpec.md b/docs/AnnotationSpec.md index 5383e3cc24..cf2fb43727 100644 --- a/docs/AnnotationSpec.md +++ b/docs/AnnotationSpec.md @@ -4,23 +4,26 @@ For good user experience and reduce user effort, we need to design a good annota If users use NNI system, they only need to: - 1. Annotation variable in code as: + 1. Use nni.next_parameter() to retrieve hyper parameters from Tuner, before using other annotation, use following annotation the begining of trial code: + '''@nni.next_parameter()''' + + 2. Annotation variable in code as: '''@nni.variable(nni.choice(2,3,5,7),name=self.conv_size)''' - 2. Annotation intermediate in code as: + 3. Annotation intermediate in code as: '''@nni.report_intermediate_result(test_acc)''' - 3. Annotation output in code as: + 4. Annotation output in code as: '''@nni.report_final_result(test_acc)''' - 4. Annotation `function_choice` in code as: + 5. Annotation `function_choice` in code as: '''@nni.function_choice(max_pool(h_conv1, self.pool_size),avg_pool(h_conv1, self.pool_size),name=max_pool)''' -In this way, they can easily realize automatic tuning on NNI. +In this way, they can easily implement automatic tuning on NNI. For `@nni.variable`, `nni.choice` is the type of search space and there are 10 types to express your search space as follows: diff --git a/docs/howto_2_CustomizedTuner.md b/docs/howto_2_CustomizedTuner.md index b89e261705..862df6885d 100644 --- a/docs/howto_2_CustomizedTuner.md +++ b/docs/howto_2_CustomizedTuner.md @@ -61,7 +61,7 @@ If the you implement the ```generate_parameters``` like this: # your code implements here. return {"dropout": 0.3, "learning_rate": 0.4} ``` - It's means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` by calling API ```nni.get_next_parameter()```. Once the trial ends with a result (normally some kind of metrics), it can send the result to Tuner by calling API ```nni.report_final_result()```, such as ```nni.report_final_result(0.93)```. Then your Tuner's ```receive_trial_result``` function will receied the result like： + It means your Tuner will always generate parameters ```{"dropout": 0.3, "learning_rate": 0.4}```. Then Trial will receive ```{"dropout": 0.3, "learning_rate": 0.4}``` by calling API ```nni.get_next_parameter()```. Once the trial ends with a result (normally some kind of metrics), it can send the result to Tuner by calling API ```nni.report_final_result()```, for example ```nni.report_final_result(0.93)```. Then your Tuner's ```receive_trial_result``` function will receied the result like： ``` parameter_id = 82347 parameters = {"dropout": 0.3, "learning_rate": 0.4} diff --git a/src/sdk/pynni/nni/platform/local.py b/src/sdk/pynni/nni/platform/local.py index 2bfec144d1..032c18e71e 100644 --- a/src/sdk/pynni/nni/platform/local.py +++ b/src/sdk/pynni/nni/platform/local.py @@ -57,8 +57,10 @@ def get_next_parameter(): else: if _param_index > 0: return None - else: + elif _param_index == 0: params_file_name = 'parameter.cfg' + else: + raise AssertionError('_param_index value ({}) should >=0'.format(_param_index)) params_filepath = os.path.join(_sysdir, params_file_name) if not os.path.isfile(params_filepath): diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/tools/nni_annotation/examples/mnist_with_annotation.py index 871c998af5..6cd5688c53 100644 --- a/tools/nni_annotation/examples/mnist_with_annotation.py +++ b/tools/nni_annotation/examples/mnist_with_annotation.py @@ -247,7 +247,7 @@ def generate_defualt_params(): if __name__ == '__main__': - """@nni.next_parameter""" + """@nni.next_parameter()""" try: main(generate_defualt_params()) except Exception as exception: From 22ad29cdfb1bb3960b606e4c25b34f0d8805ad4d Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Fri, 26 Oct 2018 19:18:15 +0800 Subject: [PATCH 6/7] updates --- tools/nni_annotation/examples/mnist_with_annotation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/tools/nni_annotation/examples/mnist_with_annotation.py index 6cd5688c53..55d09c7c27 100644 --- a/tools/nni_annotation/examples/mnist_with_annotation.py +++ b/tools/nni_annotation/examples/mnist_with_annotation.py @@ -247,7 +247,7 @@ def generate_defualt_params(): if __name__ == '__main__': - """@nni.next_parameter()""" + """@nni.get_next_parameter()""" try: main(generate_defualt_params()) except Exception as exception: From 520a1e78fc196ff521bd3d07ed0a01ddb68f7853 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Fri, 26 Oct 2018 19:20:42 +0800 Subject: [PATCH 7/7] updates --- docs/AnnotationSpec.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/AnnotationSpec.md b/docs/AnnotationSpec.md index cf2fb43727..62d2c60392 100644 --- a/docs/AnnotationSpec.md +++ b/docs/AnnotationSpec.md @@ -4,8 +4,8 @@ For good user experience and reduce user effort, we need to design a good annota If users use NNI system, they only need to: - 1. Use nni.next_parameter() to retrieve hyper parameters from Tuner, before using other annotation, use following annotation the begining of trial code: - '''@nni.next_parameter()''' + 1. Use nni.get_next_parameter() to retrieve hyper parameters from Tuner, before using other annotation, use following annotation at the begining of trial code: + '''@nni.get_next_parameter()''' 2. Annotation variable in code as: