From 17ac8cb1e63ff4d5519920fac2f6738cdab47e7b Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Tue, 19 Feb 2019 23:58:23 +0800 Subject: [PATCH 01/18] add different tuner config files for config_test --- .../tuners/auto-gbdt-metis.test.yml | 25 +++++++++++++++++ .../tuners/cifar10-network_morphism.test.yml | 28 +++++++++++++++++++ .../tuners/mnist-annotation-anneal.test.yml | 24 ++++++++++++++++ .../mnist-annotation-evolution.test.yml | 24 ++++++++++++++++ .../tuners/mnist-annotation-random.test.yml | 24 ++++++++++++++++ .../tuners/mnist-annotation-smac.test.yml | 24 ++++++++++++++++ .../tuners/mnist-annotation-tpe.test.yml | 24 ++++++++++++++++ test/config_test/tuners/mnist-batchtuner.yml | 24 ++++++++++++++++ .../tuners/mnist-gridsearch.test.yml | 23 +++++++++++++++ test/config_test/tuners/mnist-hyperband.yml | 28 +++++++++++++++++++ .../tuners/search_space_batchtuner.json | 12 ++++++++ .../tuners/search_space_gridsearch.json | 7 +++++ .../tuners/search_space_hyperband.json | 7 +++++ .../tuners/search_space_metis.json | 5 ++++ 14 files changed, 279 insertions(+) create mode 100644 test/config_test/tuners/auto-gbdt-metis.test.yml create mode 100644 test/config_test/tuners/cifar10-network_morphism.test.yml create mode 100644 test/config_test/tuners/mnist-annotation-anneal.test.yml create mode 100644 test/config_test/tuners/mnist-annotation-evolution.test.yml create mode 100644 test/config_test/tuners/mnist-annotation-random.test.yml create mode 100644 test/config_test/tuners/mnist-annotation-smac.test.yml create mode 100644 test/config_test/tuners/mnist-annotation-tpe.test.yml create mode 100644 test/config_test/tuners/mnist-batchtuner.yml create mode 100644 test/config_test/tuners/mnist-gridsearch.test.yml create mode 100644 test/config_test/tuners/mnist-hyperband.yml create mode 100644 test/config_test/tuners/search_space_batchtuner.json create mode 100644 test/config_test/tuners/search_space_gridsearch.json create mode 100644 test/config_test/tuners/search_space_hyperband.json create mode 100644 test/config_test/tuners/search_space_metis.json diff --git a/test/config_test/tuners/auto-gbdt-metis.test.yml b/test/config_test/tuners/auto-gbdt-metis.test.yml new file mode 100644 index 0000000000..e3efdbcf1c --- /dev/null +++ b/test/config_test/tuners/auto-gbdt-metis.test.yml @@ -0,0 +1,25 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 +searchSpacePath: search_space_metis.json + +tuner: + builtinTunerName: MetisTuner + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/auto-gbdt + command: python3 main.py + gpuNum: 0 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/cifar10-network_morphism.test.yml b/test/config_test/tuners/cifar10-network_morphism.test.yml new file mode 100644 index 0000000000..48d97aa0c7 --- /dev/null +++ b/test/config_test/tuners/cifar10-network_morphism.test.yml @@ -0,0 +1,28 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 15m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: NetworkMorphism + classArgs: + optimize_mode: maximize + task: cv + input_width: 32 + input_channel: 3 + n_output_node: 10 +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/network_morphism/cifar10 + command: python3 cifar10_keras.py --epochs 1 + gpuNum: 1 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-annotation-anneal.test.yml b/test/config_test/tuners/mnist-annotation-anneal.test.yml new file mode 100644 index 0000000000..f01cd1f488 --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-anneal.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: Anneal + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-annotation-evolution.test.yml b/test/config_test/tuners/mnist-annotation-evolution.test.yml new file mode 100644 index 0000000000..f2fc200888 --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-evolution.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: Evolution + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-annotation-random.test.yml b/test/config_test/tuners/mnist-annotation-random.test.yml new file mode 100644 index 0000000000..5843f1c66e --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-random.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: Random + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-annotation-smac.test.yml b/test/config_test/tuners/mnist-annotation-smac.test.yml new file mode 100644 index 0000000000..d943c9918c --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-smac.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: SMAC + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-annotation-tpe.test.yml b/test/config_test/tuners/mnist-annotation-tpe.test.yml new file mode 100644 index 0000000000..8e4a887705 --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-tpe.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-batchtuner.yml b/test/config_test/tuners/mnist-batchtuner.yml new file mode 100644 index 0000000000..6073065cef --- /dev/null +++ b/test/config_test/tuners/mnist-batchtuner.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 +searchSpacePath: search_space_batchtuner.json + +tuner: + builtinTunerName: BatchTuner +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-batch-tune-keras + command: python3 mnist-keras.py --epochs 1 + gpuNum: 0 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local + diff --git a/test/config_test/tuners/mnist-gridsearch.test.yml b/test/config_test/tuners/mnist-gridsearch.test.yml new file mode 100644 index 0000000000..570fc8e1e5 --- /dev/null +++ b/test/config_test/tuners/mnist-gridsearch.test.yml @@ -0,0 +1,23 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 +searchSpacePath: search_space_gridsearch.json + +tuner: + builtinTunerName: GridSearch +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local diff --git a/test/config_test/tuners/mnist-hyperband.yml b/test/config_test/tuners/mnist-hyperband.yml new file mode 100644 index 0000000000..fe0fd1024f --- /dev/null +++ b/test/config_test/tuners/mnist-hyperband.yml @@ -0,0 +1,28 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 +searchSpacePath: search_space_hyperband.json + +advisor: + builtinAdvisorName: Hyperband + classArgs: + optimize_mode: maximize + R: 60 + eta: 3 +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-hyperband + command: python3 mnist.py + gpuNum: 0 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local + diff --git a/test/config_test/tuners/search_space_batchtuner.json b/test/config_test/tuners/search_space_batchtuner.json new file mode 100644 index 0000000000..a0be3609da --- /dev/null +++ b/test/config_test/tuners/search_space_batchtuner.json @@ -0,0 +1,12 @@ +{ + "combine_params": + { + "_type" : "choice", + "_value" : [{"optimizer": "Adam", "learning_rate": 0.00001}, + {"optimizer": "Adam", "learning_rate": 0.0001}, + {"optimizer": "Adam", "learning_rate": 0.001}, + {"optimizer": "SGD", "learning_rate": 0.01}, + {"optimizer": "SGD", "learning_rate": 0.005}, + {"optimizer": "SGD", "learning_rate": 0.0002}] + } +} \ No newline at end of file diff --git a/test/config_test/tuners/search_space_gridsearch.json b/test/config_test/tuners/search_space_gridsearch.json new file mode 100644 index 0000000000..c081d144d1 --- /dev/null +++ b/test/config_test/tuners/search_space_gridsearch.json @@ -0,0 +1,7 @@ +{ + "dropout_rate":{"_type":"quniform","_value":[0.5, 0.9, 2]}, + "conv_size":{"_type":"choice","_value":[2,3,5,7]}, + "hidden_size":{"_type":"choice","_value":[124, 512, 1024]}, + "batch_size": {"_type":"choice", "_value": [1, 4, 8, 16, 32]}, + "learning_rate":{"_type":"choice","_value":[0.0001, 0.001, 0.01, 0.1]} +} diff --git a/test/config_test/tuners/search_space_hyperband.json b/test/config_test/tuners/search_space_hyperband.json new file mode 100644 index 0000000000..540f2708cb --- /dev/null +++ b/test/config_test/tuners/search_space_hyperband.json @@ -0,0 +1,7 @@ +{ + "dropout_rate":{"_type":"uniform","_value":[0.5,0.9]}, + "conv_size":{"_type":"choice","_value":[2,3,5,7]}, + "hidden_size":{"_type":"choice","_value":[124, 512, 1024]}, + "batch_size": {"_type":"choice","_value":[8, 16, 32, 64]}, + "learning_rate":{"_type":"choice","_value":[0.0001, 0.001, 0.01, 0.1]} +} diff --git a/test/config_test/tuners/search_space_metis.json b/test/config_test/tuners/search_space_metis.json new file mode 100644 index 0000000000..32b00f7be4 --- /dev/null +++ b/test/config_test/tuners/search_space_metis.json @@ -0,0 +1,5 @@ +{ + "num_leaves":{"_type":"choice","_value":[31, 28, 24, 20]}, + "learning_rate":{"_type":"choice","_value":[0.01, 0.05, 0.1, 0.2]}, + "bagging_freq":{"_type":"choice","_value":[1, 2, 4, 8, 10]} +} From b6f2c56d8d634b953efefc292dca96506b072f5b Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Wed, 20 Feb 2019 14:12:31 +0800 Subject: [PATCH 02/18] change MetisTuner config test due to no lightgbm python module in integration test --- test/config_test/tuners/mnist-gridsearch.test.yml | 2 +- .../{auto-gbdt-metis.test.yml => mnist-metis.test.yml} | 6 +++--- .../{search_space_gridsearch.json => search_space.json} | 0 test/config_test/tuners/search_space_metis.json | 5 ----- 4 files changed, 4 insertions(+), 9 deletions(-) rename test/config_test/tuners/{auto-gbdt-metis.test.yml => mnist-metis.test.yml} (75%) rename test/config_test/tuners/{search_space_gridsearch.json => search_space.json} (100%) delete mode 100644 test/config_test/tuners/search_space_metis.json diff --git a/test/config_test/tuners/mnist-gridsearch.test.yml b/test/config_test/tuners/mnist-gridsearch.test.yml index 570fc8e1e5..8e45db313a 100644 --- a/test/config_test/tuners/mnist-gridsearch.test.yml +++ b/test/config_test/tuners/mnist-gridsearch.test.yml @@ -3,7 +3,7 @@ experimentName: default_test maxExecDuration: 5m maxTrialNum: 2 trialConcurrency: 1 -searchSpacePath: search_space_gridsearch.json +searchSpacePath: search_space.json tuner: builtinTunerName: GridSearch diff --git a/test/config_test/tuners/auto-gbdt-metis.test.yml b/test/config_test/tuners/mnist-metis.test.yml similarity index 75% rename from test/config_test/tuners/auto-gbdt-metis.test.yml rename to test/config_test/tuners/mnist-metis.test.yml index e3efdbcf1c..6df4c51b34 100644 --- a/test/config_test/tuners/auto-gbdt-metis.test.yml +++ b/test/config_test/tuners/mnist-metis.test.yml @@ -3,7 +3,7 @@ experimentName: default_test maxExecDuration: 5m maxTrialNum: 2 trialConcurrency: 1 -searchSpacePath: search_space_metis.json +searchSpacePath: search_space.json tuner: builtinTunerName: MetisTuner @@ -14,8 +14,8 @@ assessor: classArgs: optimize_mode: maximize trial: - codeDir: ../../../examples/trials/auto-gbdt - command: python3 main.py + codeDir: ../../../examples/trials/mnist + command: python3 mnist.py --batch_num 100 gpuNum: 0 useAnnotation: false diff --git a/test/config_test/tuners/search_space_gridsearch.json b/test/config_test/tuners/search_space.json similarity index 100% rename from test/config_test/tuners/search_space_gridsearch.json rename to test/config_test/tuners/search_space.json diff --git a/test/config_test/tuners/search_space_metis.json b/test/config_test/tuners/search_space_metis.json deleted file mode 100644 index 32b00f7be4..0000000000 --- a/test/config_test/tuners/search_space_metis.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "num_leaves":{"_type":"choice","_value":[31, 28, 24, 20]}, - "learning_rate":{"_type":"choice","_value":[0.01, 0.05, 0.1, 0.2]}, - "bagging_freq":{"_type":"choice","_value":[1, 2, 4, 8, 10]} -} From b3afb3afef2ca17c16fe41d0e27f77bd92c36d65 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Wed, 20 Feb 2019 14:51:58 +0800 Subject: [PATCH 03/18] install smac package in azure-pipelines --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 74f3034097..e83d634f81 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ jobs: python3 -m pip install torchvision==0.2.1 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.10.0 --user + nnictl package install --name=SMAC displayName: 'Install dependencies for integration tests' - script: | cd test From 1b850dfbf6f907b32de700d88910ee41f78ba665 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Wed, 20 Feb 2019 16:21:39 +0800 Subject: [PATCH 04/18] SMAC need swig to be installed --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e83d634f81..c346e73b76 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,7 @@ jobs: python3 -m pip install torchvision==0.2.1 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.10.0 --user + apt-get install swig -y nnictl package install --name=SMAC displayName: 'Install dependencies for integration tests' - script: | From 130dec7e87d28ca3bb8d8f065b25aa408087e980 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Wed, 20 Feb 2019 18:28:44 +0800 Subject: [PATCH 05/18] Try to install swig from source code --- azure-pipelines.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c346e73b76..f2c9898bf0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,7 +18,11 @@ jobs: python3 -m pip install torchvision==0.2.1 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.10.0 --user - apt-get install swig -y + wget http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz + tar -xzvf swig-3.0.12.tar.gz + cd swig-3.0.12 && ./configure --prefix=/home/$USER/swig-3.0.12 + make + make install nnictl package install --name=SMAC displayName: 'Install dependencies for integration tests' - script: | From 150403453496c91dead2ddb9a19a752b03395e38 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Thu, 21 Feb 2019 10:21:01 +0800 Subject: [PATCH 06/18] remove SMAC test because the dependency can not be installed --- azure-pipelines.yml | 6 ----- .../tuners/mnist-annotation-smac.test.yml | 24 ------------------- 2 files changed, 30 deletions(-) delete mode 100644 test/config_test/tuners/mnist-annotation-smac.test.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2c9898bf0..74f3034097 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,12 +18,6 @@ jobs: python3 -m pip install torchvision==0.2.1 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.10.0 --user - wget http://prdownloads.sourceforge.net/swig/swig-3.0.12.tar.gz - tar -xzvf swig-3.0.12.tar.gz - cd swig-3.0.12 && ./configure --prefix=/home/$USER/swig-3.0.12 - make - make install - nnictl package install --name=SMAC displayName: 'Install dependencies for integration tests' - script: | cd test diff --git a/test/config_test/tuners/mnist-annotation-smac.test.yml b/test/config_test/tuners/mnist-annotation-smac.test.yml deleted file mode 100644 index d943c9918c..0000000000 --- a/test/config_test/tuners/mnist-annotation-smac.test.yml +++ /dev/null @@ -1,24 +0,0 @@ -authorName: nni -experimentName: default_test -maxExecDuration: 5m -maxTrialNum: 2 -trialConcurrency: 1 - -tuner: - builtinTunerName: SMAC - classArgs: - optimize_mode: maximize -assessor: - builtinAssessorName: Medianstop - classArgs: - optimize_mode: maximize -trial: - codeDir: ../../../examples/trials/mnist-annotation - command: python3 mnist.py --batch_num 100 - gpuNum: 0 - -useAnnotation: true -multiPhase: false -multiThread: false - -trainingServicePlatform: local From ee2630626b1e07a8508b903fcd0597d52b97aea5 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Thu, 21 Feb 2019 14:26:33 +0800 Subject: [PATCH 07/18] use sudo to install the swig --- azure-pipelines.yml | 2 ++ .../tuners/mnist-annotation-smac.test.yml | 24 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/config_test/tuners/mnist-annotation-smac.test.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 74f3034097..7e99de3489 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -18,6 +18,8 @@ jobs: python3 -m pip install torchvision==0.2.1 --user python3 -m pip install keras==2.1.6 --user python3 -m pip install tensorflow-gpu==1.10.0 --user + sudo apt-get install swig -y + nnictl package install --name=SMAC displayName: 'Install dependencies for integration tests' - script: | cd test diff --git a/test/config_test/tuners/mnist-annotation-smac.test.yml b/test/config_test/tuners/mnist-annotation-smac.test.yml new file mode 100644 index 0000000000..d943c9918c --- /dev/null +++ b/test/config_test/tuners/mnist-annotation-smac.test.yml @@ -0,0 +1,24 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 + +tuner: + builtinTunerName: SMAC + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Medianstop + classArgs: + optimize_mode: maximize +trial: + codeDir: ../../../examples/trials/mnist-annotation + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: true +multiPhase: false +multiThread: false + +trainingServicePlatform: local From fdca615c906daae93a9218e951e24d977e8479ab Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Fri, 22 Feb 2019 15:39:12 +0800 Subject: [PATCH 08/18] sleep 10s to make sure the port has been released --- test/config_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/config_test.py b/test/config_test.py index 375a18e1ca..76bd7cd33c 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -94,8 +94,8 @@ def run(args): for config_file in config_files: try: - # sleep 5 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict - time.sleep(5) + # sleep 10 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict + time.sleep(10) run_test(config_file, args.ts, args.local_gpu) print(GREEN + 'Test %s: TEST PASS' % (config_file) + CLEAR) except Exception as error: From eae62657181e236d9bd7de1c310ec44db281e99b Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Fri, 22 Feb 2019 17:43:46 +0800 Subject: [PATCH 09/18] remove tuner test for networkmorphism because it uses more than 30s to release the tcp port --- test/config_test.py | 4 +-- .../tuners/cifar10-network_morphism.test.yml | 28 ------------------- 2 files changed, 2 insertions(+), 30 deletions(-) delete mode 100644 test/config_test/tuners/cifar10-network_morphism.test.yml diff --git a/test/config_test.py b/test/config_test.py index 76bd7cd33c..375a18e1ca 100644 --- a/test/config_test.py +++ b/test/config_test.py @@ -94,8 +94,8 @@ def run(args): for config_file in config_files: try: - # sleep 10 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict - time.sleep(10) + # sleep 5 seconds here, to make sure previous stopped exp has enough time to exit to avoid port conflict + time.sleep(5) run_test(config_file, args.ts, args.local_gpu) print(GREEN + 'Test %s: TEST PASS' % (config_file) + CLEAR) except Exception as error: diff --git a/test/config_test/tuners/cifar10-network_morphism.test.yml b/test/config_test/tuners/cifar10-network_morphism.test.yml deleted file mode 100644 index 48d97aa0c7..0000000000 --- a/test/config_test/tuners/cifar10-network_morphism.test.yml +++ /dev/null @@ -1,28 +0,0 @@ -authorName: nni -experimentName: default_test -maxExecDuration: 15m -maxTrialNum: 2 -trialConcurrency: 1 - -tuner: - builtinTunerName: NetworkMorphism - classArgs: - optimize_mode: maximize - task: cv - input_width: 32 - input_channel: 3 - n_output_node: 10 -assessor: - builtinAssessorName: Medianstop - classArgs: - optimize_mode: maximize -trial: - codeDir: ../../../examples/trials/network_morphism/cifar10 - command: python3 cifar10_keras.py --epochs 1 - gpuNum: 1 - -useAnnotation: false -multiPhase: false -multiThread: false - -trainingServicePlatform: local From 6cdf26490a06add1f2c7a1c73860f0f0decad552 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Fri, 22 Feb 2019 19:59:54 +0800 Subject: [PATCH 10/18] word "down" to "done" --- examples/trials/mnist-hyperband/mnist.py | 4 ++-- examples/trials/mnist/mnist.py | 4 ++-- examples/trials/mnist/mnist_before.py | 4 ++-- tools/nni_annotation/examples/mnist_generated.py | 4 ++-- tools/nni_annotation/examples/mnist_with_annotation.py | 4 ++-- tools/nni_annotation/examples/mnist_without_annotation.py | 4 ++-- tools/nni_annotation/testcase/annotated/mnist.py | 2 +- tools/nni_annotation/testcase/usercode/mnist.py | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/trials/mnist-hyperband/mnist.py b/examples/trials/mnist-hyperband/mnist.py index 46df38224a..cc3544f2e0 100644 --- a/examples/trials/mnist-hyperband/mnist.py +++ b/examples/trials/mnist-hyperband/mnist.py @@ -149,8 +149,8 @@ def main(params): ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net diff --git a/examples/trials/mnist/mnist.py b/examples/trials/mnist/mnist.py index 0e305cf2d1..fa9dc61b4a 100644 --- a/examples/trials/mnist/mnist.py +++ b/examples/trials/mnist/mnist.py @@ -150,8 +150,8 @@ def main(params): ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net diff --git a/examples/trials/mnist/mnist_before.py b/examples/trials/mnist/mnist_before.py index c67fc01681..e597cbaf35 100644 --- a/examples/trials/mnist/mnist_before.py +++ b/examples/trials/mnist/mnist_before.py @@ -150,8 +150,8 @@ def main(params): ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net diff --git a/tools/nni_annotation/examples/mnist_generated.py b/tools/nni_annotation/examples/mnist_generated.py index aa1a3a231c..ec9c475e51 100644 --- a/tools/nni_annotation/examples/mnist_generated.py +++ b/tools/nni_annotation/examples/mnist_generated.py @@ -129,8 +129,8 @@ def main(params): Main function, build mnist network, run and send result to NNI. """ mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'], channel_2_num=params['channel_2_num'], conv_size=params['conv_size' ], hidden_size=params['hidden_size'], pool_size=params['pool_size'], diff --git a/tools/nni_annotation/examples/mnist_with_annotation.py b/tools/nni_annotation/examples/mnist_with_annotation.py index 55d09c7c27..00198b957a 100644 --- a/tools/nni_annotation/examples/mnist_with_annotation.py +++ b/tools/nni_annotation/examples/mnist_with_annotation.py @@ -175,8 +175,8 @@ def main(params): ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net diff --git a/tools/nni_annotation/examples/mnist_without_annotation.py b/tools/nni_annotation/examples/mnist_without_annotation.py index 82719e7f3b..6eeae8b85c 100644 --- a/tools/nni_annotation/examples/mnist_without_annotation.py +++ b/tools/nni_annotation/examples/mnist_without_annotation.py @@ -179,8 +179,8 @@ def main(params): ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) - print('Mnist download data down.') - logger.debug('Mnist download data down.') + print('Mnist download data done.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net diff --git a/tools/nni_annotation/testcase/annotated/mnist.py b/tools/nni_annotation/testcase/annotated/mnist.py index 113502e577..e1db356b54 100644 --- a/tools/nni_annotation/testcase/annotated/mnist.py +++ b/tools/nni_annotation/testcase/annotated/mnist.py @@ -127,7 +127,7 @@ def bias_variable(shape): def main(): data_dir = '/tmp/tensorflow/mnist/input_data' mnist = input_data.read_data_sets(data_dir, one_hot=True) - logger.debug('Mnist download data down.') + logger.debug('Mnist download data done.') mnist_network = MnistNetwork() mnist_network.build_network() logger.debug('Mnist build network done.') diff --git a/tools/nni_annotation/testcase/usercode/mnist.py b/tools/nni_annotation/testcase/usercode/mnist.py index d640ae8a19..4c50cd828e 100644 --- a/tools/nni_annotation/testcase/usercode/mnist.py +++ b/tools/nni_annotation/testcase/usercode/mnist.py @@ -137,7 +137,7 @@ def main(): # Import data data_dir= '/tmp/tensorflow/mnist/input_data' mnist = input_data.read_data_sets(data_dir, one_hot=True) - logger.debug('Mnist download data down.') + logger.debug('Mnist download data done.') # Create the model # Build the graph for the deep net From 55b5e3c2e749642a0b2425157deba0c92c2042cd Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Fri, 22 Feb 2019 20:09:54 +0800 Subject: [PATCH 11/18] add config test for Curvefitting assessor --- .../tuners/mnist-curvefitting.test.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 test/config_test/tuners/mnist-curvefitting.test.yml diff --git a/test/config_test/tuners/mnist-curvefitting.test.yml b/test/config_test/tuners/mnist-curvefitting.test.yml new file mode 100644 index 0000000000..0e41eb93d1 --- /dev/null +++ b/test/config_test/tuners/mnist-curvefitting.test.yml @@ -0,0 +1,28 @@ +authorName: nni +experimentName: default_test +maxExecDuration: 5m +maxTrialNum: 2 +trialConcurrency: 1 +searchSpacePath: search_space.json + +tuner: + builtinTunerName: TPE + classArgs: + optimize_mode: maximize +assessor: + builtinAssessorName: Curvefitting + classArgs: + epoch_num: 20 + optimize_mode: maximize + start_step: 6 + threshold: 0.95 +trial: + codeDir: ../../../examples/trials/mnist + command: python3 mnist.py --batch_num 100 + gpuNum: 0 + +useAnnotation: false +multiPhase: false +multiThread: false + +trainingServicePlatform: local \ No newline at end of file From 94430b87a049a53b6769a338397a9eb3cb9c1fcc Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Fri, 22 Feb 2019 20:24:11 +0800 Subject: [PATCH 12/18] change file name --- .../tuners/{mnist-batchtuner.yml => mnist-batchtuner.test.yml} | 0 .../tuners/{mnist-hyperband.yml => mnist-hyperband.test.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename test/config_test/tuners/{mnist-batchtuner.yml => mnist-batchtuner.test.yml} (100%) rename test/config_test/tuners/{mnist-hyperband.yml => mnist-hyperband.test.yml} (100%) diff --git a/test/config_test/tuners/mnist-batchtuner.yml b/test/config_test/tuners/mnist-batchtuner.test.yml similarity index 100% rename from test/config_test/tuners/mnist-batchtuner.yml rename to test/config_test/tuners/mnist-batchtuner.test.yml diff --git a/test/config_test/tuners/mnist-hyperband.yml b/test/config_test/tuners/mnist-hyperband.test.yml similarity index 100% rename from test/config_test/tuners/mnist-hyperband.yml rename to test/config_test/tuners/mnist-hyperband.test.yml From a43b1ba48e78451e38e4f7fc8966dd0a402da2d6 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Mon, 4 Mar 2019 17:29:35 +0800 Subject: [PATCH 13/18] Fix data type not match bug --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index f0cbed6c3a..b3d705b52b 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -245,7 +245,7 @@ def receive_trial_result(self, parameter_id, parameters, value): # calculate y aggregation median = get_median(temp_y) - self.samples_y_aggregation[idx] = median + self.samples_y_aggregation[idx] = [median] else: self.samples_x.append(sample_x) self.samples_y.append([value]) From d87ae8c0102b9733e89ba8e13f1c4bd58c544793 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Mon, 4 Mar 2019 20:51:06 +0800 Subject: [PATCH 14/18] Optimize MetisTunner --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 70 ++++++++++++-------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index b3d705b52b..fe850eebd5 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -67,25 +67,25 @@ class MetisTuner(Tuner): """ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=True, - selection_num_starting_points=10, cold_start_num=10): + selection_num_starting_points=600, cold_start_num=10): """ Parameters ---------- optimize_mode : str optimize_mode is a string that including two mode "maximize" and "minimize" - + no_resampling : bool True or False. Should Metis consider re-sampling as part of the search strategy? If you are confident that the training dataset is noise-free, then you do not need re-sampling. - + no_candidates: bool True or False. Should Metis suggest parameters for the next benchmark? If you do not plan to do more benchmarks, Metis can skip this step. - + selection_num_starting_points: int how many times Metis should try to find the global optimal in the search space? The higher the number, the longer it takes to output the solution. - + cold_start_num: int Metis need some trial result to get cold start. when the number of trial result is less than cold_start_num, Metis will randomly sample hyper-parameter for trial. @@ -94,6 +94,7 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=T self.samples_x = [] self.samples_y = [] self.samples_y_aggregation = [] + self.history_parameters = set() self.space = None self.no_resampling = no_resampling self.no_candidates = no_candidates @@ -128,7 +129,7 @@ def update_search_space(self, search_space): except Exception as ex: logger.exception(ex) raise RuntimeError("The format search space contains \ - some key that didn't define in key_order.") + some key that didn't define in key_order." ) if key_type == 'quniform': if key_range[2] == 1: @@ -191,7 +192,7 @@ def generate_parameters(self, parameter_id): Parameters ---------- parameter_id : int - + Returns ------- result : dict @@ -200,6 +201,8 @@ def generate_parameters(self, parameter_id): init_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] results = self._pack_output(init_parameter) else: + self.minimize_starting_points = _rand_init(self.x_bounds, self.x_types, \ + self.selection_num_starting_points) results = self._selection(self.samples_x, self.samples_y_aggregation, self.samples_y, self.x_bounds, self.x_types, threshold_samplessize_resampling=(None if self.no_resampling is True else 50), @@ -264,17 +267,21 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, candidates = [] samples_size_all = sum([len(i) for i in samples_y]) samples_size_unique = len(samples_y) - + # ===== STEP 1: Compute the current optimum ===== #sys.stderr.write("[%s] Predicting the optimal configuration from the current training dataset...\n" % (os.path.basename(__file__))) gp_model = gp_create_model.create_model(samples_x, samples_y_aggregation) - lm_current = gp_selection.selection("lm", samples_y_aggregation, x_bounds, - x_types, gp_model['model'], - minimize_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + lm_current = gp_selection.selection( + "lm", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) if not lm_current: return None - + if no_candidates is False: candidates.append({'hyperparameter': lm_current['hyperparameter'], 'expected_mu': lm_current['expected_mu'], @@ -284,10 +291,14 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, # ===== STEP 2: Get recommended configurations for exploration ===== #sys.stderr.write("[%s] Getting candidates for exploration...\n" #% \(os.path.basename(__file__))) - results_exploration = gp_selection.selection("lc", samples_y_aggregation, - x_bounds, x_types, gp_model['model'], - minimize_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + results_exploration = gp_selection.selection( + "lc", + samples_y_aggregation, + x_bounds, + x_types, + gp_model['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) if results_exploration is not None: if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: @@ -326,9 +337,9 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, logger.info("DEBUG: No suitable exploitation_gmm candidates were found\n") except ValueError as exception: - # The exception: ValueError: Fitting the mixture model failed - # because some components have ill-defined empirical covariance - # (for instance caused by singleton or collapsed samples). + # The exception: ValueError: Fitting the mixture model failed + # because some components have ill-defined empirical covariance + # (for instance caused by singleton or collapsed samples). # Try to decrease the number of components, or increase reg_covar. logger.info("DEBUG: No suitable exploitation_gmm candidates were found due to exception.") logger.info(exception) @@ -340,7 +351,6 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, results_outliers = gp_outlier_detection.outlierDetection_threaded(samples_x, samples_y_aggregation) if results_outliers is not None: - temp = len(candidates) for results_outlier in results_outliers: if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: @@ -357,7 +367,10 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, logger.info("Evaluating information gain of %d candidates...\n") next_improvement = 0 - threads_inputs = [[candidate, samples_x, samples_y, x_bounds, x_types, minimize_constraints_fun, minimize_starting_points] for candidate in candidates] + threads_inputs = [[ + candidate, samples_x, samples_y, x_bounds, x_types, + minimize_constraints_fun, minimize_starting_points + ] for candidate in candidates] threads_pool = ThreadPool(4) # Evaluate what would happen if we actually sample each candidate threads_results = threads_pool.map(_calculate_lowest_mu_threaded, threads_inputs) @@ -368,10 +381,10 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, if threads_result['expected_lowest_mu'] < lm_current['expected_mu']: # Information gain temp_improvement = threads_result['expected_lowest_mu'] - lm_current['expected_mu'] - + if next_improvement > temp_improvement: logger.infor("DEBUG: \"next_candidate\" changed: \ - lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ + lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ lm_current['expected_mu'], str(lm_current['hyperparameter']),\ threads_result['expected_lowest_mu'],\ str(threads_result['candidate']['hyperparameter']),\ @@ -381,8 +394,10 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, next_candidate = threads_result['candidate'] else: # ===== STEP 6: If we have no candidates, randomly pick one ===== - logger.info("DEBUG: No candidates from exploration, exploitation,\ - and resampling. We will random a candidate for next_candidate\n") + logger.info( + "DEBUG: No candidates from exploration, exploitation,\ + and resampling. We will random a candidate for next_candidate\n" + ) next_candidate = _rand_with_constraints(x_bounds, x_types) \ if minimize_starting_points is None else minimize_starting_points[0] @@ -392,6 +407,9 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, 'expected_mu': expected_mu, 'expected_sigma': expected_sigma} outputs = self._pack_output(lm_current['hyperparameter']) + if outputs in self.history_parameters: + outputs = self._pack_output(next_candidate['hyperparameter']) + self.history_parameters.add(outputs) return outputs From 74cdcf8d7b158700d238230d87fc8ae4ac6a36c2 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Mon, 4 Mar 2019 21:13:45 +0800 Subject: [PATCH 15/18] pretty the code --- .../Regression_GP/OutlierDetection.py | 26 +++++----- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 47 ++++++++++--------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py index 353c56f2b0..95a8abfdc6 100644 --- a/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py +++ b/src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py @@ -37,13 +37,13 @@ def _outlierDetection_threaded(inputs): sys.stderr.write("[%s] DEBUG: Evaluating %dth of %d samples\n"\ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) outlier = None - + # Create a diagnostic regression model which removes the sample that we want to evaluate - diagnostic_regressor_gp = gp_create_model.createModel(\ + diagnostic_regressor_gp = gp_create_model.create_model(\ samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) mu, sigma = gp_prediction.predict(samples_x[samples_idx], diagnostic_regressor_gp['model']) - + # 2.33 is the z-score for 98% confidence level if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): outlier = {"samples_idx": samples_idx, @@ -51,26 +51,26 @@ def _outlierDetection_threaded(inputs): "expected_sigma": sigma, "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)} return outlier - + def outlierDetection_threaded(samples_x, samples_y_aggregation): - ''' + ''' Use Multi-thread to detect the outlier ''' outliers = [] - + threads_inputs = [[samples_idx, samples_x, samples_y_aggregation]\ for samples_idx in range(0, len(samples_x))] threads_pool = ThreadPool(min(4, len(threads_inputs))) threads_results = threads_pool.map(_outlierDetection_threaded, threads_inputs) threads_pool.close() threads_pool.join() - + for threads_result in threads_results: if threads_result is not None: outliers.append(threads_result) else: print("error here.") - + outliers = None if len(outliers) == 0 else outliers return outliers @@ -79,21 +79,19 @@ def outlierDetection(samples_x, samples_y_aggregation): ''' outliers = [] for samples_idx in range(0, len(samples_x)): - #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" + #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) - diagnostic_regressor_gp = gp_create_model.createModel(\ + diagnostic_regressor_gp = gp_create_model.create_model(\ samples_x[0:samples_idx] + samples_x[samples_idx + 1:],\ samples_y_aggregation[0:samples_idx] + samples_y_aggregation[samples_idx + 1:]) mu, sigma = gp_prediction.predict(samples_x[samples_idx], - diagnostic_regressor_gp['model']) + diagnostic_regressor_gp['model']) # 2.33 is the z-score for 98% confidence level if abs(samples_y_aggregation[samples_idx] - mu) > (2.33 * sigma): outliers.append({"samples_idx": samples_idx, "expected_mu": mu, "expected_sigma": sigma, "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) - + outliers = None if len(outliers) == 0 else outliers return outliers - - \ No newline at end of file diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index 10f58853d5..7a5c491ef5 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -24,22 +24,20 @@ import random import statistics import sys - -import numpy as np - from enum import Enum, unique from multiprocessing.dummy import Pool as ThreadPool -from nni.tuner import Tuner +import numpy as np -import nni.metis_tuner.lib_data as lib_data import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation -import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model -import nni.metis_tuner.Regression_GP.Selection as gp_selection -import nni.metis_tuner.Regression_GP.Prediction as gp_prediction -import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection +import nni.metis_tuner.lib_data as lib_data import nni.metis_tuner.Regression_GMM.CreateModel as gmm_create_model import nni.metis_tuner.Regression_GMM.Selection as gmm_selection +import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model +import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection +import nni.metis_tuner.Regression_GP.Prediction as gp_prediction +import nni.metis_tuner.Regression_GP.Selection as gp_selection +from nni.tuner import Tuner logger = logging.getLogger("Metis_Tuner_AutoML") @@ -129,7 +127,7 @@ def update_search_space(self, search_space): except Exception as ex: logger.exception(ex) raise RuntimeError("The format search space contains \ - some key that didn't define in key_order." ) + some key that didn't define in key_order." ) if key_type == 'quniform': if key_range[2] == 1: @@ -209,7 +207,7 @@ def generate_parameters(self, parameter_id): no_candidates=self.no_candidates, minimize_starting_points=self.minimize_starting_points, minimize_constraints_fun=self.minimize_constraints_fun) - + logger.info("Generate paramageters:\n" + str(results)) return results @@ -319,12 +317,13 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, print("Getting candidates for exploitation...\n") try: gmm = gmm_create_model.create_model(samples_x, samples_y_aggregation) - results_exploitation = gmm_selection.selection(x_bounds, - x_types, - gmm['clusteringmodel_good'], - gmm['clusteringmodel_bad'], - minimize_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + results_exploitation = gmm_selection.selection( + x_bounds, + x_types, + gmm['clusteringmodel_good'], + gmm['clusteringmodel_bad'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) if results_exploitation is not None: if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: @@ -384,7 +383,7 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, if next_improvement > temp_improvement: logger.infor("DEBUG: \"next_candidate\" changed: \ - lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ + lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ lm_current['expected_mu'], str(lm_current['hyperparameter']),\ threads_result['expected_lowest_mu'],\ str(threads_result['candidate']['hyperparameter']),\ @@ -455,10 +454,14 @@ def _calculate_lowest_mu_threaded(inputs): # Aggregates multiple observation of the sample sampling points temp_y_aggregation = [statistics.median(temp_sample_y) for temp_sample_y in temp_samples_y] temp_gp = gp_create_model.create_model(temp_samples_x, temp_y_aggregation) - temp_results = gp_selection.selection("lm", temp_y_aggregation, - x_bounds, x_types, temp_gp['model'], - minimize_starting_points, - minimize_constraints_fun=minimize_constraints_fun) + temp_results = gp_selection.selection( + "lm", + temp_y_aggregation, + x_bounds, + x_types, + temp_gp['model'], + minimize_starting_points, + minimize_constraints_fun=minimize_constraints_fun) if outputs["expected_lowest_mu"] is None or outputs["expected_lowest_mu"] > temp_results['expected_mu']: outputs["expected_lowest_mu"] = temp_results['expected_mu'] From 7c999c3f856f8a26c039b632bf674702ae8ffa38 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Tue, 5 Mar 2019 11:31:15 +0800 Subject: [PATCH 16/18] Follow the review comment --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index 7a5c491ef5..de6f748f6f 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -382,7 +382,7 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, temp_improvement = threads_result['expected_lowest_mu'] - lm_current['expected_mu'] if next_improvement > temp_improvement: - logger.infor("DEBUG: \"next_candidate\" changed: \ + logger.info("DEBUG: \"next_candidate\" changed: \ lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ lm_current['expected_mu'], str(lm_current['hyperparameter']),\ threads_result['expected_lowest_mu'],\ @@ -406,6 +406,7 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, 'expected_mu': expected_mu, 'expected_sigma': expected_sigma} outputs = self._pack_output(lm_current['hyperparameter']) + # ===== STEP 7: If current optimal hyperparameter occurs in the history, take next config as exploration step ===== if outputs in self.history_parameters: outputs = self._pack_output(next_candidate['hyperparameter']) self.history_parameters.add(outputs) From dce983e165f64b849613f26dda1b5d3b12949a43 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Tue, 5 Mar 2019 12:17:12 +0800 Subject: [PATCH 17/18] add exploration probability --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index de6f748f6f..4c743651aa 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -65,7 +65,7 @@ class MetisTuner(Tuner): """ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=True, - selection_num_starting_points=600, cold_start_num=10): + selection_num_starting_points=600, cold_start_num=10, exploration_probability=0.1): """ Parameters ---------- @@ -87,6 +87,9 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=T cold_start_num: int Metis need some trial result to get cold start. when the number of trial result is less than cold_start_num, Metis will randomly sample hyper-parameter for trial. + + exploration_probability: float + The probability of Metis to select parameter from exploration instead of exploitation. """ self.samples_x = [] @@ -100,6 +103,7 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=T self.key_order = [] self.cold_start_num = cold_start_num self.selection_num_starting_points = selection_num_starting_points + self.exploration_probability = exploration_probability self.minimize_constraints_fun = None self.minimize_starting_points = None @@ -405,11 +409,12 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, next_candidate = {'hyperparameter': next_candidate, 'reason': "random", 'expected_mu': expected_mu, 'expected_sigma': expected_sigma} + # ===== STEP 7: If current optimal hyperparameter occurs in the history or exploration probability is less than the threshold, take next config as exploration step ===== outputs = self._pack_output(lm_current['hyperparameter']) - # ===== STEP 7: If current optimal hyperparameter occurs in the history, take next config as exploration step ===== - if outputs in self.history_parameters: + ap = random.uniform(0, 1) + if outputs in self.history_parameters or ap<=self.exploration_probability: outputs = self._pack_output(next_candidate['hyperparameter']) - self.history_parameters.add(outputs) + self.history_parameters.add(outputs) return outputs From 30bfa10c6c12e402099d99a9d00feabd7aeac273 Mon Sep 17 00:00:00 2001 From: Lee Joe Date: Tue, 5 Mar 2019 21:31:00 +0800 Subject: [PATCH 18/18] Avoid None type object generating --- src/sdk/pynni/nni/metis_tuner/metis_tuner.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py index 4c743651aa..bd914e20e4 100644 --- a/src/sdk/pynni/nni/metis_tuner/metis_tuner.py +++ b/src/sdk/pynni/nni/metis_tuner/metis_tuner.py @@ -95,7 +95,7 @@ def __init__(self, optimize_mode="maximize", no_resampling=True, no_candidates=T self.samples_x = [] self.samples_y = [] self.samples_y_aggregation = [] - self.history_parameters = set() + self.history_parameters = [] self.space = None self.no_resampling = no_resampling self.no_candidates = no_candidates @@ -131,7 +131,7 @@ def update_search_space(self, search_space): except Exception as ex: logger.exception(ex) raise RuntimeError("The format search space contains \ - some key that didn't define in key_order." ) + some key that didn't define in key_order." ) if key_type == 'quniform': if key_range[2] == 1: @@ -387,7 +387,7 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, if next_improvement > temp_improvement: logger.info("DEBUG: \"next_candidate\" changed: \ - lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ + lowest mu might reduce from %f (%s) to %f (%s), %s\n" %\ lm_current['expected_mu'], str(lm_current['hyperparameter']),\ threads_result['expected_lowest_mu'],\ str(threads_result['candidate']['hyperparameter']),\ @@ -413,8 +413,12 @@ def _selection(self, samples_x, samples_y_aggregation, samples_y, outputs = self._pack_output(lm_current['hyperparameter']) ap = random.uniform(0, 1) if outputs in self.history_parameters or ap<=self.exploration_probability: - outputs = self._pack_output(next_candidate['hyperparameter']) - self.history_parameters.add(outputs) + if next_candidate is not None: + outputs = self._pack_output(next_candidate['hyperparameter']) + else: + random_parameter = _rand_init(self.x_bounds, self.x_types, 1)[0] + outputs = self._pack_output(random_parameter) + self.history_parameters.append(outputs) return outputs