Skip to content

Commit

Permalink
changed template test
Browse files Browse the repository at this point in the history
  • Loading branch information
@wedeling
wedeling committed Mar 2, 2023
1 parent 629f7eb commit 4e194e0
Show file tree
Hide file tree
Showing 3 changed files with 245 additions and 10 deletions.
0 tests/grid_search/test_grid.py → tests/grid_search/test_grid.template
View file
File renamed without changes.
2 changes: 1 addition & 1 deletion tests/test_grid_sampler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def campaign():
params["x2"] = {"type": "boolean", "default": True}

# python file is its own template
encoder = uq.encoders.GenericEncoder('tests/grid_search/test_grid.py',
encoder = uq.encoders.GenericEncoder('tests/grid_search/test_grid.template',
target_filename='test_grid.py')

execute = ExecuteLocal("python3 test_grid.py")
Expand Down
253 changes: 244 additions & 9 deletions tutorials/hyperparameter_tuning_tutorial_with_fabsim.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 10,
"id": "7b9cb1b8",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -396,8 +396,30 @@
"2023-03-02 11:36:59.710945: W tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:265] failed call to cuInit: UNKNOWN ERROR (303)\n",
"2023-03-02 11:36:59.710971: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (wouter-XPS-13-7390): /proc/driver/nvidia/version does not exist\n",
"2023-03-02 11:36:59.711346: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
"2023-03-02 11:37:11.878783: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
"2023-03-02 11:37:12.045043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
"2023-03-02 11:37:12.045066: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n",
"2023-03-02 11:37:12.946743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
"2023-03-02 11:37:12.946813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
"2023-03-02 11:37:12.946822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
"2023-03-02 11:37:14.633522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory\n",
"2023-03-02 11:37:14.633546: W tensorflow/compiler/xla/stream_executor/cuda/cuda_driver.cc:265] failed call to cuInit: UNKNOWN ERROR (303)\n",
"2023-03-02 11:37:14.633564: I tensorflow/compiler/xla/stream_executor/cuda/cuda_diagnostics.cc:156] kernel driver does not appear to be running on this host (wouter-XPS-13-7390): /proc/driver/nvidia/version does not exist\n",
"2023-03-02 11:37:14.633830: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\n",
"To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
]
},
{
"data": {
"text/plain": [
"True"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
Expand All @@ -414,21 +436,38 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 11,
"id": "9d2c0ddb",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Executing fabsim localhost fetch_results\n",
"Executing fabsim localhost verify_last_ensemble:grid_search,campaign_dir=/tmp/grid_testrebm6ntq,target_filename=output.csv,machine=localhost\n"
]
}
],
"source": [
"# check if all output files are retrieved from the remote machine, returns a Boolean flag\n",
"all_good = fab.verify(CONFIG, campaign.campaign_dir, TARGET_FILENAME, machine=MACHINE)"
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 12,
"id": "c2b9838b",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Executing fabsim localhost get_uq_samples:grid_search,campaign_dir=/tmp/grid_testrebm6ntq,number_of_samples=6,skip=0\n"
]
}
],
"source": [
"if all_good:\n",
" # copy the results from the FabSim results dir to the EasyVVUQ results dir\n",
Expand Down Expand Up @@ -469,10 +508,148 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 13,
"id": "0b55725a",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th>run_id</th>\n",
" <th>iteration</th>\n",
" <th>n_neurons</th>\n",
" <th>learning_rate</th>\n",
" <th>dropout_prob_in</th>\n",
" <th>dropout_prob_hidden</th>\n",
" <th>accuracy_train</th>\n",
" <th>accuracy_test</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>64</td>\n",
" <td>0.005</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.959267</td>\n",
" <td>0.9544</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>2</td>\n",
" <td>0</td>\n",
" <td>64</td>\n",
" <td>0.010</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.974133</td>\n",
" <td>0.9653</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3</td>\n",
" <td>0</td>\n",
" <td>64</td>\n",
" <td>0.015</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.979717</td>\n",
" <td>0.9712</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>4</td>\n",
" <td>0</td>\n",
" <td>128</td>\n",
" <td>0.005</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.963333</td>\n",
" <td>0.9592</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>5</td>\n",
" <td>0</td>\n",
" <td>128</td>\n",
" <td>0.010</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.978667</td>\n",
" <td>0.9718</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>6</td>\n",
" <td>0</td>\n",
" <td>128</td>\n",
" <td>0.015</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0.983650</td>\n",
" <td>0.9744</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" run_id iteration n_neurons learning_rate dropout_prob_in \\\n",
" 0 0 0 0 0 \n",
"0 1 0 64 0.005 0.0 \n",
"1 2 0 64 0.010 0.0 \n",
"2 3 0 64 0.015 0.0 \n",
"3 4 0 128 0.005 0.0 \n",
"4 5 0 128 0.010 0.0 \n",
"5 6 0 128 0.015 0.0 \n",
"\n",
" dropout_prob_hidden accuracy_train accuracy_test \n",
" 0 0 0 \n",
"0 0.0 0.959267 0.9544 \n",
"1 0.0 0.974133 0.9653 \n",
"2 0.0 0.979717 0.9712 \n",
"3 0.0 0.963333 0.9592 \n",
"4 0.0 0.978667 0.9718 \n",
"5 0.0 0.983650 0.9744 "
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#############################################\n",
"# All output files are present, decode them #\n",
Expand Down Expand Up @@ -509,10 +686,68 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 14,
"id": "99ba74e2",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Best hyperparameters with 97.44% test accuracy:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th>n_neurons</th>\n",
" <th>learning_rate</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>0</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>128</td>\n",
" <td>0.015</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" n_neurons learning_rate\n",
" 0 0\n",
"5 128 0.015"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"print(\"Best hyperparameters with %.2f%% test accuracy:\" % (data_frame['accuracy_test'].max().values * 100,))\n",
"data_frame.loc[data_frame['accuracy_test'].idxmax()][vary.keys()]"
Expand Down

0 comments on commit 4e194e0

Please sign in to comment.