-
Notifications
You must be signed in to change notification settings - Fork 215
/
run_local_test.py
171 lines (148 loc) · 6.48 KB
/
run_local_test.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
################################################################################
# Name: Run Local Test Tool
# Author: Zhengying Liu
# Created on: 20 Sep 2018
# Update time: 5 May 2019
# Usage: python run_local_test.py -dataset_dir=<dataset_dir> -code_dir=<code_dir>
VERISION = "v20190505"
DESCRIPTION =\
"""This script allows participants to run local test of their method within the
downloaded starting kit folder (and avoid using submission quota on CodaLab). To
do this, run:
```
python run_local_test.py -dataset_dir=./AutoDL_sample_data/miniciao -code_dir=./AutoDL_sample_code_submission/
```
in the starting kit directory. If you want to test the performance of a
different algorithm on a different dataset, please specify them using respective
arguments.
If you want to use default folders (i.e. those in above command line), simply
run
```
python run_local_test.py
```
"""
# ALL INFORMATION, SOFTWARE, DOCUMENTATION, AND DATA ARE PROVIDED "AS-IS".
# ISABELLE GUYON, CHALEARN, AND/OR OTHER ORGANIZERS OR CODE AUTHORS DISCLAIM
# ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ANY PARTICULAR PURPOSE, AND THE
# WARRANTY OF NON-INFRINGEMENT OF ANY THIRD PARTY'S INTELLECTUAL PROPERTY RIGHTS.
# IN NO EVENT SHALL ISABELLE GUYON AND/OR OTHER ORGANIZERS BE LIABLE FOR ANY SPECIAL,
# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF SOFTWARE, DOCUMENTS, MATERIALS,
# PUBLICATIONS, OR INFORMATION MADE AVAILABLE FOR THE CHALLENGE.
################################################################################
# Verbosity level of logging.
# Can be: NOTSET, DEBUG, INFO, WARNING, ERROR, CRITICAL
verbosity_level = 'INFO'
import logging
import os
import tensorflow as tf
import time
import shutil # for deleting a whole directory
import webbrowser
from multiprocessing import Process
logging.basicConfig(
level=getattr(logging, verbosity_level),
format='%(asctime)s %(levelname)s %(filename)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S'
)
def _HERE(*args):
h = os.path.dirname(os.path.realpath(__file__))
return os.path.join(h, *args)
def get_path_to_ingestion_program(starting_kit_dir):
return os.path.join(starting_kit_dir,
'AutoDL_ingestion_program', 'ingestion.py')
def get_path_to_scoring_program(starting_kit_dir):
return os.path.join(starting_kit_dir,
'AutoDL_scoring_program', 'score.py')
def remove_dir(output_dir):
"""Remove the directory `output_dir`.
This aims to clean existing output of last run of local test.
"""
if os.path.isdir(output_dir):
logging.info("Cleaning existing output directory of last run: {}"\
.format(output_dir))
shutil.rmtree(output_dir)
def get_basename(path):
if len(path) == 0:
return ""
if path[-1] == os.sep:
path = path[:-1]
return path.split(os.sep)[-1]
def run_baseline(dataset_dir, code_dir, time_budget=1200):
logging.info("#"*50)
logging.info("Begin running local test using")
logging.info("code_dir = {}".format(get_basename(code_dir)))
logging.info("dataset_dir = {}".format(get_basename(dataset_dir)))
logging.info("#"*50)
# Current directory containing this script
starting_kit_dir = os.path.dirname(os.path.realpath(__file__))
path_ingestion = get_path_to_ingestion_program(starting_kit_dir)
path_scoring = get_path_to_scoring_program(starting_kit_dir)
# Run ingestion and scoring at the same time
command_ingestion =\
"python {} --dataset_dir={} --code_dir={} --time_budget={}"\
.format(path_ingestion, dataset_dir, code_dir, time_budget)
command_scoring =\
'python {} --solution_dir={}'\
.format(path_scoring, dataset_dir)
def run_ingestion():
print(command_ingestion)
exit_code = os.system(command_ingestion)
assert exit_code == 0
def run_scoring():
print(command_scoring)
exit_code = os.system(command_scoring)
assert exit_code == 0
ingestion_process = Process(name='ingestion', target=run_ingestion)
scoring_process = Process(name='scoring', target=run_scoring)
ingestion_output_dir = os.path.join(starting_kit_dir,
'AutoDL_sample_result_submission')
score_dir = os.path.join(starting_kit_dir,
'AutoDL_scoring_output')
remove_dir(ingestion_output_dir)
remove_dir(score_dir)
ingestion_process.start()
scoring_process.start()
detailed_results_page = os.path.join(starting_kit_dir,
'AutoDL_scoring_output',
'detailed_results.html')
detailed_results_page = os.path.abspath(detailed_results_page)
# Open detailed results page in a browser
time.sleep(2)
for i in range(30):
if os.path.isfile(detailed_results_page):
webbrowser.open('file://'+detailed_results_page, new=2)
break
time.sleep(1)
ingestion_process.join()
scoring_process.join()
if not ingestion_process.exitcode == 0:
logging.info("Some error occurred in ingestion program.")
if not scoring_process.exitcode == 0:
raise Exception("Some error occurred in scoring program.")
if __name__ == '__main__':
default_starting_kit_dir = _HERE()
# The default dataset is 'miniciao' under the folder AutoDL_sample_data/
default_dataset_dir = os.path.join(default_starting_kit_dir,
'AutoDL_sample_data', 'miniciao')
default_code_dir = os.path.join(default_starting_kit_dir,
'AutoDL_sample_code_submission')
default_time_budget = 1200
tf.flags.DEFINE_string('dataset_dir', default_dataset_dir,
"Directory containing the content (e.g. adult.data/ + "
"adult.solution) of an AutoDL dataset. Specify this "
"argument if you want to test on a different dataset.")
tf.flags.DEFINE_string('code_dir', default_code_dir,
"Directory containing a `model.py` file. Specify this "
"argument if you want to test on a different algorithm."
)
tf.flags.DEFINE_float('time_budget', default_time_budget,
"Time budget for running ingestion " +
"(training + prediction)."
)
FLAGS = tf.flags.FLAGS
dataset_dir = FLAGS.dataset_dir
code_dir = FLAGS.code_dir
time_budget = FLAGS.time_budget
run_baseline(dataset_dir, code_dir, time_budget)