Skip to content

Commit

Permalink
progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@jgfouca
jgfouca committed Aug 25, 2016
1 parent bae3a8c commit cbc8a68
Show file tree
Hide file tree
Showing 5 changed files with 203 additions and 203 deletions.
267 changes: 168 additions & 99 deletions scripts/create_test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,31 @@ logger = logging.getLogger(__name__)
###############################################################################
def parse_command_line(args, description):
###############################################################################
parser = argparse.ArgumentParser(
usage="""\n%s <TEST|SUITE> [<TEST|SUITE> ...] [--verbose]
OR
model = CIME.utils.get_model()

if model == "cesm":
help_str = \
"""
%s --xml-category [CATEGORY] [--xml-machine ...] [--xml-compiler ...] [ --xml-testlist ...]
OR
%s --help
OR
%s --test
\033[1mEXAMPLES:\033[0m
\033[1;32m# Run all tests in the xml prealpha category and yellowstone machine \033[0m
> %s --xml-machine yellowstone --xml-category prealpha
""" % ((os.path.basename(args[0]), ) * 4)

else:
help_str = \
"""
%s <TEST|SUITE> [<TEST|SUITE> ...] [--verbose]
OR
%s --help
OR
%s --test
\033[1mEXAMPLES:\033[0m
\033[1;32m# Run single test \033[0m
> %s <TESTNAME>
Expand All @@ -50,23 +66,14 @@ OR
\033[1;32m# Run all tests in a suite except for tests that are in another suite \033[0m
> %s <SUITE1> ^<SUITE2>
""" % ((os.path.basename(args[0]), ) * 9)

\033[1;32m# Run all tests in the xml prealpha category and yellowstone machine \033[0m
> %s --xml-machine yellowstone --xml-category prealpha
""" % ((os.path.basename(args[0]), ) * 11),

description=description,

formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser = argparse.ArgumentParser(usage=help_str,
description=description,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)

CIME.utils.setup_standard_logging_options(parser)

parser.add_argument("testargs", nargs="*",
help="Tests or test suites to run."
" Testnames expect in form CASE.GRID.COMPSET")

parser.add_argument("--no-run", action="store_true",
help="Do not run generated tests")

Expand Down Expand Up @@ -104,32 +111,59 @@ OR
"all object executables, and data files will"
" be removed after tests are run")

parser.add_argument("-c", "--compare", const=True, nargs="?",
help="While testing, compare baselines,"
" optionally provide a compare directory ")
if model == "cesm":
parser.add_argument("-c", "--compare",
help="While testing, compare baselines"
" against the given compare directory ")

parser.add_argument("-g", "--generate",
help="While testing, generate baselines,"
" to the given generate directory")

parser.add_argument("--xml-machine",
help="Use this machine key in the lookup in testlist.xml, default is all if any --xml- argument is used")

parser.add_argument("--xml-compiler",
help="Use this compiler key in the lookup in testlist.xml, default is all if any --xml- argument is used")

parser.add_argument("--xml-category",
help="Use this category key in the lookup in testlist.xml, default is all if any --xml- argument is used")

parser.add_argument("--xml-testlist",
help="Use this testlist to lookup tests, default specified in config_files.xml")

parser.add_argument("testargs", nargs="*",
help="Tests or test suites to run."
" Testnames expect in form CASE.GRID.COMPSET[.MACHINE_COMPILER]")

parser.add_argument("-m", "--machine",
help="The machine for which to build tests, this machine must be defined"
" in the config_machines.xml file for the given model. "
"Default is to match the name of the machine in the test name or "
"the name of the machine this script is run on to the "
"NODENAME_REGEX field in config_machines.xml")

else:

parser.add_argument("-g", "--generate", const=True, nargs="?",
help="While testing, generate baselines,"
" optionally provide a generate directory")
parser.add_argument("testargs", nargs="+",
help="Tests or test suites to run."
" Testnames expect in form CASE.GRID.COMPSET[.MACHINE_COMPILER]")

parser.add_argument("-b", "--baseline-name",
help="If comparing or generating baselines with default paths,"
" use this directory under baseline root. "
"Default will be current branch name. Do NOT add the compiler to"
" this argument that will be done for you. If you provide directories"
" in the generate and or compare argument do not use this option.")
parser.add_argument("-b", "--baseline-name",
help="If comparing or generating baselines,"
" use this directory under baseline root. "
"Default will be current branch name.")

parser.add_argument("-c", "--compare", action="store_true",
help="While testing, compare baselines")

parser.add_argument("-g", "--generate", action="store_true",
help="While testing, generate baselines")

parser.add_argument("--compiler",
help="Compiler to use to build cime. Default will be the name in"
" the Testnames or the default defined for the machine.")

parser.add_argument("-m", "--machine",
help="The machine for which to build tests, this machine must be defined"
" in the config_machines.xml file for the given model. "
"Default is to match the name of the machine in the test name or "
"the name of the machine this script is run on to the "
"NODENAME_REGEX field in config_machines.xml")

parser.add_argument("-n", "--namelists-only", action="store_true",
help="Only perform namelist actions for tests")

Expand Down Expand Up @@ -159,44 +193,43 @@ OR
parser.add_argument("-q", "--queue", default=None,
help="Force batch system to use a certain queue")

parser.add_argument("--xml-machine",
help="Use this machine key in the lookup in testlist.xml, default is all if any --xml- argument is used")
parser.add_argument("--xml-compiler",
help="Use this compiler key in the lookup in testlist.xml, default is all if any --xml- argument is used")
parser.add_argument("--xml-category",
help="Use this category key in the lookup in testlist.xml, default is all if any --xml- argument is used")
parser.add_argument("--xml-testlist",
help="Use this testlist to lookup tests, default specified in config_files.xml")
parser.add_argument("--testfile",
parser.add_argument("-f", "--testfile",
help="A file containing an ascii list of tests to run")

parser.add_argument("-o", "--allow-baseline-overwrite", action="store_true",
help="If the generate baseline option is specified with a baseline directory option "
"existing tests in that directory will not be overwritten unless this flag is provided.")
"existing tests in that directory will not be overwritten unless this flag is provided."
"You can use bless-test-results to bless baselines from a recent run.")

args = parser.parse_args(args[1:])

CIME.utils.handle_standard_logging_options(args)

# generate and compare flags may not point to the same directory
if args.generate is not None:
expect(not (args.generate == args.compare),
"Cannot generate and compare baselines at the same time")
expect(not (args.baseline_name is not None and (not args.compare and not args.generate)),
"Provided baseline name but did not specify compare or generate")
if model == "cesm":
if args.generate is not None:
expect(not (args.generate == args.compare),
"Cannot generate and compare baselines at the same time")

if args.xml_testlist is not None:
expect(not (args.xml_machine is None and args.xml_compiler
is None and args.xml_category is None),
"If an xml-testlist is present at least one of --xml-machine, "
"--xml-compiler, --xml-category must also be present")

else:
expect(not (args.baseline_name is not None and (not args.compare and not args.generate)),
"Provided baseline name but did not specify compare or generate")
expect(not (args.compare and args.generate),
"Tried to compare and generate at same time")

expect(not (args.namelists_only and not (args.generate or args.compare)),
"Must provide either --compare or --generate with --namelists-only")

if args.parallel_jobs is not None:
expect(args.parallel_jobs > 0,
"Invalid value for parallel_jobs: %d" % args.parallel_jobs)

if args.xml_testlist is not None:
expect(not (args.xml_machine is None and args.xml_compiler
is None and args.xml_category is None),
"If an xml-testlist is present at least one of --xml-machine, "
"--xml-compiler, --xml-category must also be present")

if args.use_existing:
expect(args.test_id is not None, "Must provide test-id of pre-existing cases")

Expand All @@ -223,15 +256,66 @@ OR
args.test_id = CIME.utils.get_utc_timestamp()

if args.testfile is not None:
with open(args.testfile[0], "r") as fd:
args.testargs.extend( fd.read().splitlines() )
# Remove empty items if any
args.testargs = filter(None, args.testargs)

return args.testargs, args.compiler, args.machine, args.no_run, args.no_build, args.no_setup, args.no_batch,\
args.test_root, args.baseline_root, args.clean, args.compare, args.generate, \
args.baseline_name, args.namelists_only, args.project, args.test_id, args.parallel_jobs, \
args.xml_machine, args.xml_compiler, args.xml_category, args.xml_testlist, args.walltime, \
with open(args.testfile, "r") as fd:
args.testargs.extend( [line.strip() for line in fd.read().splitlines() if line.strip()] )

# Compute list of fully-resolved test_names
test_extra_data = {}
if model == "cesm":
machine_name = args.xml_machine if args.machine is None else args.machine

# If it's still unclear what machine to use, look at test names
if machine_name is None:
for test in args.testargs:
testsplit = CIME.utils.parse_test_name(test)
if testsplit[4] is not None:
if machine_name is None:
machine_name = testsplit[4]
else:
expect(machine_name == testsplit[4],
"ambiguity in machine, please use the --machine option")

mach_obj = Machines(machine=machine_name)
args.compiler = args.xml_compiler if args.compiler is None else args.compiler
args.compiler = mach_obj.get_default_compiler() if args.compiler is None else args.compiler

test_data = CIME.test_utils.get_tests_from_xml(args.xml_machine, args.xml_category,
args.xml_compiler, args.xml_testlist,
machine_name, args.compiler)
test_names = [item["name"] for item in test_data]
logger.info("Testnames: %s" % test_names)
for test_datum in test_data:
test_extra_data[test_datum["name"]] = test_datum
else:
mach_obj = Machines()
args.compiler = mach_obj.get_default_compiler() if args.compiler is None else args.compiler

test_names = update_acme_tests.get_full_test_names(args.testargs, mach_obj.get_machine_name(), args.compiler)

expect(mach_obj.is_valid_compiler(args.compiler),
"Compiler %s not valid for machine %s" % (args.compiler, mach_obj.get_machine_name()))

# Normalize compare/generate between the models
baseline_cmp_name = None
baseline_gen_name = None
if args.compare or args.generate:
if model == "cesm":
if args.compare is not None:
baseline_cmp_name = args.compare
if args.generate is not None:
baseline_gen_name = args.generate
else:
baseline_name = args.baseline_name if args.baseline_name else CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root())
expect(baseline_name is not None,
"Could not determine baseline name from branch, please use -b option")
if args.compare:
baseline_cmp_name = baseline_name
elif args.generate:
baseline_gen_name = baseline_name

return test_names, test_extra_data, args.compiler, mach_obj.get_machine_name(), args.no_run, args.no_build, args.no_setup, args.no_batch,\
args.test_root, args.baseline_root, args.clean, baseline_cmp_name, baseline_gen_name, \
args.namelists_only, args.project, args.test_id, args.parallel_jobs, args.walltime, \
args.single_submit, args.proc_pool, args.use_existing, args.save_timing, args.queue, args.allow_baseline_overwrite

###############################################################################
Expand Down Expand Up @@ -334,33 +418,20 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost
run_cmd_no_fail(submit_cmd, input_str=script, arg_stdout=None, arg_stderr=None, verbose=True)

###############################################################################
def create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, compare, generate,
baseline_name, namelists_only, project, test_id, parallel_jobs,
xml_machine, xml_compiler, xml_category, xml_testlist, walltime,
single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite):
def create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs,
walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite):
###############################################################################
if testargs and machine_name is None and compiler is None:
for test in testargs:
testsplit = CIME.utils.parse_test_name(test)
if testsplit[4] is not None:
if machine_name is None:
machine_name = testsplit[4]
else:
expect(machine_name == testsplit[4],
"ambiguity in machine, please use the --machine option")

impl = TestScheduler(testargs,
no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
test_root=test_root, test_id=test_id,
baseline_root=baseline_root, baseline_name=baseline_name,
clean=clean, machine_name=machine_name, compiler=compiler,
compare=compare, generate=generate, namelists_only=namelists_only,
project=project, parallel_jobs=parallel_jobs,
xml_machine=xml_machine, xml_compiler=xml_compiler,
xml_category=xml_category, xml_testlist=xml_testlist, walltime=walltime,
proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing,
queue=queue, allow_baseline_overwrite=allow_baseline_overwrite)
impl = TestScheduler(test_names, test_data=test_data,
no_run=no_run, no_build=no_build, no_setup=no_setup, no_batch=no_batch,
test_root=test_root, test_id=test_id,
baseline_root=baseline_root, baseline_cmp_name=baseline_cmp_name,
baseline_gen_name=baseline_gen_name,
clean=clean, machine_name=machine_name, compiler=compiler,
namelists_only=namelists_only,
project=project, parallel_jobs=parallel_jobs, walltime=walltime,
proc_pool=proc_pool, use_existing=use_existing, save_timing=save_timing,
queue=queue, allow_baseline_overwrite=allow_baseline_overwrite)

success = impl.run_tests()

Expand Down Expand Up @@ -396,16 +467,14 @@ def _main_func(description):
CIME.utils.get_python_libs_root(), arg_stdout=None, arg_stderr=None)
return

testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \
compare, generate, baseline_name, namelists_only, project, test_id, parallel_jobs, \
xml_machine, xml_compiler, xml_category, xml_testlist, walltime, single_submit, proc_pool, \
use_existing, save_timing, queue, allow_baseline_overwrite \
test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root, baseline_root, clean, \
baseline_cmp_name, baseline_gen_name, namelists_only, project, test_id, parallel_jobs, \
walltime, single_submit, proc_pool, use_existing, save_timing, queue, allow_baseline_overwrite \
= parse_command_line(sys.argv, description)

sys.exit(create_test(testargs, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, compare, generate, baseline_name, namelists_only,
project, test_id, parallel_jobs, xml_machine, xml_compiler, xml_category,
xml_testlist, walltime, single_submit, proc_pool, use_existing, save_timing,
sys.exit(create_test(test_names, test_data, compiler, machine_name, no_run, no_build, no_setup, no_batch, test_root,
baseline_root, clean, baseline_cmp_name, baseline_gen_name, namelists_only,
project, test_id, parallel_jobs, walltime, single_submit, proc_pool, use_existing, save_timing,
queue, allow_baseline_overwrite))

###############################################################################
Expand Down
2 changes: 1 addition & 1 deletion utils/python/CIME/bless_test_results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def bless_namelists(test_name, report_only, force, baseline_name, baseline_root)
def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, report_only, force):
###############################################################################
with Case(testcase_dir_for_test) as case:
baseline_full_dir = os.path.join(baseline_root, case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID"))
baseline_full_dir = os.path.join(baseline_root, baseline_name, case.get_value("CASEBASEID"))
result, comments = compare_baseline(case, baseline_dir=baseline_full_dir)
if result:
return True, None
Expand Down
Loading

0 comments on commit cbc8a68

Please sign in to comment.