Skip to content

Commit

Permalink
Merge branch 'jgfouca/scripts/add_titan_to_nightlies' into next (PR #302
Browse files Browse the repository at this point in the history 
)

Significant refactor to support titan in our ACME scripts.

1) Add more informative defaults where possible. Instead of defaulting to
None and computing the real default later, try to compute the real default
before options are parsed so it can be presented.
2) Compiler now needs to be taken as an options for most ACME scripts
since it affects where baselines are.
3) The subdirectory from baseline_root where baselines live is now
$ROOT/$COMPILER/$BRANCH instead of $ROOT/$BRANCH. This will make
life much easier on machines that have several supported compilers for
ACME.
4) Change default project for titan to cli115. The default should match
what the acmetest account uses to run ACME.
5) Fix significant problems in acme scripts regression tests.
5.a) Errors were being hidden due to trying to supported threaded and
non-threaded execution of a subprocess within the same function. These
have been split out so this doesn't happen again.
5.b) Check dashboard results in wait_for_tests regr tests.
5.c) Slight changes to jenkins_generic_job testing due to recent
change to CDash build groups.
6) Changes to CESG create_test to have better support for namelist-only
mode.

Baselines on all ACME machines will need to be moved.

[BFB]
  • Loading branch information
Jeffrey Johnson committed Aug 19, 2015
2 parents 7b2219f + eb8c573 commit 034c06d
Show file tree
Hide file tree
Showing 10 changed files with 236 additions and 213 deletions.
34 changes: 22 additions & 12 deletions scripts/acme/acme_bisect
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ from acme_util import expect, warning, verbose_print, run_cmd
def parse_command_line(args, description):
###############################################################################
parser = argparse.ArgumentParser(
usage="""\n%s <testname> <testroot> <project> [--compare=<baseline-id>] [--no-submit] [--verbose]
usage="""\n%s <testname> [--compare=<baseline-id>] [--no-batch] [--verbose]
OR
%s --help
OR
Expand All @@ -36,13 +36,20 @@ description=description,
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)

default_compiler, _, _, default_project, default_testroot, _, _ = acme_util.get_machine_info(raw=True)

parser.add_argument("testname", help="Name of failing test.")

parser.add_argument("testroot", help="Path to testroot to use for testcases for bisect.")
parser.add_argument("-r", "--test-root", action="store", dest="testroot", default=default_testroot,
help="Path to testroot to use for testcases for bisect.")

parser.add_argument("-c", "--compiler", action="store", dest="compiler", default=default_compiler,
help="What compiler to use to build ACME")

parser.add_argument("project", help="Project to be given to create_test.")
parser.add_argument("-p", "--project", action="store", dest="project", default=default_project,
help="Project to be given to create_test.")

parser.add_argument("-c", "--compare", action="store", dest="compare", default=None,
parser.add_argument("-b", "--baseline-name", action="store", dest="baseline_name", default=None,
help="Baseline id for comparing baselines. Not specifying means no comparisons will be done.")

parser.add_argument("-v", "--verbose", action="store_true", dest="verbose", default=False,
Expand All @@ -58,16 +65,19 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
help="Consider a commit to be broken if memory check fails (fail if tests footprint grows)")

parser.add_argument("--no-batch", action="store_true", dest="no_batch", default=False,
help="Do not submit job to queue, run locally. Will default to whatever is standard on your machine")
help="Force scripts to not submit tests as batch jobs. Default will be to do whatever is standard on the current machine")

args = parser.parse_args(args[1:])

acme_util.set_verbosity(args.verbose)

return args.testname, args.testroot, args.project, args.compare, args.check_namelists, args.check_throughput, args.check_memory, args.no_batch
if (args.test_root == default_testroot):
args.test_root = acme_util.get_machine_info(project=args.project)

return args.testname, args.testroot, args.compiler, args.project, args.baseline_name, args.check_namelists, args.check_throughput, args.check_memory, args.no_batch

###############################################################################
def acme_bisect(testname, testroot, project, compare, check_namelists, check_throughput, check_memory, no_batch):
def acme_bisect(testname, testroot, compiler, project, baseline_name, check_namelists, check_throughput, check_memory, no_batch):
###############################################################################
expect(os.path.exists("scripts/acme/create_test"), "Please run from root of repository")

Expand All @@ -80,9 +90,9 @@ def acme_bisect(testname, testroot, project, compare, check_namelists, check_thr
# Formulate and run create_test command

extra_args = "--no-batch" if no_batch else ""
compare_args = "-c -b %s" % compare if compare is not None else ""
create_test_cmd = "scripts/acme/create_test %s --test-root %s -t %s -p %s %s %s" % \
(testname, testroot, current_sha, project, compare_args, extra_args)
compare_args = "-c -b %s" % baseline_name if baseline_name is not None else ""
create_test_cmd = "scripts/acme/create_test %s --test-root %s --compiler %s -t %s -p %s %s %s" % \
(testname, testroot, compiler, current_sha, project, compare_args, extra_args)
run_cmd(create_test_cmd)

# Find testcase area
Expand Down Expand Up @@ -119,11 +129,11 @@ def _main_func(description):

acme_util.stop_buffering_output()

testname, testroot, project, compare, check_namelists, check_throughput, check_memory, no_batch = \
testname, testroot, compiler, project, baseline_name, check_namelists, check_throughput, check_memory, no_batch = \
parse_command_line(sys.argv, description)

try:
rv = acme_bisect(testname, testroot, project, compare, check_namelists, check_throughput, check_memory, no_batch)
rv = acme_bisect(testname, testroot, compiler, project, baseline_name, check_namelists, check_throughput, check_memory, no_batch)
except:
print >> sys.stderr, "Exception in script, aborting bisect entirely:"
e = sys.exc_info()[1]
Expand Down
23 changes: 15 additions & 8 deletions scripts/acme/acme_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,9 @@
"pgi",
"acme_integration",
True,
"cli112",
"cli115",
"/lustre/atlas/scratch/<USER>/<PROJECT>",
"/lustre/atlas1/cli900/world-shared/cesm/baselines",
"/lustre/atlas1/cli900/world-shared/cesm/acme/baselines",
None
),
"mira" : (
Expand Down Expand Up @@ -370,7 +370,7 @@ def get_batch_system_info(batch_system=None):
return BATCH_INFO[batch_system]

###############################################################################
def get_machine_info(machine=None, user=None, project=None):
def get_machine_info(machine=None, user=None, project=None, raw=False):
###############################################################################
"""
Return information on machine. If no arg provided, probe for machine.
Expand All @@ -379,22 +379,29 @@ def get_machine_info(machine=None, user=None, project=None):
(compiler, test_suite, use_batch, project, testroot, baseline_root, proxy)
"""
import getpass
user = getpass.getuser() if user is None else user

if (machine is None):
machine = probe_machine_name()
expect(machine is not None, "Failed to probe machine")
expect(machine in MACHINE_INFO, "No info for machine '%s'" % machine)
user = getpass.getuser() if user is None else user
project = project if project is not None else MACHINE_INFO[machine][3]

return [item.replace("<USER>", user).replace("<PROJECT>", project) if type(item) is str else item for item in MACHINE_INFO[machine]]
machine_info_copy = list(MACHINE_INFO[machine])
machine_info_copy[3] = project if project is not None else machine_info_copy[3]
project = machine_info_copy[3]

if (raw):
return machine_info_copy
else:
return [item.replace("<USER>", user).replace("<PROJECT>", project) if type(item) is str else item for item in machine_info_copy]

###############################################################################
def get_utc_timestamp(format="%Y%m%d_%H%M%S"):
def get_utc_timestamp(timestamp_format="%Y%m%d_%H%M%S"):
###############################################################################
"""
Get a string representing the current UTC time in format: YYMMDD_HHMMSS
The format can be changed if needed.
"""
utc_time_tuple = time.gmtime()
return time.strftime(format, utc_time_tuple)
return time.strftime(timestamp_format, utc_time_tuple)
49 changes: 22 additions & 27 deletions scripts/acme/bless_test_results
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ with versions of the files that you should not bless.
import acme_util
acme_util.check_minimum_python_version(2, 7)

import argparse, sys, os, socket, shutil, glob, doctest
import argparse, sys, os, glob, doctest

import wait_for_tests, compare_namelists, simple_compare
from acme_util import expect, warning, verbose_print, run_cmd
Expand Down Expand Up @@ -44,20 +44,28 @@ description=description,
formatter_class=argparse.ArgumentDefaultsHelpFormatter
)

default_baseline_name = acme_util.get_current_branch(repo=acme_util.get_source_repo())

default_compiler, _, _, _, acme_root, _, _ = acme_util.get_machine_info()
default_testroot = os.path.join(acme_root, "jenkins")

parser.add_argument("-v", "--verbose", action="store_true",
help="Print extra information")

parser.add_argument("-n", "--namelists-only", action="store_true",
help="Only analyze namelists.")

parser.add_argument("-b", "--branch",
help="Force us to use baselines for a specific branch instead of the current branch.")
parser.add_argument("-b", "--baseline-name", default=default_baseline_name,
help="Name of baselines to use.")

parser.add_argument("-c", "--compiler", default=default_compiler,
help="Compiler of run you want to bless")

parser.add_argument("-r", "--report-only", action="store_true",
help="Only report what files will be overwritten and why. Caution is a good thing when updating baselines")

parser.add_argument("-t", "--test-root",
help="Instead of looking at last night's Jenkins results, use a custom test area")
parser.add_argument("-t", "--test-root", default=default_testroot,
help="Path to test results that are being blessed")

parser.add_argument("-f", "--force", action="store_true",
help="Update every diff without asking. VERY DANGEROUS. Should only be used within testing scripts.")
Expand All @@ -72,31 +80,18 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter

acme_util.set_verbosity(args.verbose)

return args.branch, args.namelists_only, args.report_only, args.test_root, args.force, args.bless_tests
return args.baseline_name, args.test_root, args.compiler, args.namelists_only, args.report_only, args.force, args.bless_tests

###############################################################################
def bless_test_results(baseline_branch=None, namelists_only=False, report_only=False, test_root=None, force=False, bless_tests=None):
def bless_test_results(baseline_name, test_root, compiler, namelists_only=False, report_only=False, force=False, bless_tests=None):
###############################################################################
acme_machine = acme_util.probe_machine_name()
expect(acme_machine is not None,
"Did not recognize current machine '%s'" % socket.gethostname())

acme_repo = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
expect(os.path.isdir(os.path.join(acme_repo, ".git")),
"Expected '%s' to be a git repository" % acme_repo)

_, _, _, _, acme_root, baseline_root, _ = acme_util.get_machine_info(acme_machine)
if (test_root is None):
test_root = os.path.join(acme_root, "jenkins")

git_branch = acme_util.get_current_branch(repo=acme_repo) if baseline_branch is None else baseline_branch

test_results = wait_for_tests.get_test_results(glob.glob("%s/*/TestStatus" % test_root),
True, # no wait
False, # don't check throughput
False) # don't ignore namelist diffs

baseline_area = os.path.join(baseline_root, git_branch)
baseline_root = acme_util.get_machine_info()[5]
baseline_area = os.path.join(baseline_root, compiler, baseline_name)

for test_name, test_data in test_results.iteritems():
if (bless_tests in [[], None] or acme_util.match_any(test_name, bless_tests)):
Expand All @@ -123,8 +118,8 @@ def bless_test_results(baseline_branch=None, namelists_only=False, report_only=F
case = os.path.basename(testcase_dir_for_test)

# Get user that test was run as (affects loc of hist files)
user = run_cmd(r"""grep CCSMUSER %s/env_case.xml | sed -E 's/.+value="(.+)".+/\1/g'""" % testcase_dir_for_test)
acme_root = acme_util.get_machine_info(acme_machine, user=user)[4]
user = run_cmd(r"""grep CCSMUSER %s/env_case.xml | sed -E 's/.+value="(.+)".+/\1/g'""" % testcase_dir_for_test).strip()
acme_root = acme_util.get_machine_info(user=user)[4]

# Find files of various types
namelist_files = []
Expand Down Expand Up @@ -168,7 +163,7 @@ def bless_test_results(baseline_branch=None, namelists_only=False, report_only=F
warning("Skipping hist files for this test")
continue
hist_file = hist_files[0]
hist_compare_script = os.path.join(os.path.dirname(__file__), "..", "ccsm_utils", "Tools", "hist_compare.csh")
hist_compare_script = os.path.join(acme_util.get_source_repo(), "scripts", "ccsm_utils", "Tools", "hist_compare.csh")
stat = run_cmd("%s %s %s" % (hist_compare_script, hist_file, baseline_file), arg_stdout=None, arg_stderr=None, ok_to_fail=True)[0]
if (stat != 0):
print "Hist files '%s' and '%s' did not match" % (baseline_file, hist_file)
Expand All @@ -194,10 +189,10 @@ def _main_func(description):

acme_util.stop_buffering_output()

baseline_branch, namelists_only, report_only, test_root, force, bless_tests = \
baseline_name, test_root, compiler, namelists_only, report_only, force, bless_tests = \
parse_command_line(sys.argv, description)

bless_test_results(baseline_branch, namelists_only, report_only, test_root, force, bless_tests)
bless_test_results(baseline_name, test_root, compiler, namelists_only, report_only, force, bless_tests)

###############################################################################

Expand Down
Loading

0 comments on commit 034c06d

Please sign in to comment.