Skip to content

Commit

Permalink
Merge branch 'jgfouca/scripts-acme/improve_robustness_of_bless_test_r…
Browse files Browse the repository at this point in the history 
…esults' into next (PR #538)

Instead of dying with a SystemExit when there's a problem, warn
the user of the problem and move on to the next test. Also, fix
logic to handle namelist-only / hist-only correctly. Also, reorg
and clean code a bit.

[BFB]

* jgfouca/scripts-acme/improve_robustness_of_bless_test_results:
  bless_test_results: Make much more robust
  • Loading branch information
@rljacob
rljacob committed Dec 16, 2015
2 parents eabfd6b + ec39b7f commit c505fc1
Showing 1 changed file with 99 additions and 64 deletions.
163 changes: 99 additions & 64 deletions cime/scripts-acme/bless_test_results
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,43 @@ def bless_namelists(test_name, baseline_dir_for_test, testcase_dir_for_test, rep
if (namelist_files):
acme_util.safe_copy(testcase_dir_for_test, baseline_dir_for_test, namelist_files)

###############################################################################
def bless_history(test_name, baseline_tag, baseline_dir_for_test, testcase_dir_for_test, report_only, force):
###############################################################################
# Get user that test was run as (affects loc of hist files)
user = run_cmd(r"""grep CCSMUSER %s/env_case.xml | sed -E 's/.+value="(.+)".+/\1/g'""" % testcase_dir_for_test)
acme_root = acme_util.get_machine_info("CESMSCRATCHROOT", user=user)

case = os.path.basename(testcase_dir_for_test)
cime_root = acme_util.get_cime_root()
compgen = os.path.join(cime_root, "scripts", "Tools", "component_compgen_baseline.sh")
machine_env = os.path.join(cime_root, "machines-acme", "env_mach_specific.%s" % acme_util.probe_machine_name())
run_dir = os.path.join(acme_root, case, "run")
cprnc_loc = acme_util.get_machine_info("CPRNC_ROOT")
compgen_cmd = "tcsh -c 'source %s && %s -baseline_dir %s -testcase %s -testcase_base %s -test_dir %s -cprnc_exe %s -generate_tag %s'" % \
(machine_env, compgen, baseline_dir_for_test, case, test_name, run_dir, cprnc_loc, baseline_tag)

check_compare = os.path.join(cime_root, "scripts", "Tools", "component_write_comparefail.pl")
stat, out, _ = run_cmd("%s %s 2>&1" % (check_compare, run_dir), ok_to_fail=True)

if (stat != 0):
# found diff, offer rebless
print out

if (not report_only and
(force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])):
stat = run_cmd(compgen_cmd, ok_to_fail=True, verbose=True)[0]
if (stat != 0):
warning("Hist file bless FAILED for test %s" % test_name)
return False
else:
return True
else:
return True
else:
warning("Test '%s' was marked as DIFF but cprnc did not find diff?" % test_name)
return False

###############################################################################
def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelists_only=False, hist_only=False, report_only=False, force=False, bless_tests=None):
###############################################################################
Expand All @@ -148,75 +185,73 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis
if (bless_tests in [[], None] or acme_util.match_any(test_name, bless_tests)):
overall_result = wait_for_tests.reduce_stati(test_result)

expect(wait_for_tests.RUN_PHASE in test_result,
"Test '%s' had no run phase" % test_name)
expect(create_test_impl.NAMELIST_PHASE in test_result,
"Test '%s' had no namelist phase" % test_name)
# Compute namelists status, False implies it diffed
if (not hist_only):
if (create_test_impl.NAMELIST_PHASE in test_result):
nl_no_bless = test_result[create_test_impl.NAMELIST_PHASE] == wait_for_tests.TEST_PASS_STATUS
else:
warning("Test '%s' did not make it to namelist phase" % test_name)
broken_blesses.append(test_name)
nl_no_bless = True
else:
nl_no_bless = True

# Compute hist status, False implies it diffed
if (not namelists_only):
if (wait_for_tests.RUN_PHASE not in test_result):
broken_blesses.append(test_name)
warning("Test '%s' did not make it to run phase" % test_name)
hist_no_bless = True
elif (test_result[wait_for_tests.RUN_PHASE] == wait_for_tests.TEST_PASS_STATUS):
if (wait_for_tests.HIST_COMPARE_PHASE not in test_result):
broken_blesses.append(test_name)
warning("Test '%s' had no history compare phase" % test_name)
hist_no_bless = True
else:
hist_no_bless = test_result[wait_for_tests.HIST_COMPARE_PHASE] == wait_for_tests.TEST_PASS_STATUS
else:
broken_blesses.append(test_name)
warning("Test '%s' did not pass, not safe to bless" % test_name)
hist_no_bless = True
else:
hist_no_bless = True

run_phase_pass = test_result[wait_for_tests.RUN_PHASE] == wait_for_tests.TEST_PASS_STATUS
nl_pass = test_result[create_test_impl.NAMELIST_PHASE] == wait_for_tests.TEST_PASS_STATUS
# Now, do the bless
if ( (nl_no_bless and hist_no_bless) or (nl_no_bless and namelists_only) or (hist_no_bless and hist_only) ):
print "Nothing to bless for test:", test_name, " overall status:", overall_result
else:

if (not run_phase_pass):
warning("Test '%s' did not run successfully, it is not safe to bless results" % test_name)
print "###############################################################################"
print "Blessing results for test:", test_name, "most recent result:", overall_result
print "###############################################################################"
time.sleep(2)
else:
expect(wait_for_tests.HIST_COMPARE_PHASE in test_result,
"Test '%s' had no history compare phase" % test_name)
hist_pass = test_result[wait_for_tests.HIST_COMPARE_PHASE] == wait_for_tests.TEST_PASS_STATUS

if ( (nl_pass and hist_pass) or (nl_pass and namelists_only) or (hist_pass and hist_only) ):
print "Nothing to bless for test:", test_name, " overall status:", overall_result
else:
# Get baseline dir for this test
baseline_dir_for_test = os.path.join(baseline_area, test_name)
if (not os.path.isdir(baseline_dir_for_test)):
warning("Problem, baseline dir '%s' does not exist" % baseline_dir_for_test)
broken_blesses.append(test_name)
continue

# Get testcase dir for this test
globs = glob.glob("%s/%s%s" % (test_root, test_name, test_id_glob))
if (len(globs) != 1):
warning("Expected exactly one match for testcase area for test '%s', found '%s'" % (test_name, globs))
broken_blesses.append(test_name)
continue

testcase_dir_for_test = globs[0]

# Bless namelists
if (not nl_no_bless):
bless_namelists(test_name, baseline_dir_for_test, testcase_dir_for_test, report_only, force)

# Bless hist files
if (not hist_no_bless):
if (not bless_history(test_name, baseline_tag, baseline_dir_for_test, testcase_dir_for_test, report_only, force)):
broken_blesses.append(test_name)

print "###############################################################################"
print "Blessing results for test:", test_name, "most recent result:", overall_result
print "###############################################################################"
time.sleep(2)

# Get baseline dir for this test
baseline_dir_for_test = os.path.join(baseline_area, test_name)
expect(os.path.isdir(baseline_dir_for_test),
"Problem, baseline dir '%s' does not exist" % baseline_dir_for_test)

# Get testcase dir for this test
globs = glob.glob("%s/%s%s" % (test_root, test_name, test_id_glob))
expect(len(globs) == 1, "Expected exactly one match for testcase area for test '%s', found '%s'" % (test_name, globs))
testcase_dir_for_test = globs[0]
case = os.path.basename(testcase_dir_for_test)

# Get user that test was run as (affects loc of hist files)
user = run_cmd(r"""grep CCSMUSER %s/env_case.xml | sed -E 's/.+value="(.+)".+/\1/g'""" % testcase_dir_for_test)
acme_root = acme_util.get_machine_info("CESMSCRATCHROOT", user=user)

# Bless namelists
if (not hist_only and not nl_pass):
bless_namelists(test_name, baseline_dir_for_test, testcase_dir_for_test, report_only, force)

# Bless hist files
if (not namelists_only and not hist_pass):
cime_root = acme_util.get_cime_root()
compgen = os.path.join(cime_root, "scripts", "Tools", "component_compgen_baseline.sh")
machine_env = os.path.join(cime_root, "machines-acme", "env_mach_specific.%s" % acme_util.probe_machine_name())
run_dir = os.path.join(acme_root, case, "run")
cprnc_loc = acme_util.get_machine_info("CPRNC_ROOT")
compgen_cmd = "tcsh -c 'source %s && %s -baseline_dir %s -testcase %s -testcase_base %s -test_dir %s -cprnc_exe %s -generate_tag %s'" % \
(machine_env, compgen, baseline_dir_for_test, case, test_name, run_dir, cprnc_loc, baseline_tag)

check_compare = os.path.join(cime_root, "scripts", "Tools", "component_write_comparefail.pl")
stat, out, _ = run_cmd("%s %s 2>&1" % (check_compare, run_dir), ok_to_fail=True)

if (stat != 0):
# found diff, offer rebless
print out

if (not report_only and
(force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])):
stat = run_cmd(compgen_cmd, ok_to_fail=True, verbose=True)[0]
if (stat != 0):
warning("Hist file bless FAILED for test %s" % test_name)
broken_blesses.append(test_name)

# Make sure user knows that some tests were not bless
# Make sure user knows that some tests were not blessed
for broken_bless in broken_blesses:
warning("FAILED TO BLESS TEST: %s" % broken_bless)

Expand Down

0 comments on commit c505fc1

Please sign in to comment.