work on test rerunability

cime_config/cesm/machines/config_machines.xml

@@ -575,7 +575,7 @@
 
   <machine MACH="hobart">
     <DESC>"NCAR CGD Linux Cluster 48 pes/node, batch system is PBS"</DESC>
-    <NODENAME_REGEX>hobart</NODENAME_REGEX>
+    <NODENAME_REGEX>^hob.*</NODENAME_REGEX>
     <COMPILERS>intel,pgi,nag,gnu</COMPILERS>
     <MPILIBS compiler="nag">openmpi</MPILIBS>
     <MPILIBS compiler="intel">mvapich2</MPILIBS>

utils/python/CIME/SystemTests/sms.py

@@ -2,6 +2,7 @@
 CIME smoke test  This class inherits from SystemTestsCommon
 It does a startup run with restarts off and optionally compares to or generates baselines
 """
+
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 

utils/python/CIME/SystemTests/system_tests_common.py

@@ -145,6 +145,10 @@ def update_test_status(self, status, phase):
                     f.write(line+"\n")
             if not found:
                     f.write("%s %s %d\n"%(status, string, total_time))
+        # This prevents the __del__ method from overwritting an up to date status
+        if phase.startswith("RUN"):
+           self._runstatus = None
+
 
     def _coupler_log_indicates_run_complete(self, coupler_log_path):
         newestcpllogfile = self._get_latest_cpl_log(coupler_log_path)

utils/python/CIME/test_scheduler.py

@@ -617,18 +617,23 @@ def _update_test_status_file(self, test):
         # the phases we have taken care of and then let the run scrips go from there
         # Eventually, it would be nice to have TestStatus management encapsulated
         # into a single place.
+
         str_to_write = ""
         made_it_to_phase = self._get_test_phase(test)
+
         made_it_to_phase_idx = self._phases.index(made_it_to_phase)
         for phase in self._phases[0:made_it_to_phase_idx+1]:
-            str_to_write += "%s %s %s\n" % (self._get_test_status(test, phase), test, phase)
+            if "BUILD" not in phase:
+                # the build phase status is always write by the test itself
+                str_to_write += "%s %s %s\n" % (self._get_test_status(test, phase), test, phase)
 
         if not self._no_run and not self._is_broken(test) and made_it_to_phase == MODEL_BUILD_PHASE:
             # Ensure PEND state always gets added to TestStatus file if we are
             # about to run test
             str_to_write += "%s %s %s\n" % (TEST_PENDING_STATUS, test, RUN_PHASE)
 
         test_status_file = os.path.join(self._get_test_dir(test), TEST_STATUS_FILENAME)
+
         with open(test_status_file, "w") as fd:
             fd.write(str_to_write)
 
@@ -674,7 +679,7 @@ def _handle_test_status_file(self, test, test_phase, success):
         # This complexity is due to sharing of TestStatus responsibilities
         #
         try:
-            if test_phase != RUN_PHASE and (not success or test_phase == MODEL_BUILD_PHASE
+            if test_phase != RUN_PHASE and (not success or test_phase == SETUP_PHASE
                                             or test_phase == self._phases[-1]):
                 self._update_test_status_file(test)