create ability to run tests in same case more than once

scripts/Testing/Testcases/config_tests.xml

@@ -10,7 +10,7 @@ The following are the test functionality categories:
   5) multi-instance tests
   6) archiving (short-term and long-term) tests
   7) performance tests
-  8) spinup tests 
+  8) spinup tests
 
 NOTES:
 - unless otherwise noted everything is run in one executable directory
@@ -129,7 +129,7 @@ OCP    pop performance test
     SPINUP tests
 ======================================================================
 
-SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)  
+SSP    smoke CLM spinup test (only valid for CLM compsets with CLM45)
        do an initial spin test (setting CLM_ACCELERATED_SPINUP to on)
          write restarts at the end of the run
          short term archiving is on
@@ -403,6 +403,10 @@ LAR    long term archive test
     <INFO_DBUG>1</INFO_DBUG>
     <CCSM_TCOST>-1</CCSM_TCOST>
     <DOUT_S>FALSE</DOUT_S>
+    <CONTINUE_RUN>FALSE</CONTINUE_RUN>
+    <REST_OPTION>none</REST_OPTION>
+    <HIST_OPTION>$STOP_OPTION</HIST_OPTION>
+    <HIST_N>$STOP_N</HIST_N>
   </test>
 
   <test NAME="SSP">
@@ -412,7 +416,7 @@ LAR    long term archive test
     <STOP_OPTION>ndays</STOP_OPTION>
     <RUN_TYPE>startup</RUN_TYPE>
     <STOP_N>4</STOP_N>
-    <DOUT_S>FALSE</DOUT_S> 
+    <DOUT_S>FALSE</DOUT_S>
   </test>
 
 </config_test>

utils/python/CIME/SystemTests/sms.py

@@ -1,5 +1,6 @@
 """
 CIME smoke test  This class inherits from SystemTestsCommon
+It does a startup run with restarts off and optionally compares to or generates baselines
 """
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
@@ -15,16 +16,7 @@ def __init__(self, case):
         SystemTestsCommon.__init__(self, case)
 
     def run(self):
-        self._case.set_value("CONTINUE_RUN",False)
-        self._case.set_value("REST_OPTION","none")
-        self._case.set_value("HIST_OPTION","$STOP_OPTION")
-        self._case.set_value("HIST_N","$STOP_N")
-        stop_n = self._case.get_value("STOP_N")
-        stop_option = self._case.get_value("STOP_OPTION")
-
-        self._case.flush()
-        logger.info("doing an %d %s initial test, no restarts written" % (stop_n, stop_option))
-        return SystemTestsCommon._run(self)
+        return SystemTestsCommon.run(self)
 
     def report(self):
         SystemTestsCommon.report(self)

utils/python/CIME/SystemTests/system_tests_common.py

@@ -27,6 +27,7 @@ def __init__(self, case, expected=None):
         self._caseroot = caseroot
         self._orig_caseroot = caseroot
         self._runstatus = None
+        self._casebaseid = self._case.get_value("CASEBASEID")
         # Needed for sh scripts
         os.environ["CASEROOT"] = caseroot
 
@@ -48,22 +49,25 @@ def __init__(self, case, expected=None):
         self._case.set_value("TEST",True)
         self._case.flush()
 
-    def fail_test(self):
-        self._runstatus = "FAIL"
 
     def has_failed(self):
         return self._runstatus == "FAIL"
 
-    def pass_test(self):
-        if not self.has_failed():
-            self._runstatus = "PASS"
 
     def has_passed(self):
         return self._runstatus == "PASS"
 
     def build(self, sharedlib_only=False, model_only=False):
-        build.case_build(self._caseroot, case=self._case,
-                         sharedlib_only=sharedlib_only, model_only=model_only)
+        status = "PASS"
+        try:
+            build.case_build(self._caseroot, case=self._case,
+                             sharedlib_only=sharedlib_only, model_only=model_only)
+        except:
+            status = "FAIL"
+        if not model_only:
+            self.update_test_status(status, "SHAREDLIB_BUILD")
+        if not sharedlib_only:
+            self.update_test_status(status, "MODEL_BUILD")
 
     def clean_build(self, comps=None):
         build.clean(self._case, cleanlist=comps)
@@ -79,39 +83,68 @@ def _set_active_case(self, case):
         self._caseroot = case.get_value("CASEROOT")
 
     def _run(self, suffix="base", coupler_log_path=None, st_archive=False):
+        if self._case.get_value("IS_FIRST_RUN"):
+            self._runstatus = "PEND"
+        self.update_test_status(self._runstatus,"RUN with suffix %s"%suffix)
+
+        stop_n = self._case.get_value("STOP_N")
+        stop_option = self._case.get_value("STOP_OPTION")
+        run_type = self._case.get_value("RUN_TYPE")
+        rest_option = self._case.get_value("REST_OPTION")
+        rest_n = self._case.get_value("REST_N")
+        infostr = "doing an %d %s %s test" % (stop_n, stop_option,run_type)
+        if rest_option == "none":
+            infostr += ", no restarts written"
+        else:
+            infostr += ", with restarts every %d %s"%(rest_n, rest_option)
+        logger.info(infostr)
+
         try:
             success = case_run(self._case)
             if success and st_archive:
                 success = case_st_archive(self._case)
 
             if success and self._coupler_log_indicates_run_complete(coupler_log_path):
-                self.pass_test()
+                self._runstatus = "PASS"
+                self.update_test_status("PASS","RUN with suffix %s"%suffix)
             else:
                 success = False
-                self.fail_test()
+                self._runstatus = "FAIL"
+                self.update_test_status("FAIL","RUN with suffix %s"%suffix)
 
             if success and suffix is not None:
                 self._component_compare_move(suffix)
         except:
             # An exception must not prevent the TestStatus file from
             # being marked FAIL
             success = False
-            self.fail_test()
+            self._runstatus = "FAIL"
+            self.update_test_status("FAIL","RUN with suffix %s"%suffix)
             logger.warning("Exception during run: %s" % (sys.exc_info()[1]))
 
         return success
 
     def __del__(self):
         if self._runstatus is not None:
-            test_status = os.path.join(self._orig_caseroot, "TestStatus")
-            with open(test_status, 'r') as f:
-                teststatusfile = f.read()
-            li = teststatusfile.rsplit('PEND', 1)
-            teststatusfile = self._runstatus.join(li)
-            total_time = int(time.time() - self._start_time)
-            with open(test_status, 'w') as f:
-                f.write(teststatusfile)
-                f.write("COMMENT TIME %d\n" % total_time)
+            self.update_test_status(self._runstatus, "RUN")
+
+    def update_test_status(self, status, phase):
+        test_status = os.path.join(self._orig_caseroot, "TestStatus")
+        with open(test_status, 'r') as f:
+            teststatusfile = f.read()
+        string = "%s %s"%(self._casebaseid, phase)
+        total_time = int(time.time() - self._start_time)
+        self._start_time = time.time()
+        found = False
+        with open(test_status, 'w') as f:
+            for line in teststatusfile.splitlines():
+                if string in line:
+                    f.write("%s %s %d\n"%(status, string, total_time))
+                    found = True
+                else:
+                    f.write(line+"\n")
+            if not found:
+                    f.write("%s %s %d\n"%(status, string, total_time))
 
     def _coupler_log_indicates_run_complete(self, coupler_log_path):
         newestcpllogfile = self._get_latest_cpl_log(coupler_log_path)
@@ -208,13 +241,11 @@ def _check_for_memleak(self, cpllog):
             if memdiff < 0:
                 append_status("COMMENT: insuffiencient data for memleak test",sfile="TestStatus")
             elif memdiff < 0.1:
-                append_status("PASS %s memleak"%(self._case.get_value("CASEBASEID")),
-                             sfile="TestStatus")
+                self.update_test_status("PASS","memleak")
             else:
                 append_status("memleak detected, memory went from %f to %f in %d days"
                              %(originalmem, finalmem, finaldate-originaldate),sfile="TestStatus.log")
-                append_status("FAIL %s memleak"%(self._case.get_value("CASEBASEID")),
-                             sfile="TestStatus")
+                self.update_test_status("FAIL","memleak")
 
     def compare_env_run(self, expected=None):
         f1obj = EnvRun(self._caseroot, "env_run.xml")
@@ -343,7 +374,7 @@ def generate_baseline(self):
 class FakeTest(SystemTestsCommon):
     '''
     Inheriters of the FakeTest Class are intended to test the code.
-    
+
     All members of the FakeTest Class must
     have names beginnig with "TEST" this is so that the find_system_test
     in utils.py will work with these classes.