Merge pull request #4501 from ESMCI/jgfouca/jenkins_save_timing

Jenkins_generic_job: Add --save-timing

Test suite: scripts_regression_tests.py test_sys_jenkins_generic_job.TestJenkinsGenericJob
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes [CIME Github issue #]

User interface changes?:

Update gh-pages html (Y/N)?:

CIME/Tools/jenkins_generic_job

@@ -180,6 +180,12 @@ OR
         help="Do not fail if there are memleaks",
     )
 
+    parser.add_argument(
+        "--save-timing",
+        action="store_true",
+        help="Tell create_test to save timings of tests",
+    )
+
     parser.add_argument(
         "--pes-file",
         help="Full pathname of an optional pes specification file. The file"
@@ -259,6 +265,7 @@ OR
         args.check_throughput,
         args.check_memory,
         args.ignore_memleak,
+        args.save_timing,
         args.pes_file,
         args.jenkins_id,
         args.queue,
@@ -289,6 +296,7 @@ def _main_func(description):
         check_throughput,
         check_memory,
         ignore_memleak,
+        save_timing,
         pes_file,
         jenkins_id,
         queue,
@@ -317,6 +325,7 @@ def _main_func(description):
             check_throughput,
             check_memory,
             ignore_memleak,
+            save_timing,
             pes_file,
             jenkins_id,
             queue,

CIME/get_tests.py

@@ -100,6 +100,10 @@
             "SMS_P16.T42_T42.S",
         ),
     },
+    "cime_test_timing": {
+        "time": "0:10:00",
+        "tests": ("SMS_P1.T42_T42.S",),
+    },
     "cime_test_repeat": {
         "tests": (
             "TESTRUNPASS_P1.f19_g16_rx1.A",

CIME/jenkins_generic_job.py

@@ -279,6 +279,7 @@ def jenkins_generic_job(
     check_throughput,
     check_memory,
     ignore_memleak,
+    save_timing,
     pes_file,
     jenkins_id,
     queue,
@@ -361,16 +362,19 @@ def jenkins_generic_job(
         create_test_args.append("-j {:d}".format(parallel_jobs))
 
     if walltime is not None:
-        create_test_args.append(" --walltime " + walltime)
+        create_test_args.append("--walltime " + walltime)
 
     if baseline_root is not None:
-        create_test_args.append(" --baseline-root " + baseline_root)
+        create_test_args.append("--baseline-root " + baseline_root)
 
     if pes_file is not None:
-        create_test_args.append(" --pesfile " + pes_file)
+        create_test_args.append("--pesfile " + pes_file)
 
     if queue is not None:
-        create_test_args.append(" --queue " + queue)
+        create_test_args.append("--queue " + queue)
+
+    if save_timing:
+        create_test_args.append("--save-timing")
 
     create_test_cmd = "./create_test " + " ".join(create_test_args)
 

CIME/tests/test_sys_jenkins_generic_job.py

@@ -61,18 +61,17 @@ def threaded_test(self, expect_works, extra_args, build_name=None):
             self._thread_error = str(e)
 
     def assert_num_leftovers(self, suite):
-        num_tests_in_tiny = len(get_tests.get_test_suite(suite))
+        num_tests_in_suite = len(get_tests.get_test_suite(suite))
 
-        jenkins_dirs = glob.glob(
-            "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize())
-        )  # case dirs
+        case_glob = "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize())
+        jenkins_dirs = glob.glob(case_glob)  # Case dirs
         # scratch_dirs = glob.glob("%s/*%s*/" % (self._testroot, test_id)) # blr/run dirs
 
         self.assertEqual(
-            num_tests_in_tiny,
+            num_tests_in_suite,
             len(jenkins_dirs),
-            msg="Wrong number of leftover directories in %s, expected %d, see %s"
-            % (self._jenkins_root, num_tests_in_tiny, jenkins_dirs),
+            msg="Wrong number of leftover directories in %s, expected %d, see %s. Glob checked %s"
+            % (self._jenkins_root, num_tests_in_suite, jenkins_dirs, case_glob),
         )
 
         # JGF: Can't test this at the moment due to root change flag given to jenkins_generic_job
@@ -97,6 +96,21 @@ def test_jenkins_generic_job(self):
         )  # jenkins_generic_job should have automatically cleaned up leftovers from prior run
         self.assert_dashboard_has_build(build_name)
 
+    def test_jenkins_generic_job_save_timing(self):
+        self.simple_test(
+            True, "-t cime_test_timing --save-timing -b %s" % self._baseline_name
+        )
+        self.assert_num_leftovers("cime_test_timing")
+
+        jenkins_dirs = glob.glob(
+            "%s/*%s*/" % (self._jenkins_root, self._baseline_name.capitalize())
+        )  # case dirs
+        case = jenkins_dirs[0]
+        result = self.run_cmd_assert_result(
+            "./xmlquery --value SAVE_TIMING", from_dir=case
+        )
+        self.assertEqual(result, "TRUE")
+
     def test_jenkins_generic_job_kill(self):
         build_name = "jenkins_generic_job_kill_%s" % utils.get_timestamp()
         run_thread = threading.Thread(