Merge pull request #1471 from ESMCI/erichlf/case_submit/skip_pnl

Add skip-preview-namelist to case.submit and case_run

Added '--skip-preview-namelist' to case.build

Added '--skip-preview-namelist' to case.build. This will allow the user
to specify if the namelist should be rebuilt from user_nl_*.

Test suite: scripts_regression_tests.py
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #1276, E3SM-Project/E3SM#1466

User interface changes?: Added --skip-preview-namelist to case.build and
template.case.run

Code review: @jgfouca, @rljacob, @jedwards4b

config/acme/machines/template.case.run

@@ -32,35 +32,36 @@ import argparse, doctest
 def parse_command_line(args, description):
 ###############################################################################
     parser = argparse.ArgumentParser(
-        usage="""\n%s [--verbose]
+        usage="""\n{0} [--verbose]
 OR
-%s --help
+{0} --help
 OR
-%s --test
+{0} --test
 
 \033[1mEXAMPLES:\033[0m
     \033[1;32m# case.run SMS\033[0m
-    > %s
-""" % ((os.path.basename(args[0]), ) * 4),
-
-description=description,
-
-formatter_class=argparse.ArgumentDefaultsHelpFormatter
-)
-
-    parser = argparse.ArgumentParser()
+    > {0}
+""".format(os.path.basename(args[0])),
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
 
     CIME.utils.setup_standard_logging_options(parser)
 
     parser.add_argument("--caseroot",
                         help="Case directory to build")
 
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
+    parser.add_argument("--skip-preview-namelist", action="store_true",
+                        help="Skip calling preview-namelist during case.run")
 
     if args.caseroot is not None:
         os.chdir(args.caseroot)
 
-    return args.caseroot
+    if args.skip_preview_namelist is None:
+        args.skip_preview_namelist = False
+
+    return args.caseroot, args.skip_preview_namelist
 
 ###############################################################################
 def _main_func(description):
@@ -69,9 +70,9 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
-    caseroot = parse_command_line(sys.argv, description)
+    caseroot, skip_pnl = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
-        success = case_run(case)
+        success = case_run(case, skip_pnl=skip_pnl)
 
     sys.exit(0 if success else 1)
 

config/cesm/machines/template.case.run

@@ -29,35 +29,37 @@ import argparse, doctest
 def parse_command_line(args, description):
 ###############################################################################
     parser = argparse.ArgumentParser(
-        usage="""\n%s [--verbose]
+        usage="""\n{0} [--verbose]
 OR
-%s --help
+{0} --help
 OR
-%s --test
+{0} --test
 
 \033[1mEXAMPLES:\033[0m
     \033[1;32m# case.run SMS\033[0m
-    > %s
-""" % ((os.path.basename(args[0]), ) * 4),
-
-description=description,
-
-formatter_class=argparse.ArgumentDefaultsHelpFormatter
-)
-
-    parser = argparse.ArgumentParser()
+    > {0}
+""".format(os.path.basename(args[0])),
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
 
     CIME.utils.setup_standard_logging_options(parser)
 
     parser.add_argument("--caseroot",
                         help="Case directory to build")
 
+    parser.add_argument("--skip-preview-namelist", action="store_true",
+                        help="Skip calling preview-namelist during case.run")
+
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
     if args.caseroot is not None:
         os.chdir(args.caseroot)
 
-    return args.caseroot
+    if args.skip_preview_namelist is None:
+        args.skip_preview_namelist = False
+
+    return args.caseroot, args.skip_preview_namelist
 
 ###############################################################################
 def _main_func(description):
@@ -66,9 +68,9 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
-    caseroot = parse_command_line(sys.argv, description)
+    caseroot, skip_pnl = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
-        success = case_run(case)
+        success = case_run(case, skip_pnl=skip_pnl)
 
     sys.exit(0 if success else 1)
 

scripts/Tools/case.submit

@@ -12,16 +12,15 @@ from CIME.case        import Case
 def parse_command_line(args, description):
 ###############################################################################
     parser = argparse.ArgumentParser(
-        usage="""\n%s [<casedir>] [--verbose]
+        usage="""\n{0} [<casedir>] [--verbose]
 OR
-%s --help
+{0} --help
 OR
-%s --test
-
+{0} --test
 \033[1mEXAMPLES:\033[0m
     \033[1;32m# Setup case \033[0m
-    > %s
-""" % ((os.path.basename(args[0]), ) * 4),
+    > {0}
+""".format(os.path.basename(args[0])),
         description=description,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
@@ -31,6 +30,9 @@ OR
     parser.add_argument("caseroot", nargs="?", default=os.getcwd(),
                         help="Case directory to setup")
 
+    parser.add_argument("--test", action="store_true",
+                        help="Run case as a test.")
+
     parser.add_argument("--job", "-j",
                         help="Name of the job to be submitted, default is case.run"
                         " can be any of the jobs listed in env_batch.xml")
@@ -45,25 +47,30 @@ OR
     parser.add_argument("--resubmit", action="store_true",
                         help="Used with tests only, to continue rather than restart a test. ")
 
+    parser.add_argument("--skip-preview-namelist", action="store_true",
+                        help="Skip calling preview-namelist during case.run")
+
     parser.add_argument("-a", "--batch-args",
                         help="Used to pass additional arguments to batch system. ")
 
     args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser)
 
     CIME.utils.expect(args.prereq is None, "--prereq not currently supported")
 
-    return args.caseroot, args.job, args.no_batch, args.resubmit, args.batch_args
+    return args.test, args.caseroot, args.job, args.no_batch, args.resubmit, args.skip_preview_namelist, args.batch_args
 
 ###############################################################################
 def _main_func(description):
 ###############################################################################
-    if "--test" in sys.argv:
+    test, caseroot, job, no_batch, resubmit, skip_pnl, batch_args = parse_command_line(sys.argv, description)
+    if test:
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
-    caseroot, job, no_batch, resubmit, batch_args = parse_command_line(sys.argv, description)
     with Case(caseroot, read_only=False) as case:
-        submit(case, job=job, no_batch=no_batch, resubmit=resubmit, batch_args=batch_args)
+        submit(case, job=job, no_batch=no_batch,
+               resubmit=resubmit, skip_pnl=skip_pnl,
+               batch_args=batch_args)
 
 if __name__ == "__main__":
     _main_func(__doc__)

scripts/lib/CIME/XML/env_batch.py

@@ -44,7 +44,7 @@ def set_value(self, item, value, subgroup=None, ignore_type=False):
                 elif value.count(":") == 2:
                     t_spec = "%H:%M:%S"
                 else:
-                    expect(False, "could not interpret format for wallclock time %s"%value)
+                    expect(False, "could not interpret format for wallclock time {}".format(value))
                 value = format_time(walltime_format, t_spec, value)
 
         # allow the user to set item for all jobs if subgroup is not provided
@@ -91,7 +91,7 @@ def get_type_info(self, vid):
                 type_info = new_type_info
             else:
                 expect( type_info == new_type_info,
-                        "Inconsistent type_info for entry id=%s %s %s" % (vid, new_type_info, type_info))
+                        "Inconsistent type_info for entry id={} {} {}".format(vid, new_type_info, type_info))
         return type_info
 
     def get_jobs(self):
@@ -153,7 +153,7 @@ def set_batch_system(self, batchobj, batch_system_type=None):
             self.root.append(deepcopy(batchobj.machine_node))
 
     def make_batch_script(self, input_template, job, case, total_tasks, tasks_per_node, num_nodes, thread_count):
-        expect(os.path.exists(input_template), "input file '%s' does not exist" % input_template)
+        expect(os.path.exists(input_template), "input file '{}' does not exist".format(input_template))
 
         self.tasks_per_node = tasks_per_node
         self.num_tasks = total_tasks
@@ -198,7 +198,7 @@ def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=N
 
             if force_queue:
                 if not self.queue_meets_spec(force_queue, task_count, walltime=walltime, job=job):
-                    logger.warning("WARNING: User-requested queue '%s' does not meet requirements for job '%s'" % (force_queue, job))
+                    logger.warning("WARNING: User-requested queue '{}' does not meet requirements for job '{}'".format(force_queue, job))
                 queue = force_queue
             else:
                 queue = self.select_best_queue(task_count, walltime=walltime, job=job)
@@ -209,9 +209,9 @@ def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=N
                         # It was, override the walltime if a test, otherwise just warn the user
                         new_walltime = self.get_queue_specs(queue)[3]
                         expect(new_walltime is not None, "Should never make it here")
-                        logger.warning("WARNING: Requested walltime '%s' could not be matched by any queue" % walltime)
+                        logger.warning("WARNING: Requested walltime '{}' could not be matched by any queue".format(walltime))
                         if allow_walltime_override:
-                            logger.warning("  Using walltime '%s' instead" % new_walltime)
+                            logger.warning("  Using walltime '{}' instead".format(new_walltime))
                             walltime = new_walltime
                         else:
                             logger.warning("  Continuing with suspect walltime, batch submission may fail")
@@ -235,7 +235,7 @@ def set_job_defaults(self, batch_jobs, pesize=None, walltime=None, force_queue=N
 
             self.set_value("JOB_QUEUE", queue, subgroup=job)
             self.set_value("JOB_WALLCLOCK_TIME", walltime, subgroup=job)
-            logger.debug("Job %s queue %s walltime %s" % (job, queue, walltime))
+            logger.debug("Job {} queue {} walltime {}".format(job, queue, walltime))
 
     def get_batch_directives(self, case, job, raw=False):
         """
@@ -255,7 +255,7 @@ def get_batch_directives(self, case, job, raw=False):
                         directive = transform_vars(directive, case=case, subgroup=job, default=default, check_members=self)
                     elif default is not None:
                         directive = transform_vars(directive, default=default)
-                    result.append("%s %s" % (directive_prefix, directive))
+                    result.append("{} {}".format(directive_prefix, directive))
 
         return "\n".join(result)
 
@@ -278,7 +278,7 @@ def get_submit_args(self, case, job):
                     continue
 
             if name is None:
-                submitargs+=" %s"%flag
+                submitargs+=" {}".format(flag)
             else:
                 if name.startswith("$"):
                     name = name[1:]
@@ -302,23 +302,23 @@ def get_submit_args(self, case, job):
 
                     if flag.rfind("=", len(flag)-1, len(flag)) >= 0 or\
                        flag.rfind(":", len(flag)-1, len(flag)) >= 0:
-                        submitargs+=" %s%s"%(flag,str(rval).strip())
+                        submitargs+=" {}{}".format(flag,str(rval).strip())
                     else:
-                        submitargs+=" %s %s"%(flag,str(rval).strip())
+                        submitargs+=" {} {}".format(flag,str(rval).strip())
 
         return submitargs
 
-    def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=False):
+    def submit_jobs(self, case, no_batch=False, job=None, skip_pnl=False, batch_args=None, dry_run=False):
         alljobs = self.get_jobs()
         startindex = 0
         jobs = []
         firstjob = job
         if job is not None:
-            expect(job in alljobs, "Do not know about batch job %s"%job)
+            expect(job in alljobs, "Do not know about batch job {}".format(job))
             startindex = alljobs.index(job)
 
         for index, job in enumerate(alljobs):
-            logger.debug( "Index %d job %s startindex %d" % (index, job, startindex))
+            logger.debug( "Index {:d} job {} startindex {:d}".format(index, job, startindex))
             if index < startindex:
                 continue
             try:
@@ -329,8 +329,7 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=F
                     prereq = case.get_resolved_value(prereq)
                     prereq = eval(prereq)
             except:
-                expect(False,"Unable to evaluate prereq expression '%s' for job '%s'"%(self.get_value('prereq',subgroup=job), job))
-
+                expect(False,"Unable to evaluate prereq expression '{}' for job '{}'".format(self.get_value('prereq',subgroup=job), job))
             if prereq:
                 jobs.append((job, self.get_value('dependency', subgroup=job)))
 
@@ -362,8 +361,8 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=F
             if slen == 0:
                 jobid = None
 
-            logger.warn("job is %s" % job)
-            result = self._submit_single_job(case, job, jobid, no_batch=no_batch, batch_args=batch_args, dry_run=dry_run)
+            logger.warn("job is {}".format(job))
+            result = self._submit_single_job(case, job, jobid, no_batch=no_batch, skip_pnl=skip_pnl, batch_args=batch_args, dry_run=dry_run)
             batch_job_id = str(alljobs.index(job)) if dry_run else result
             depid[job] = batch_job_id
             jobcmds.append( (job, result) )
@@ -375,16 +374,16 @@ def submit_jobs(self, case, no_batch=False, job=None, batch_args=None, dry_run=F
         else:
             return sorted(list(depid.values()))
 
-    def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=None, dry_run=False):
-        logger.warn("Submit job %s"%job)
+    def _submit_single_job(self, case, job, depid=None, no_batch=False, skip_pnl=False, batch_args=None, dry_run=False):
+        logger.warn("Submit job {}".format(job))
         batch_system = self.get_value("BATCH_SYSTEM", subgroup=None)
         if batch_system is None or batch_system == "none" or no_batch:
             # Import here to avoid circular include
             from CIME.case_test       import case_test # pylint: disable=unused-variable
             from CIME.case_run        import case_run # pylint: disable=unused-variable
             from CIME.case_st_archive import case_st_archive # pylint: disable=unused-variable
 
-            logger.info("Starting job script %s" % job)
+            logger.info("Starting job script {}".format(job))
 
             function_name = job.replace(".", "_")
             if not dry_run:
@@ -414,13 +413,16 @@ def _submit_single_job(self, case, job, depid=None, no_batch=False, batch_args=N
             if  string is not None:
                 submitcmd += string + " "
 
+        if job == 'case.run' and skip_pnl:
+            submitcmd += " --skip-preview-namelist"
+
         if dry_run:
             return submitcmd
         else:
-            logger.info("Submitting job script %s"%submitcmd)
+            logger.info("Submitting job script {}".format(submitcmd))
             output = run_cmd_no_fail(submitcmd, combine_output=True)
             jobid = self.get_job_id(output)
-            logger.info("Submitted job id is %s"%jobid)
+            logger.info("Submitted job id is {}".format(jobid))
             return jobid
 
     def get_batch_system_type(self):
@@ -439,7 +441,7 @@ def get_job_id(self, output):
         expect(jobid_pattern is not None, "Could not find jobid_pattern in env_batch.xml")
         search_match = re.search(jobid_pattern, output)
         expect(search_match is not None,
-               "Couldn't match jobid_pattern '%s' within submit output:\n '%s'" % (jobid_pattern, output))
+               "Couldn't match jobid_pattern '{}' within submit output:\n '{}'".format(jobid_pattern, output))
         jobid = search_match.group(1)
         return jobid
 

scripts/lib/CIME/case.py

@@ -1103,9 +1103,9 @@ def create_clone(self, newcase, keepexe=False, mach_dir=None, project=None, cime
 
         return newcase
 
-    def submit_jobs(self, no_batch=False, job=None, batch_args=None, dry_run=False):
+    def submit_jobs(self, no_batch=False, job=None, skip_pnl=False, batch_args=None, dry_run=False):
         env_batch = self.get_env('batch')
-        return env_batch.submit_jobs(self, no_batch=no_batch, job=job, batch_args=batch_args, dry_run=dry_run)
+        return env_batch.submit_jobs(self, no_batch=no_batch, job=job, skip_pnl=skip_pnl, batch_args=batch_args, dry_run=dry_run)
 
     def get_mpirun_cmd(self, job="case.run"):
         env_mach_specific = self.get_env('mach_specific')