Merge branch 'jedwards4b-more_mira_port'

E3SM-Project · Jul 19, 2016 · 1f9f0b3 · 1f9f0b3
2 parents 367d47f + 1df3edd
commit 1f9f0b3
Show file tree

Hide file tree

Showing 14 changed files with 108 additions and 85 deletions.
diff --git a/cime_config/acme/machines/config_batch.xml b/cime_config/acme/machines/config_batch.xml
@@ -39,14 +39,15 @@
      <batch_directive></batch_directive>
      <jobid_pattern>(\d+)</jobid_pattern>
      <depend_string> --dependencies</depend_string>
-    <walltime_format>%H:%M:%S</walltime_format>
+     <walltime_format>%M</walltime_format>
      <submit_args>
-       <arg flag="--cwd" name="$CASEROOT"/>
-       <arg flag="-A" name="$PROJECT"/>
-       <arg flag="-t" name="$JOB_WALLCLOCK_TIME"/>
+       <arg flag="--cwd" name="CASEROOT"/>
+       <arg flag="-A" name="PROJECT"/>
+       <arg flag="-t" name="JOB_WALLCLOCK_TIME"/>
        <arg flag="-n" name="$TOTALPES/$PES_PER_NODE"/>
-       <arg flag="-q" name="$JOB_QUEUE"/>
+       <arg flag="-q" name="JOB_QUEUE"/>
        <arg flag="--mode script"/>
+       <arg flag="--env" name='"CIMEROOT=$CIMEROOT"'/>
      </submit_args>
    </batch_system>
 

diff --git a/cime_config/acme/machines/config_machines.xml b/cime_config/acme/machines/config_machines.xml
@@ -820,7 +820,6 @@
                <arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
                <arg name="omp_stacksize"> --envs OMP_STACKSIZE=64M</arg>
                <arg name="thread_count"> --envs OMP_NUM_THREADS={{ thread_count }}</arg>
-               <arg name="colon">:</arg>
              </arguments>
          </mpirun>
     <module_system type="soft">

diff --git a/cime_config/cesm/allactive/config_compsets.xml b/cime_config/cesm/allactive/config_compsets.xml
@@ -13,7 +13,7 @@
     TIME_ATM[%phys]_LND[%phys]_ICE[%phys]_OCN[%phys]_ROF[%phys]_GLC[%phys]_WAV[%phys][_ESP%phys][_BGC%phys]
     Where for the CAM specific compsets below the following is supported
     TIME = Time period (e.g. 2000, HIST, RCP8...)
-    ATM  = [CAM4, CAM5]
+    ATM  = [CAM40, CAM50, CAM60]
     LND  = [CLM40, CLM45, CLM50, SLND]
     ICE  = [CICE, DICE, SICE]
     OCN  = [DOCN, ,AQUAP, SOCN]
@@ -42,17 +42,17 @@
 
   <compset>
     <alias>B1850Ws</alias>
-    <lname>1850_CAM55_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD</lname>
+    <lname>1850_CAM60_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD</lname>
   </compset>
 
   <compset>
     <alias>B1850</alias>
-    <lname>1850_CAM55_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_WW3_BGC%BDRD</lname>
+    <lname>1850_CAM60_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_WW3_BGC%BDRD</lname>
   </compset>
 
   <compset>
-    <alias>B1850CwWs</alias>  
-    <lname>1850_CAM55%WTSM_CLM50%BGC_CICE_POP2%ECO_MOSART_SGLC_SWAV</lname>
+    <alias>B1850CwWs</alias>
+    <lname>1850_CAM60%WTSM_CLM50%BGC_CICE_POP2%ECO_MOSART_SGLC_SWAV</lname>
   </compset>
 
 
@@ -144,7 +144,7 @@
     <lname>HIST_CAM50_CLM40%CN_CICE_POP2_RTM_SGLC_SWAV</lname>
   </compset>
 
-  <compset> 
+  <compset>
     <alias>BiHISTC5CN</alias>
     <lname>HIST_CAM50_CLM40%CN_WISO_CICE%WISO_POP2%ISO_RTM%WISO_SGLC_SWAV</lname>
   </compset>
@@ -261,7 +261,7 @@
 
   <compset>
     <alias>ETEST</alias>
-    <lname>2000_CAM55_CLM50_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST</lname>
+    <lname>2000_CAM60_CLM50_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST</lname>
   </compset>
 
   <compset>
@@ -281,17 +281,17 @@
 
  <entry id="RUN_TYPE">
       <values>
-	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"	compset="1850_CAM55_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD"	   >hybrid</value>
+	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"	compset="1850_CAM60_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD"	   >hybrid</value>
       </values>
     </entry>
     <entry id="RUN_REFCASE">
       <values>
-	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"   compset="1850_CAM55_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD">b.e15.B1850G.f09_g16.pi_control.25</value>
+	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"   compset="1850_CAM60_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD">b.e15.B1850G.f09_g16.pi_control.25</value>
       </values>
     </entry>
     <entry id="RUN_REFDATE">
       <values>
-	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"   compset="1850_CAM55_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD"			   >0041-01-01</value>
+	<value grid="a%0.9x1.25_l%0.9x1.25_oi%gx1v6_r%r05_m%gx1v6_g%gland5UM_w%null"   compset="1850_CAM60_CLM50%BGC_CICE_POP2%ECO_MOSART_CISM1%NOEVOLVE_SWAV_BGC%BDRD"			   >0041-01-01</value>
       </values>
     </entry>
 <!-- Reference cases from older cesm versions are no longer valid

diff --git a/cime_config/cesm/machines/config_batch.xml b/cime_config/cesm/machines/config_batch.xml
@@ -63,13 +63,14 @@
      <depend_string> --dependencies</depend_string>
     <walltime_format>%M</walltime_format>
      <submit_args>
-       <arg flag="--cwd" name="$CASEROOT"/>
-       <arg flag="-A" name="$PROJECT"/>
-       <arg flag="-t" name="$JOB_WALLCLOCK_TIME"/>
+       <arg flag="--cwd" name="CASEROOT"/>
+       <arg flag="-A" name="PROJECT"/>
+       <arg flag="-t" name="JOB_WALLCLOCK_TIME"/>
        <arg flag="-n" name="$TOTALPES/$PES_PER_NODE"/>
-       <arg flag="-q" name="$JOB_QUEUE"/>
+       <arg flag="-q" name="JOB_QUEUE"/>
        <arg flag="--mode script"/>
-     </submit_args>
+       <arg flag="--env" name='"CIMEROOT=$CIMEROOT"'/>
+      </submit_args>
    </batch_system>
 
   <batch_system type="lsf" version="9.1">
@@ -142,21 +143,21 @@
    </batch_system>
 
   <!-- babbage is PBS -->
-  <batch_system MACH="babbage" version="x.y">
+  <batch_system MACH="babbage" type="pbs">
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
   </batch_system>
 
   <!-- babbageKnc is PBS -->
-  <batch_system MACH="babbageKnc" version="x.y">
+  <batch_system MACH="babbageKnc" type="pbs">
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
   </batch_system>
 
   <!-- brutus is PBS -->
-  <batch_system type="pbs" MACH="brutus" version="x.y">
+  <batch_system type="pbs" MACH="brutus" >
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
@@ -186,25 +187,25 @@
    </batch_system>
 
   <!-- brutus is PBS -->
-  <batch_system MACH="brutus" version="x.y">
+  <batch_system MACH="brutus" type="pbs">
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
   </batch_system>
 
-    <!-- eos is PBS -->
-    <batch_system MACH="eos" version="x.y">
+  <!-- eos is PBS -->
+  <batch_system MACH="eos" type="pbs">
     <jobid_pattern>^(\d+)</jobid_pattern>
     <directives>
       <directive>-A $PROJECT</directive>
       <directive>-l mppwidth={{ mppwidth }}</directive>
       <directive>-l  nodes={{ num_nodes }}</directive>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
-   </batch_system>
+  </batch_system>
 
   <!-- erebus is PBS -->
-  <batch_system MACH="erebus" version="x.y">
+  <batch_system MACH="erebus" type="pbs">
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>
@@ -232,14 +233,6 @@
      </walltimes>
    </batch_system>
 
-  <!-- goldbach is PBS -->
-  <batch_system MACH="goldbach" version="x.y">
-    <directives>
-      <directive>-l nodes={{ num_nodes }}:ppn={{ tasks_per_node }}</directive>
-      <directive default="/bin/bash" > -S {{ shell }}  </directive>
-    </directives>
-  </batch_system>
-
   <!-- hobart is PBS -->
   <batch_system type="pbs" MACH="hobart" version="x.y">
     <directives>
@@ -253,28 +246,14 @@
   </batch_system>
 
    <!-- hera is SLURM -->
-   <batch_system MACH="hera">
+   <batch_system MACH="hera" type="slurm">
    <batch_directive>#MSUB</batch_directive>
    <directives>
      <directive>-A ees</directive>
      <directive>-l gres=lscratchd</directive>
    </directives>
    </batch_system>
 
-  <!-- hopper is PBS -->
-  <batch_system MACH="hopper" version="x.y">
-    <directives>
-      <directive default="/bin/bash" > -S {{ shell }}  </directive>
-    </directives>
-  </batch_system>
-
-  <!-- janus is PBS -->
-  <batch_system MACH="janus" version="x.y">
-    <directives>
-      <directive default="/bin/bash" > -S {{ shell }}  </directive>
-    </directives>
-  </batch_system>
-
   <batch_system MACH="mira" type="cobalt">
     <queues>
       <queue waltimemin="0" walltimemax="360" jobmin="512" jobmax="4096" default="true">default</queue>
@@ -361,7 +340,7 @@
   </batch_system>
 
   <!-- all pleiades machines are PBS -->
-  <batch_system MACH="pleiades-wes" version="x.y">
+  <batch_system MACH="pleiades-wes" version="x.y" type="pbs">
     <jobid_pattern>^(\S+)</jobid_pattern>
     <directives>
       <directive>-W group_list=$PROJECT </directive>
@@ -372,7 +351,7 @@
   </batch_system>
 
   <!-- sierra is SLURM -->
-   <batch_system MACH="sierra">
+   <batch_system MACH="sierra" type="slurm">
    <batch_directive>#MSUB</batch_directive>
    <directives>
      <directive> </directive>
@@ -431,7 +410,7 @@
    </batch_system>
 
   <!-- titan is PBS -->
-  <batch_system MACH="titan" version="x.y">
+  <batch_system MACH="titan" type="pbs">
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
     </directives>

diff --git a/cime_config/cesm/machines/config_machines.xml b/cime_config/cesm/machines/config_machines.xml
@@ -789,18 +789,17 @@
     <PES_PER_NODE>8</PES_PER_NODE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
-      <executable>/usr/bin/runjob</executable>
+       <executable>/usr/bin/runjob</executable>
       <arguments>
 	<arg name="label"> --label short</arg>
 	<!-- Ranks per node!! -->
-	<arg name="tasks_per_node"> -p {{ tasks_per_node }}</arg>
+	<arg name="tasks_per_node"> --ranks-per-node $PES_PER_NODE</arg>
 	<!-- Total MPI Tasks -->
-	<arg name="num_tasks"> -n {{ num_tasks }}</arg>
-	<arg name="locargs">  $LOCARGS</arg>
+	<arg name="num_tasks"> --np $TOTALPES</arg>
+	<arg name="locargs">--block $COBALT_PARTNAME --envs OMP_WAIT_POLICY=active --envs BG_SMP_FAST_WAKEUP=yes $LOCARGS</arg>
 	<arg name="bg_threadlayout"> --envs BG_THREADLAYOUT=1</arg>
 	<arg name="omp_stacksize"> --envs OMP_STACKSIZE=32M</arg>
 	<arg name="thread_count"> --envs OMP_NUM_THREADS={{ thread_count }}</arg>
-	<arg name="colon">:</arg>
       </arguments>
     </mpirun>
     <module_system type="soft">

diff --git a/scripts/Tools/code_checker b/scripts/Tools/code_checker
@@ -11,6 +11,7 @@ from CIME.utils import run_cmd, run_cmd_no_fail, expect, get_python_libs_root
 
 import argparse, sys, os, doctest
 from multiprocessing.dummy import Pool as ThreadPool
+from distutils.spawn import find_executable
 
 ###############################################################################
 def parse_command_line(args, description):
@@ -49,7 +50,10 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
 ###############################################################################
 def run_pylint(on_file):
 ###############################################################################
-    cmd = "pylint --disable I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement %s" % on_file
+    pylint = find_executable("pylint")
+    expect(pylint is not None,"pylint not found")
+
+    cmd = "%s --disable I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement %s" % (pylint, on_file)
     stat = run_cmd(cmd)[0]
     if stat != 0:
         sys.stdout.write("File %s has pylint problems, please fix\n    Use command: %s\n" % (on_file, cmd))

diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py
@@ -202,8 +202,12 @@ def _check_for_memleak(self, cpllog):
             originaldate = int(memlist[0][0])
             finalmem = float(memlist[-1][1])
             originalmem = float(memlist[0][1])
-            memdiff = (finalmem - originalmem)/originalmem
-            if memdiff < 0.1:
+            memdiff = -1
+            if originalmem > 0:
+                memdiff = (finalmem - originalmem)/originalmem
+            if memdiff < 0:
+                append_status("COMMENT: insuffiencient data for memleak test",sfile="TestStatus")
+            elif memdiff < 0.1:
                 append_status("PASS %s memleak"%(self._case.get_value("CASEBASEID")),
                              sfile="TestStatus")
             else:

diff --git a/utils/python/CIME/XML/env_batch.py b/utils/python/CIME/XML/env_batch.py
@@ -315,6 +315,12 @@ def get_submit_args(self, case, job):
         for arg in submit_arg_nodes:
             flag = arg.get("flag")
             name = arg.get("name")
+            if self.batchtype == "cobalt" and job == "case.st_archive":
+                if flag == "-n":
+                    name = 'task_count'
+                if flag == "--mode":
+                    continue
+
             if name is None:
                 submitargs+=" %s"%flag
             else:
@@ -346,16 +352,18 @@ def submit_jobs(self, case, no_batch=False, job=None):
             startindex = alljobs.index(job)
 
         for index, job in enumerate(alljobs):
+            logger.debug( "Index %d job %s startindex %d"%(index, job, startindex))
             if index < startindex:
                 continue
-            logger.debug( "Index %d job %s"%(index, job))
             try:
                 prereq = case.get_resolved_value(self.get_value('prereq', subgroup=job))
                 prereq = eval(prereq)
             except:
                 expect(False,"Unable to evaluate prereq expression '%s' for job '%s'"%(self.get_value('prereq',subgroup=job), job))
             if prereq:
                 jobs.append((job,self.get_value('dependency', subgroup=job)))
+            if self.batchtype == "cobalt":
+                break
         depid = {}
         for job, dependency in jobs:
             if dependency is not None:
@@ -379,9 +387,13 @@ def submit_jobs(self, case, no_batch=False, job=None):
             if slen == 0:
                 jobid = None
 
+            logger.warn("job is %s"%job)
             depid[job] = self.submit_single_job(case, job, jobid, no_batch=no_batch)
+            if self.batchtype == "cobalt":
+                break
 
     def submit_single_job(self, case, job, depid=None, no_batch=False):
+        logger.warn("Submit job %s"%job)
         caseroot = case.get_value("CASEROOT")
         batch_system = self.get_value("BATCH_SYSTEM", subgroup=None)
         if batch_system is None or batch_system == "none" or no_batch:

diff --git a/utils/python/CIME/XML/machines.py b/utils/python/CIME/XML/machines.py
@@ -337,7 +337,10 @@ def get_full_mpirun(self, check_members, case, job):
         executable, args = self.get_mpirun(mpi_attribs, check_members, case, job)
 
         mpi_arg_string = " ".join(args.values())
-
+        batch_system = self.get_value("BATCH_SYSTEM")
+        if batch_system == "cobalt":
+            mpi_arg_string += " : "
+
         return "%s %s %s" % (executable if executable is not None else "", mpi_arg_string, default_run_suffix)
 
     def print_values(self):