Merge branch 'azamat/perf/add-bench-tests' (PR #3310)

Add performance benchmarking tests for scaling and historical checkpointing: * high- and low-res F-compset without IO * high- and low-res G-compset without IO * high- and low-res W-compset with production IO Also, update PE-layouts on Anvil and Cori-KNL. [BFB]
ESMCI · Dec 16, 2019 · d6e5198 · d6e5198
2 parents e9908f8 + eebfd4e
commit d6e5198
Show file tree

Hide file tree

Showing 12 changed files with 181 additions and 83 deletions.
diff --git a/config/e3sm/allactive/config_pesall.xml b/config/e3sm/allactive/config_pesall.xml
@@ -6443,15 +6443,15 @@
   <grid name="a%ne30np4_">
     <mach name="cori-knl">
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="L">
-        <comment>"cori-knl ne30 coupled compest on 120 nodes, 64x1 (2 threads CPL/OCN/ICE), (kmod125) sypd=4.1"</comment>
+        <comment>"cori-knl ne30 coupled compest on 105 nodes, 64x1 (2 threads CPL/OCN/ICE), (kmod125) sypd=3.6"</comment>
         <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
         <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>6080</ntasks_atm>
           <ntasks_lnd>822</ntasks_lnd>
           <ntasks_rof>128</ntasks_rof>
           <ntasks_ice>5120</ntasks_ice>
-          <ntasks_ocn>1920</ntasks_ocn>
+          <ntasks_ocn>640</ntasks_ocn>
           <ntasks_glc>64</ntasks_glc>
           <ntasks_wav>64</ntasks_wav>
           <ntasks_cpl>5952</ntasks_cpl>
@@ -6478,15 +6478,15 @@
         </rootpe>
       </pes>
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="any">
-        <comment>"cori-knl ne30 coupled compest on 60 nodes, 67x2, (kmod060b) sypd=2.86"</comment>
+        <comment>"cori-knl ne30 coupled compest on 50 nodes, 67x2, (kmod060b) sypd=2.79"</comment>
         <MAX_MPITASKS_PER_NODE>67</MAX_MPITASKS_PER_NODE>
         <MAX_TASKS_PER_NODE>268</MAX_TASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>2700</ntasks_atm>
           <ntasks_lnd>268</ntasks_lnd>
           <ntasks_rof>134</ntasks_rof>
           <ntasks_ice>2560</ntasks_ice>
-          <ntasks_ocn>960</ntasks_ocn>
+          <ntasks_ocn>335</ntasks_ocn>
           <ntasks_glc>67</ntasks_glc>
           <ntasks_wav>67</ntasks_wav>
           <ntasks_cpl>2881</ntasks_cpl>
@@ -6513,15 +6513,15 @@
         </rootpe>
       </pes>
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.*" pesize="S">
-        <comment>"cori-knl ne30 coupled compest on 31 nodes, 67x2, (kmod031b) sypd=1.71"</comment>
+        <comment>"cori-knl ne30 coupled compest on 26 nodes, 67x2, (kmod031b) sypd=1.79"</comment>
         <MAX_MPITASKS_PER_NODE>67</MAX_MPITASKS_PER_NODE>
         <MAX_TASKS_PER_NODE>134</MAX_TASKS_PER_NODE>
         <ntasks>
           <ntasks_atm>1350</ntasks_atm>
           <ntasks_lnd>670</ntasks_lnd>
           <ntasks_rof>64</ntasks_rof>
           <ntasks_ice>1350</ntasks_ice>
-          <ntasks_ocn>600</ntasks_ocn>
+          <ntasks_ocn>268</ntasks_ocn>
           <ntasks_glc>67</ntasks_glc>
           <ntasks_wav>67</ntasks_wav>
           <ntasks_cpl>1350</ntasks_cpl>
@@ -7796,102 +7796,51 @@
   <grid name="a%ne30np4_l%.+_oi%oEC60to30">
     <mach name="anvil|bebop">
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="S">
-        <comment> -compset A_WCYCL* -res ne30_oEC* on 27 nodes pure-MPI </comment>
+        <comment> A_WCYCL*/ne30_oEC* on 29 nodes pure-MPI, sypd=2.88 </comment>
         <ntasks>
-          <ntasks_atm>675</ntasks_atm>
-          <ntasks_lnd>108</ntasks_lnd>
-          <ntasks_rof>108</ntasks_rof>
-          <ntasks_ice>576</ntasks_ice>
-          <ntasks_ocn>288</ntasks_ocn>
-          <ntasks_cpl>576</ntasks_cpl>
-          <ntasks_glc>1</ntasks_glc>
-          <ntasks_wav>1</ntasks_wav>
+          <ntasks_atm>900</ntasks_atm>
+          <ntasks_lnd>180</ntasks_lnd>
+          <ntasks_rof>180</ntasks_rof>
+          <ntasks_ice>720</ntasks_ice>
+          <ntasks_cpl>720</ntasks_cpl>
+          <ntasks_ocn>144</ntasks_ocn>
         </ntasks>
-        <nthrds>
-          <nthrds_atm>1</nthrds_atm>
-          <nthrds_lnd>1</nthrds_lnd>
-          <nthrds_rof>1</nthrds_rof>
-          <nthrds_ice>1</nthrds_ice>
-          <nthrds_ocn>1</nthrds_ocn>
-          <nthrds_glc>1</nthrds_glc>
-          <nthrds_wav>1</nthrds_wav>
-          <nthrds_cpl>1</nthrds_cpl>
-        </nthrds>
         <rootpe>
-          <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>576</rootpe_lnd>
-          <rootpe_rof>576</rootpe_rof>
-          <rootpe_ice>0</rootpe_ice>
-          <rootpe_ocn>684</rootpe_ocn>
-          <rootpe_cpl>0</rootpe_cpl>
-          <rootpe_glc>0</rootpe_glc>
-          <rootpe_wav>0</rootpe_wav>
+          <rootpe_lnd>720</rootpe_lnd>
+          <rootpe_rof>720</rootpe_rof>
+          <rootpe_ocn>900</rootpe_ocn>
         </rootpe>
       </pes>
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="any">
-        <comment> -compset A_WCYCL* -res ne30_oEC* on 54 nodes pure-MPI </comment>
+        <comment> A_WCYCL*/ne30_oEC* on 44 nodes pure-MPI, sypd=4.05 </comment>
         <ntasks>
           <ntasks_atm>1350</ntasks_atm>
-          <ntasks_lnd>216</ntasks_lnd>
-          <ntasks_rof>216</ntasks_rof>
-          <ntasks_ice>1152</ntasks_ice>
-          <ntasks_ocn>576</ntasks_ocn>
-          <ntasks_cpl>1152</ntasks_cpl>
-          <ntasks_glc>1</ntasks_glc>
-          <ntasks_wav>1</ntasks_wav>
+          <ntasks_lnd>72</ntasks_lnd>
+          <ntasks_rof>72</ntasks_rof>
+          <ntasks_ice>1296</ntasks_ice>
+          <ntasks_cpl>1296</ntasks_cpl>
+          <ntasks_ocn>216</ntasks_ocn>
         </ntasks>
-        <nthrds>
-          <nthrds_atm>1</nthrds_atm>
-          <nthrds_lnd>1</nthrds_lnd>
-          <nthrds_rof>1</nthrds_rof>
-          <nthrds_ice>1</nthrds_ice>
-          <nthrds_ocn>1</nthrds_ocn>
-          <nthrds_glc>1</nthrds_glc>
-          <nthrds_wav>1</nthrds_wav>
-          <nthrds_cpl>1</nthrds_cpl>
-        </nthrds>
         <rootpe>
-          <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>1152</rootpe_lnd>
-          <rootpe_rof>1152</rootpe_rof>
-          <rootpe_ice>0</rootpe_ice>
+          <rootpe_lnd>1296</rootpe_lnd>
+          <rootpe_rof>1296</rootpe_rof>
           <rootpe_ocn>1368</rootpe_ocn>
-          <rootpe_cpl>0</rootpe_cpl>
-          <rootpe_glc>0</rootpe_glc>
-          <rootpe_wav>0</rootpe_wav>
         </rootpe>
       </pes>
       <pes compset=".*CAM5.+CLM45.+MPASSI.+MPASO.+MOSART.+SGLC.+SWAV" pesize="L">
-        <comment> -compset A_WCYCL* -res ne30_oEC* on 105 nodes pure-MPI </comment>
+        <comment> A_WCYCL*/ne30_oEC* on 84 nodes pure-MPI, sypd=5.40 </comment>
         <ntasks>
           <ntasks_atm>2700</ntasks_atm>
-          <ntasks_lnd>540</ntasks_lnd>
-          <ntasks_rof>540</ntasks_rof>
-          <ntasks_ice>2160</ntasks_ice>
-          <ntasks_ocn>1080</ntasks_ocn>
-          <ntasks_cpl>2160</ntasks_cpl>
-          <ntasks_glc>1</ntasks_glc>
-          <ntasks_wav>1</ntasks_wav>
+          <ntasks_lnd>108</ntasks_lnd>
+          <ntasks_rof>108</ntasks_rof>
+          <ntasks_ice>2592</ntasks_ice>
+          <ntasks_cpl>2592</ntasks_cpl>
+          <ntasks_ocn>324</ntasks_ocn>
         </ntasks>
-        <nthrds>
-          <nthrds_atm>1</nthrds_atm>
-          <nthrds_lnd>1</nthrds_lnd>
-          <nthrds_rof>1</nthrds_rof>
-          <nthrds_ice>1</nthrds_ice>
-          <nthrds_ocn>1</nthrds_ocn>
-          <nthrds_glc>1</nthrds_glc>
-          <nthrds_wav>1</nthrds_wav>
-          <nthrds_cpl>1</nthrds_cpl>
-        </nthrds>
         <rootpe>
-          <rootpe_atm>0</rootpe_atm>
-          <rootpe_lnd>2160</rootpe_lnd>
-          <rootpe_rof>2160</rootpe_rof>
-          <rootpe_ice>0</rootpe_ice>
+          <rootpe_lnd>2592</rootpe_lnd>
+          <rootpe_rof>2592</rootpe_rof>
           <rootpe_ocn>2700</rootpe_ocn>
-          <rootpe_cpl>0</rootpe_cpl>
-          <rootpe_glc>0</rootpe_glc>
-          <rootpe_wav>0</rootpe_wav>
         </rootpe>
       </pes>
     </mach>

diff --git a/config/e3sm/testmods_dirs/bench/gmpas_noio/shell_commands b/config/e3sm/testmods_dirs/bench/gmpas_noio/shell_commands
@@ -0,0 +1,13 @@
+#!/bin/bash
+# on KNLs, run hyper-threaded with 64x2
+if [ `./xmlquery --value MACH` == theta ]||[ `./xmlquery --value MACH` == cori-knl ]; then
+  echo yes;
+  ./xmlchange MAX_MPITASKS_PER_NODE=64
+  ./xmlchange MAX_TASKS_PER_NODE=128
+  ./xmlchange NTHRDS=2
+else
+  ./xmlchange NTHRDS=1
+fi
+
+#avoid errors on over-decomposing DATM's T62 grid beyond 9600 tasks
+if [ `./xmlquery --value NTASKS_ATM` -gt 9600 ]; then ./xmlchange NTASKS_ATM=9600; fi
diff --git a/config/e3sm/testmods_dirs/bench/gmpas_noio/user_nl_mpaso b/config/e3sm/testmods_dirs/bench/gmpas_noio/user_nl_mpaso
@@ -0,0 +1,2 @@
+config_am_highfrequencyoutput_enable=false
+config_am_globalstats_enable=false
diff --git a/config/e3sm/testmods_dirs/bench/gmpas_noio/user_nl_mpassi b/config/e3sm/testmods_dirs/bench/gmpas_noio/user_nl_mpassi
@@ -0,0 +1,2 @@
+config_am_timeseriesstatsdaily_enable=false
+config_am_regionalstatistics_enable=false
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/hires/shell_commands b/config/e3sm/testmods_dirs/bench/wcycl/hires/shell_commands
@@ -0,0 +1,5 @@
+#!/bin/bash
+./xmlchange --append CAM_CONFIG_OPTS='-cosp'
+./xmlchange --id BUDGETS --val TRUE
+
+./xmlchange PIO_VERSION=2
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_cam b/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_cam
@@ -0,0 +1,13 @@
+
+use_hetfrz_classnuc = .false.
+nhtfrq = 0,-24,-6,-6,-3,-1
+mfilt = 1,30,120,120,240,720
+avgflag_pertape = 'A','A','I','A','I','A'
+fincl1 = 'IEFLX','extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp'
+fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','Z200','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN:M','TREFHTMX:X','QREFHT','TS','PS','TMQ','TUQ','TVQ'
+fincl3 = 'PSL','T200','T500','Z300','Z500','U850','V850','UBOT','VBOT','TREFHT','FLUT','TMQ','TUQ','TVQ'
+fincl4 = 'FLUT','U200','U850','PRECT','PRECC','OMEGA500','PRECSC','PRECSL'
+fincl5 =  'PRECT:A','PRECC:A'
+
+cosp_lite = .true.
+fexcl1 = 'CFAD_SR532_CAL'
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_mpaso b/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_mpaso
@@ -0,0 +1,2 @@
+
+config_am_timeseriesstatsdaily_write_on_startup = .true.
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_mpassi b/config/e3sm/testmods_dirs/bench/wcycl/hires/user_nl_mpassi
@@ -0,0 +1,3 @@
+
+config_reuse_halo_exch = true
+config_am_timeseriesstatsdaily_enable = true
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/lores/shell_commands b/config/e3sm/testmods_dirs/bench/wcycl/lores/shell_commands
@@ -0,0 +1,5 @@
+#!/bin/bash
+./xmlchange --append CAM_CONFIG_OPTS='-cosp'
+./xmlchange --id BUDGETS --val TRUE
+
+./xmlchange PIO_VERSION=2
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/lores/user_nl_cam b/config/e3sm/testmods_dirs/bench/wcycl/lores/user_nl_cam
@@ -0,0 +1,11 @@
+
+ nhtfrq =   -24,-24,-6,-6,-3,-24
+ mfilt  = 1,30,120,120,240,30
+ avgflag_pertape = 'A','A','I','A','A','A'
+ fexcl1 = 'CFAD_SR532_CAL'
+ fincl1 = 'IEFLX','extinct_sw_inp','extinct_lw_bnd7','extinct_lw_inp','CLD_CAL'
+ fincl2 = 'FLUT','PRECT','U200','V200','U850','V850','Z500','OMEGA500','UBOT','VBOT','TREFHT','TREFHTMN','TREFHTMX','QREFHT','TS','PS','TMQ','TUQ','TVQ'
+ fincl3 = 'PSL','T200','T500','U850','V850','UBOT','VBOT','TREFHT'
+ fincl4 = 'FLUT','U200','U850','PRECT','OMEGA500'
+ fincl5 = 'PRECT','PRECC'
+ fincl6 = 'CLDTOT_ISCCP','MEANCLDALB_ISCCP','MEANTAU_ISCCP','MEANPTOP_ISCCP','MEANTB_ISCCP','CLDTOT_CAL','CLDTOT_CAL_LIQ','CLDTOT_CAL_ICE','CLDTOT_CAL_UN','CLDHGH_CAL','CLDHGH_CAL_LIQ','CLDHGH_CAL_ICE','CLDHGH_CAL_UN','CLDMED_CAL','CLDMED_CAL_LIQ','CLDMED_CAL_ICE','CLDMED_CAL_UN','CLDLOW_CAL','CLDLOW_CAL_LIQ','CLDLOW_CAL_ICE','CLDLOW_CAL_UN'
diff --git a/config/e3sm/testmods_dirs/bench/wcycl/lores/user_nl_clm b/config/e3sm/testmods_dirs/bench/wcycl/lores/user_nl_clm
@@ -0,0 +1,2 @@
+
+ check_finidat_year_consistency = .false.
diff --git a/config/e3sm/tests.py b/config/e3sm/tests.py
@@ -231,5 +231,96 @@
                  "ERS.ne4_ne4.FC5AV1C-L.cam-thetanh_ftype2",
                  )
     },
+    "e3sm_bench_hires_g" : {
+        "share"    : True,
+        "time"     : "01:00:00",
+        "tests"    : (
+                 "PFS_P2560.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P2792.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P3072.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P3200.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P4096.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P4800.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P5120.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P5200.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P5584.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P6400.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P7200.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P8192.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P9600.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P11168.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P12000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P12800.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P16000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P16384.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P19200.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P21600.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P22400.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P24000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P25600.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P26000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P28000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P28800.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P30000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P32000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P36000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P48000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P64000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P96000.T62_oRRS18to6v3.GMPAS-IAF.bench-gmpas_noio",
+                 )
+    },
+    "e3sm_bench_hires_f" : {
+        "time"     : "01:00:00",
+        "tests"    : (
+                 "PFS_P7200.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P8640.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P10800.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P14400.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P21600.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P43200.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 "PFS_P86400.ne120_ne120.FC5AV1C-H01A.cam-bench-noio",
+                 )
+    },
+    "e3sm_bench_hires" : {
+        "inherit" : ("e3sm_bench_hires_g", "e3sm_bench_hires_f"),
+        "time"    : "01:00:00",
+        "tests"   : (
+                 "PFS_PS.ne120_oRRS18v3_ICG.A_WCYCL1950S_CMIP6_HR.bench-wcycl-hires",
+                 "PFS_PM.ne120_oRRS18v3_ICG.A_WCYCL1950S_CMIP6_HR.bench-wcycl-hires",
+                 "PFS_PL.ne120_oRRS18v3_ICG.A_WCYCL1950S_CMIP6_HR.bench-wcycl-hires",
+                 )
+    },
+    "e3sm_bench_lores_g" : {
+        "share"    : True,
+        "time"     : "01:00:00",
+        "tests"    : ( 
+                 "PFS_P320.T62_oEC60to30v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P480.T62_oEC60to30v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P640.T62_oEC60to30v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P960.T62_oEC60to30v3.GMPAS-IAF.bench-gmpas_noio",
+                 "PFS_P1280.T62_oEC60to30v3.GMPAS-IAF.bench-gmpas_noio",
+                 )
+    },
+    "e3sm_bench_lores_f" : {
+        "time"     : "01:00:00",
+        "tests"    : (
+                 "PFS_P1350.ne30_ne30.FC5AV1C-L.cam-bench-noio",
+                 "PFS_P2700.ne30_ne30.FC5AV1C-L.cam-bench-noio",
+                 "PFS_P5400.ne30_ne30.FC5AV1C-L.cam-bench-noio",
+                 )
+    },
+    "e3sm_bench_lores" : {
+        "inherit" : ("e3sm_bench_lores_g", "e3sm_bench_lores_f"),
+        "time"    : "01:00:00",
+        "tests"   : (
+                 "PFS_PS.ne30_oECv3_ICG.A_WCYCL1850S_CMIP6.bench-wcycl-lores",
+                 "PFS_PM.ne30_oECv3_ICG.A_WCYCL1850S_CMIP6.bench-wcycl-lores",
+                 "PFS_PL.ne30_oECv3_ICG.A_WCYCL1850S_CMIP6.bench-wcycl-lores",
+                 )
+    },
+    "e3sm_bench_all" : {
+        "inherit" : ("e3sm_bench_hires", "e3sm_bench_lores"),
+        "time"    : "01:00:00",
+    },
 
 }