Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Jgfouca/branch for acme split 2018 03 27 #2406

Merged
merged 32 commits into from
Mar 27, 2018
Merged

Jgfouca/branch for acme split 2018 03 27 #2406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
78f0a22
Adds NAG compiler machine files for Constance (PNNL)
singhbalwinder Jan 13, 2018
f11c358
Fix domain files setting for CONUS RRM
brhillman Mar 13, 2018
07291dc
Support for nersc_slurm queue type
mfdeakin-sandia Mar 13, 2018
50cda16
Add a test suite for RRM configuration
brhillman Mar 13, 2018
c951f00
Add default PE layout for RRM grids
brhillman Mar 13, 2018
a66e459
Add RRM tests to e3sm_integration suite
brhillman Mar 13, 2018
6a6e7be
Merge branch 'mfdeakin-sandia/run_e3sm_ctest_fix' (PR #2165)
mfdeakin-sandia Mar 14, 2018
8732aa8
Merge origin/brhillman/atm/fix-conus-set_horiz (PR # 2163)
wlin7 Mar 14, 2018
31ec5a9
Move RRM tests to e3sm_extra_coverage
brhillman Mar 15, 2018
3e09abe
Move allactive usermods_dir
rljacob Mar 15, 2018
fec4549
Take CONUS test out of e3sm_extra_coverage
brhillman Mar 15, 2018
cad3031
Add new v1cmip6 testmod
rljacob Mar 16, 2018
67a2013
Add CMIP6 wcycl ne30 test cases
rljacob Mar 16, 2018
aad36ea
Remove SMS.ne30_oECv3_ICG.A_WCYCL1850S from integration
rljacob Mar 16, 2018
e808b92
Make provenance use describe API
jgfouca Mar 20, 2018
91811e6
typo fix
jgfouca Mar 20, 2018
1f6a5af
Adds CHARGE_ACCOUNT for Lawrencium
Mar 21, 2018
a05319f
Updates batch settings for Lawrencium
Mar 21, 2018
7ed2bd2
Merge branch 'rljacob/add-coupled-prod' (PR #2172)
rljacob Mar 21, 2018
e6e5f54
Add a pelayout for conus-rrm test
amametjanov Mar 22, 2018
36476de
Summit machine configuration with PGI,IBM compilers
sarats Mar 22, 2018
49ca9e9
Merge branch 'sarats/machinefiles/summit' into master (PR #2188)
minxu74 Mar 23, 2018
1d40fde
Add PET test mods for Theta
amametjanov Mar 23, 2018
c0032c7
Add mach-pet test mod to all PET tests
amametjanov Mar 23, 2018
ba04902
Make test mod output verbose so that it shows up in stdout/stderr
amametjanov Mar 23, 2018
69b0321
Revert to default wall-time
amametjanov Mar 23, 2018
ce096b8
Merge singhbalwinder/atm/nag-errors-part1 (PR #2029)
wlin7 Mar 23, 2018
248b60a
Merge branch 'azamat/theta/add-pet-testmods' (PR #2190)
amametjanov Mar 23, 2018
880d98c
Merge branch 'bishtgautam/mach/updates-for-lawrencium' (PR #2191)
Mar 23, 2018
5bb6c37
Update cime to ESMCI 5.4.0-alpha.25 (PR #2166)
jgfouca Mar 15, 2018
485551c
Improve runtime of PET_Ln9.ne30_oECv3_ICG.A_WCYCL1850S integration test
jgfouca Mar 27, 2018
d49ef92
Merge esmci_remote_for_split/master
jgfouca Mar 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions config/e3sm/allactive/config_pesall.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1277,6 +1277,43 @@
</pes>
</mach>
</grid>
<grid name="any">
<mach name="sandiatoss3">
<pes compset="any" pesize="S">
<comment>none</comment>
<ntasks>
<ntasks_atm>64</ntasks_atm>
<ntasks_lnd>64</ntasks_lnd>
<ntasks_rof>64</ntasks_rof>
<ntasks_ice>64</ntasks_ice>
<ntasks_ocn>64</ntasks_ocn>
<ntasks_glc>64</ntasks_glc>
<ntasks_wav>64</ntasks_wav>
<ntasks_cpl>64</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="any">
<mach name="anlworkstation">
<pes compset="any" pesize="any">
Expand Down Expand Up @@ -8117,4 +8154,76 @@
</pes>
</mach>
</grid>
<grid name="a%ne0np4.*">
<mach name="any">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>128</ntasks_atm>
<ntasks_lnd>128</ntasks_lnd>
<ntasks_rof>128</ntasks_rof>
<ntasks_ice>128</ntasks_ice>
<ntasks_ocn>128</ntasks_ocn>
<ntasks_glc>128</ntasks_glc>
<ntasks_wav>128</ntasks_wav>
<ntasks_cpl>128</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
<mach name="theta">
<pes compset="any" pesize="any">
<comment>RRM grid on 8 Theta nodes</comment>
<ntasks>
<ntasks_atm>512</ntasks_atm>
<ntasks_lnd>512</ntasks_lnd>
<ntasks_rof>512</ntasks_rof>
<ntasks_ice>512</ntasks_ice>
<ntasks_ocn>512</ntasks_ocn>
<ntasks_glc>512</ntasks_glc>
<ntasks_wav>512</ntasks_wav>
<ntasks_cpl>512</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
</config_pes>
1 change: 1 addition & 0 deletions config/e3sm/config_archive.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@

<comp_archive_spec compname="mosart" compclass="rof">
<rest_file_extension>r</rest_file_extension>
<rest_file_extension>rh\d*</rest_file_extension>
<hist_file_extension>h\d*</hist_file_extension>
<rest_history_varname>unset</rest_history_varname>
<rpointer>
Expand Down
2 changes: 1 addition & 1 deletion config/e3sm/config_files.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@
<type>char</type>
<default_value>unset</default_value>
<values>
<value component="allactive">$CIMEROOT/config/$MODEL/allactive/usermods_dirs</value>
<value component="allactive">$CIMEROOT/config/$MODEL/usermods_dirs</value>
<value component="drv" >$CIMEROOT/src/drivers/mct/cime_config/usermods_dirs</value>
<value component="cam" >$SRCROOT/components/cam/cime_config/usermods_dirs</value>
<value component="cism" >$SRCROOT/components/cism/cime_config/usermods_dirs</value>
Expand Down
22 changes: 22 additions & 0 deletions config/e3sm/config_grids.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1309,6 +1309,7 @@
<support>For testing of the WAV model</support>
</domain>

<!-- RRM grids -->
<domain name="ne0np4_arm_x8v3_lowcon">
<nx>92558</nx>
<ny>1</ny>
Expand Down Expand Up @@ -1343,6 +1344,27 @@
<file ocn_mask="oRRS18to6v3">domain.ocn.twpx4v1_oRRS18to6v3.170629.nc</file>
</domain>

<domain name="ne0np4_conus_x4v1_lowcon">
<nx>89147</nx>
<ny>1</ny>
<desc>1-deg with 1/4-deg over CONUS (version 1):</desc>
<file atm_mask="tx0.1v2">domain.lnd.conusx4v1_tx0.1v2.161129.nc</file>
<file ice_mask="tx0.1v2">domain.ocn.conusx4v1_tx0.1v2.161129.nc</file>
<file lnd_mask="tx0.1v2">domain.lnd.conusx4v1_tx0.1v2.161129.nc</file>
<file ocn_mask="tx0.1v2">domain.ocn.conusx4v1_tx0.1v2.161129.nc</file>
</domain>

<!-- Old mask for CONUS grid -->
<domain name="tx0.1v2">
<nx>3600</nx> <ny>2400</ny>
<file atm_mask="tx0.1v2">domain.ocn.tx0.1v2.090218.nc</file>
<file ice_mask="tx0.1v2">domain.ocn.tx0.1v2.090218.nc</file>
<file lnd_mask="tx0.1v2">domain.ocn.tx0.1v2.090218.nc</file>
<file ocn_mask="tx0.1v2">domain.ocn.tx0.1v2.090218.nc</file>
<desc>tx0.1v2 is an old mask used for CONUS:</desc>
</domain>


<domain name="oQU240">
<nx>7153</nx>
<ny>1</ny>
Expand Down
56 changes: 56 additions & 0 deletions config/e3sm/machines/Depends.summit.pgiacc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
dyn_comp.o: dyn_comp.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<

microp_aero.o: microp_aero.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<



bndry_mod.o: bndry_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

derivative_mod.o: derivative_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

edge_mod.o: edge_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

element_mod.o: element_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

element_state.o: element_state.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

openacc_utils_mod.o: openacc_utils_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

prim_advance_mod.o: prim_advance_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

prim_advection_mod.o: prim_advection_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

prim_si_mod.o: prim_si_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

model_init_mod.o: model_init_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

vertremap_mod.o: vertremap_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

viscosity_mod.o: viscosity_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

prim_driver_mod.o: prim_driver_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

prim_driver_base.o: prim_driver_base.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

physics_mod.o: physics_mod.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

physconst.o: physconst.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,cc70,pinned -Minfo=accel $(FREEFLAGS) $<

60 changes: 18 additions & 42 deletions config/e3sm/machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,40 +99,8 @@
</submit_args>
</batch_system>

<batch_system type="lsf_old" version="10.1">
<batch_query args=" -w" >bjobs</batch_query>
<batch_submit>bsub</batch_submit>
<batch_cancel>bkill</batch_cancel>
<batch_redirect>&lt;</batch_redirect>
<batch_directive>#BSUB</batch_directive>
<jobid_pattern>&lt;(\d+)&gt;</jobid_pattern>
<depend_string>-w 'done(jobid)'</depend_string>
<depend_separator>&amp;&amp;</depend_separator>
<walltime_format>%H:%M</walltime_format>
<batch_mail_flag>-u</batch_mail_flag>
<batch_mail_type_flag></batch_mail_type_flag>
<batch_mail_type></batch_mail_type>
<submit_args>
<arg flag="-q" name="$JOB_QUEUE"/>
<arg flag="-W" name="$JOB_WALLCLOCK_TIME"/>
<arg flag="-P" name="$CHARGE_ACCOUNT"/>
</submit_args>
<directives>
<directive > -n {{ total_tasks }} </directive>
<directive > -R "span[ptile={{ tasks_per_node }}]"</directive>
<directive > -N </directive>
<!-- The following option causes problems with lsf version on Summitdev.
If desired, this should be in specific machine section. -->
<!-- <directive default="poe" > -a {{ poe }} </directive> -->
<directive default="acme.stdout" > -o {{ job_id }}.%J </directive>
<directive default="acme.stderr" > -e {{ job_id }}.%J </directive>
<directive > -J {{ job_id }} </directive>
</directives>
</batch_system>

<!-- This is the version on Summitdev, released as IBM release beta2 on Oct 17, 2017.
Created a new section as it conflicts with previous LSF settings-->
<batch_system type="lsf" version="2">
<!-- This is the new version on Summit, released as IBM 10.1.0.0 build 476197, Nov 21 2017. -->
<batch_system type="lsf" version="10.1">
<batch_query args=" -w" >bjobs</batch_query>
<batch_submit>bsub</batch_submit>
<batch_cancel>bkill</batch_cancel>
Expand All @@ -152,8 +120,8 @@
<directives>
<directive > -nnodes {{ num_nodes }} </directive>
<directive > -N </directive>
<directive default="acme.stdout" > -o {{ output_error_path }}.%J </directive>
<directive default="acme.stderr" > -e {{ output_error_path }}.%J </directive>
<directive default="e3sm.stdout" > -o {{ output_error_path }}.%J </directive>
<directive default="e3sm.stderr" > -e {{ output_error_path }}.%J </directive>
<directive > -J {{ job_id }} </directive>
</directives>
</batch_system>
Expand Down Expand Up @@ -510,6 +478,18 @@
</queues>
</batch_system>

<batch_system MACH="summit" type="lsf" >
<directives>
<directive>-P {{ project }}</directive>
<directive>-alloc_flags gpumps</directive>
<directive>-alloc_flags smt2</directive>
</directives>
<queues>
<queue walltimemax="02:00" nodemin="0" nodemax="64" default="true">batch</queue>
</queues>
</batch_system>


<batch_system MACH="summitdev" type="lsf" >
<directives>
<directive>-P {{ project }}</directive>
Expand All @@ -528,23 +508,19 @@
</batch_system>

<batch_system MACH="lawrencium-lr2" type="slurm" >
<submit_args>
<arg flag="--qos" name="condo_esd2"/>
</submit_args>
<directives>
<directive>--ntasks-per-node={{ tasks_per_node }}</directive>
<directive>--qos=condo_esd2 </directive>
</directives>
<queues>
<queue walltimemax="01:00:00" nodemin="0" nodemax="6" default="true">lr3</queue>
</queues>
</batch_system>

<batch_system MACH="lawrencium-lr3" type="slurm" >
<submit_args>
<arg flag="--qos" name="condo_esd2"/>
</submit_args>
<directives>
<directive>--ntasks-per-node={{ tasks_per_node }}</directive>
<directive>--qos=condo_esd2 </directive>
</directives>
<queues>
<queue walltimemax="01:00:00" nodemin="0" nodemax="4" default="true">lr3</queue>
Expand Down
Loading