Skip to content

Commit

Permalink
Merge branch 'azamat/anvil/improve-defaults' (PR #1225)
Browse files Browse the repository at this point in the history 
Improve Anvil performance:
- Minimal Intel MKL link line to avoid linking with Intel MPI
- Default compiler is Intel (faster)
- Better processor affinities for OpenMPI
- Typical OpenMP settings
- Adding default ne30 F-case PE layouts for Anvil
  - '-pecount S 'small layout for 40 nodes
  - otherwise, a layout for 77 nodes
- Simplify library paths
  - Replace (P)NETCDFROOT with (P)NETCDF_PATH
  - Remove unused MPI_PATH
  - Remove unused MPI_LIB_NAME
  - Remove duplication of common python, cmake modules
  - Replace has-based (P)NETCDF paths with output of shell utils
- Default PE layout of 20 nodes x 36 tasks x 1 thread

[non-BFB] - for Anvil out-of-box cases due to compiler change
Fixes #1135
  • Loading branch information
@amametjanov
amametjanov committed Jan 21, 2017
2 parents 22c780f + d163877 commit b3ca2b1
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 58 deletions.
105 changes: 105 additions & 0 deletions cime_config/acme/allactive/config_pesall.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3993,6 +3993,111 @@
</rootpe>
</pes>
</mach>
<mach name="anvil">
<pes compset="any" pesize="S">
<comment>40nodes-36ppn</comment>
<ntasks>
<ntasks_atm>1350</ntasks_atm>
<ntasks_lnd>72</ntasks_lnd>
<ntasks_rof>72</ntasks_rof>
<ntasks_ice>72</ntasks_ice>
<ntasks_ocn>72</ntasks_ocn>
<ntasks_cpl>72</ntasks_cpl>
<ntasks_glc>1</ntasks_glc>
<ntasks_wav>1</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_cpl>1</nthrds_cpl>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>1350</rootpe_lnd>
<rootpe_rof>1350</rootpe_rof>
<rootpe_ice>1350</rootpe_ice>
<rootpe_ocn>1350</rootpe_ocn>
<rootpe_cpl>1350</rootpe_cpl>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
</rootpe>
</pes>
<pes compset="any" pesize="any">
<comment>77nodes-6ppn-6tpp</comment>
<ntasks>
<ntasks_atm>450</ntasks_atm>
<ntasks_lnd>12</ntasks_lnd>
<ntasks_rof>12</ntasks_rof>
<ntasks_ice>12</ntasks_ice>
<ntasks_ocn>12</ntasks_ocn>
<ntasks_cpl>12</ntasks_cpl>
<ntasks_glc>1</ntasks_glc>
<ntasks_wav>1</ntasks_wav>
</ntasks>
<nthrds>
<nthrds_atm>6</nthrds_atm>
<nthrds_lnd>6</nthrds_lnd>
<nthrds_rof>6</nthrds_rof>
<nthrds_ice>6</nthrds_ice>
<nthrds_ocn>6</nthrds_ocn>
<nthrds_cpl>6</nthrds_cpl>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>450</rootpe_lnd>
<rootpe_rof>450</rootpe_rof>
<rootpe_ice>450</rootpe_ice>
<rootpe_ocn>450</rootpe_ocn>
<rootpe_cpl>450</rootpe_cpl>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
</rootpe>
</pes>
</mach>
</grid>
<grid name="any">
<mach name="anvil">
<pes compset="any" pesize="any">
<comment>default,20nodes*36tasks*1threads</comment>
<ntasks>
<ntasks_atm>720</ntasks_atm>
<ntasks_lnd>720</ntasks_lnd>
<ntasks_rof>720</ntasks_rof>
<ntasks_ice>720</ntasks_ice>
<ntasks_ocn>720</ntasks_ocn>
<ntasks_glc>720</ntasks_glc>
<ntasks_wav>720</ntasks_wav>
<ntasks_cpl>720</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne30np4_l%ne30np4_oi%gx1v6">
<mach name="mira|cetus">
Expand Down
25 changes: 3 additions & 22 deletions cime_config/acme/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1016,39 +1016,20 @@ for mct, etc.
</compiler>

<compiler COMPILER="intel" MACH="anvil">
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich">/blues/gpfs/home/software/spack-0.9.1/opt/spack/linux-centos6-x86_64/intel-16.0.3/mvapich2-2.2b-ow5ikyjehwemgdr4h6k7ii7da6hs6lfz</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich">mpi</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas </ADD_SLIBS>
<ADD_SLIBS> -Wl,-rpath -Wl,$(NETCDFROOT)/lib </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpich"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpich2"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpt"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="openmpi"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mvapich"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="impi"> -mkl=cluster </ADD_SLIBS>
<ADD_SLIBS MPILIB="mpi-serial"> -mkl </ADD_SLIBS>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas -mkl </ADD_SLIBS>
<ADD_SLIBS> -Wl,-rpath -Wl,$(NETCDF_PATH)/lib </ADD_SLIBS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

<compiler COMPILER="gnu" MACH="anvil">
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich">/soft/spack/opt/spack/linux-x86_64/gcc-4.4.7/gcc-5.3.0-fygfl7rvyuiteto27dlhmilp5cstw2o2</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich">mpi</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
<GPTL_CPPDEFS> -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_GETTIMEOFDAY</GPTL_CPPDEFS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

<compiler COMPILER="pgi" MACH="anvil">
<PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
<NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
<MPI_PATH MPILIB="mvapich">soft/spack/opt/spack/linux-x86_64/pgi-16.3-0/mvapich2-2.2b-2t45yukj4ij6ek24fwimzydo2dg6i3n2</MPI_PATH>
<MPI_LIB_NAME MPILIB="mvapich"> mpi</MPI_LIB_NAME>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nc-config --flibs) -llapack -lblas</ADD_SLIBS>
<ADD_SLIBS> -rpath $(NETCDFROOT)/lib </ADD_SLIBS>
<ADD_SLIBS> -rpath $(NETCDF_PATH)/lib </ADD_SLIBS>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
</compiler>

Expand Down
59 changes: 23 additions & 36 deletions cime_config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@
<DESC>ANL/LCRC Linux Cluster</DESC>
<NODENAME_REGEX>b.*.lcrc.anl.gov</NODENAME_REGEX>
<TESTS>acme_integration</TESTS>
<COMPILERS>gnu,intel,pgi</COMPILERS>
<COMPILERS>intel,gnu,pgi</COMPILERS>
<MPILIBS>mvapich,openmpi,mpi-serial</MPILIBS>
<CESMSCRATCHROOT>/lcrc/group/acme/$USER/acme_scratch</CESMSCRATCHROOT>
<SAVE_TIMING_DIR>/lcrc/group/acme</SAVE_TIMING_DIR>
Expand All @@ -925,12 +925,19 @@
<OS>LINUX</OS>
<BATCH_SYSTEM>pbs</BATCH_SYSTEM>
<SUPPORTED_BY>acme</SUPPORTED_BY>
<GMAKE_J>4</GMAKE_J>
<GMAKE_J>8</GMAKE_J>
<MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
<PES_PER_NODE>36</PES_PER_NODE>
<PROJECT_REQUIRED>FALSE</PROJECT_REQUIRED>
<PROJECT>ACME</PROJECT>
<mpirun mpilib="default">
<mpirun mpilib="openmpi">
<executable>mpiexec</executable>
<arguments>
<arg name="num_tasks"> -n $TOTALPES </arg>
<arg name="tasks_per_node"> --map-by ppr:{{ tasks_per_numa }}:socket:PE={{ thread_count }} --bind-to core</arg>
</arguments>
</mpirun>
<mpirun mpilib="mvapich">
<executable>mpiexec</executable>
<arguments>
<arg name="num_tasks"> -n $TOTALPES </arg>
Expand All @@ -944,11 +951,13 @@
<init_path lang="sh">/etc/profile.d/a_softenv.sh</init_path>
<cmd_path lang="csh">soft</cmd_path>
<cmd_path lang="sh">soft</cmd_path>
<modules>
<command name="add">+cmake-2.8.12</command>
<command name="add">+python-2.7</command>
</modules>
<modules compiler="intel">
<command name="add">+intel-16.0.3</command>
<command name="add">+netcdf-c-4.4.0-f77-4.4.4-intel-16.0.3-serial</command>
<command name="add">+cmake-2.8.12</command>
<command name="add">+python-2.7</command>
</modules>
<modules compiler="intel" mpilib="mvapich">
<command name="add">+mvapich2-2.2b-intel-16.0.3-acme</command>
Expand All @@ -961,8 +970,6 @@
<modules compiler="gnu">
<command name="add">+gcc-5.3.0</command>
<command name="add">+netcdf-c-4.4.0-f77-4.4.3-gcc-5.3.0-serial</command>
<command name="add">+cmake-2.8.12</command>
<command name="add">+python-2.7</command>
</modules>
<modules compiler="gnu" mpilib="mvapich">
<command name="add">+mvapich2-2.2b-gcc-5.3.0-acme</command>
Expand All @@ -975,8 +982,6 @@
<modules compiler="pgi">
<command name="add">+pgi-16.3</command>
<command name="add">+netcdf-c-4.4.0-f77-4.4.3-pgi-16.3-serial</command>
<command name="add">+cmake-2.8.12</command>
<command name="add">+python-2.7</command>
</modules>
<modules compiler="pgi" mpilib="mvapich">
<command name="add">+mvapich2-2.2b-pgi-16.3-acme</command>
Expand All @@ -988,34 +993,16 @@
</modules>
</module_system>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
</environment_variables>
<environment_variables compiler="intel">
<env name="NETCDFROOT">/soft/spack/opt/spack/linux-centos6-x86_64/intel-16.0.3/netcdf-4.4.0-ds5gqquzjytf4efz3d5njsvpkfmmrwq7</env>
</environment_variables>
<environment_variables compiler="intel" mpilib="mvapich">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-centos6-x86_64/intel-16.0.3/parallel-netcdf-1.7.0-gabvxdumr3chrvldpid5sqwtkijq6xkr</env>
</environment_variables>
<environment_variables compiler="intel" mpilib="openmpi">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-centos6-x86_64/intel-16.0.3/parallel-netcdf-1.7.0-3gxxcl2twfmvvq7rvs7z3yykaw4t2trr</env>
</environment_variables>
<environment_variables compiler="gnu">
<env name="NETCDFROOT">/soft/spack/opt/spack/linux-x86_64/gcc-5.3.0/netcdf-4.4.0-xezev3j3kyay6obokfjvwfzqc2l6uwh5</env>
<env name="NETCDF_PATH">`which nc-config | xargs dirname | xargs dirname`</env>
<env name="PNETCDF_PATH">`which pnetcdf_version | xargs dirname | xargs dirname`</env>
</environment_variables>
<environment_variables compiler="gnu" mpilib="mvapich">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-x86_64/gcc-5.3.0/parallel-netcdf-1.6.1-njywxqs3qjc75ah5mrziqln3qk2nkzg6</env>
</environment_variables>
<environment_variables compiler="gnu" mpilib="openmpi">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-x86_64/gcc-5.3.0/parallel-netcdf-1.6.1-vdvnv2sfg53mug2rcyalhb47nfxk2ki7</env>
</environment_variables>
<environment_variables compiler="pgi">
<env name="NETCDFROOT">/soft/spack/opt/spack/linux-x86_64/pgi-16.3-0/netcdf-4.4.0-3jeeee43ip5lcnvdgxssbqhnvylxw5gp</env>
</environment_variables>
<environment_variables compiler="pgi" mpilib="mvapich">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-x86_64/pgi-16.3-0/parallel-netcdf-1.6.1-4uarepsh5u2am2bbv2cbct4fl7jombz4</env>
</environment_variables>
<environment_variables compiler="pgi" mpilib="openmpi">
<env name="PNETCDFROOT">/soft/spack/opt/spack/linux-x86_64/pgi-16.3-0/parallel-netcdf-1.6.1-5lmcmthdcd2ewbvtpncpnryzocj5b3bd</env>
<environment_variables>
<env name="OMP_STACKSIZE">256M</env>
<env name="OMP_PROC_BIND">spread</env>
<env name="OMP_PLACES">threads</env>
<env name="OMP_MAX_ACTIVE_LEVELS">1</env>
<env name="KMP_HOT_TEAMS_MODE">1</env>
<env name="KMP_HOT_TEAMS_MAX_LEVEL">1</env>
</environment_variables>
</machine>

Expand Down

0 comments on commit b3ca2b1

Please sign in to comment.