Skip to content

Commit

Permalink
update modules and PE layouts for cori-knl and cori-haswell
Browse files Browse the repository at this point in the history
  • Loading branch information
@ndkeen
ndkeen committed Jan 16, 2017
1 parent be54f33 commit d35b226
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 33 deletions.
147 changes: 129 additions & 18 deletions cime/cime_config/acme/allactive/config_pesall.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,18 +591,55 @@
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="edison">
<mach name="edison|cori-haswell">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>960</ntasks_atm>
<ntasks_lnd>960</ntasks_lnd>
<ntasks_rof>960</ntasks_rof>
<ntasks_ice>960</ntasks_ice>
<ntasks_ocn>960</ntasks_ocn>
<ntasks_glc>960</ntasks_glc>
<ntasks_wav>960</ntasks_wav>
<ntasks_cpl>960</ntasks_cpl>
<ntasks_atm>192</ntasks_atm>
<ntasks_lnd>192</ntasks_lnd>
<ntasks_rof>192</ntasks_rof>
<ntasks_ice>192</ntasks_ice>
<ntasks_ocn>192</ntasks_ocn>
<ntasks_glc>192</ntasks_glc>
<ntasks_wav>192</ntasks_wav>
<ntasks_cpl>192</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>4</nthrds_atm>
<nthrds_lnd>4</nthrds_lnd>
<nthrds_rof>4</nthrds_rof>
<nthrds_ice>4</nthrds_ice>
<nthrds_ocn>4</nthrds_ocn>
<nthrds_glc>4</nthrds_glc>
<nthrds_wav>4</nthrds_wav>
<nthrds_cpl>4</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>256</ntasks_atm>
<ntasks_lnd>256</ntasks_lnd>
<ntasks_rof>256</ntasks_rof>
<ntasks_ice>256</ntasks_ice>
<ntasks_ocn>256</ntasks_ocn>
<ntasks_glc>256</ntasks_glc>
<ntasks_wav>256</ntasks_wav>
<ntasks_cpl>256</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>4</nthrds_atm>
Expand Down Expand Up @@ -1663,6 +1700,43 @@
</pes>
</mach>
</grid>
<grid name="a%1.9x2.5">
<mach name="cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>192</ntasks_atm>
<ntasks_lnd>192</ntasks_lnd>
<ntasks_rof>192</ntasks_rof>
<ntasks_ice>192</ntasks_ice>
<ntasks_ocn>192</ntasks_ocn>
<ntasks_glc>192</ntasks_glc>
<ntasks_wav>192</ntasks_wav>
<ntasks_cpl>192</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%1.9x2.5">
<mach name="any">
<pes compset="CAM.+CLM.+DOCN." pesize="T">
Expand Down Expand Up @@ -1811,6 +1885,43 @@
</pes>
</mach>
</grid>
<grid name="a%0.9x1.25">
<mach name="cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>64</ntasks_atm>
<ntasks_lnd>64</ntasks_lnd>
<ntasks_rof>64</ntasks_rof>
<ntasks_ice>64</ntasks_ice>
<ntasks_ocn>64</ntasks_ocn>
<ntasks_glc>64</ntasks_glc>
<ntasks_wav>64</ntasks_wav>
<ntasks_cpl>64</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%0.9x1.25">
<mach name="any">
<pes compset="any" pesize="T">
Expand Down Expand Up @@ -2034,18 +2145,18 @@
</mach>
</grid>
<grid name="a%ne16np4">
<mach name="edison">
<mach name="edison|cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>48</ntasks_atm>
<ntasks_lnd>48</ntasks_lnd>
<ntasks_rof>48</ntasks_rof>
<ntasks_ice>48</ntasks_ice>
<ntasks_ocn>48</ntasks_ocn>
<ntasks_glc>48</ntasks_glc>
<ntasks_wav>48</ntasks_wav>
<ntasks_cpl>48</ntasks_cpl>
<ntasks_atm>96</ntasks_atm>
<ntasks_lnd>96</ntasks_lnd>
<ntasks_rof>96</ntasks_rof>
<ntasks_ice>96</ntasks_ice>
<ntasks_ocn>96</ntasks_ocn>
<ntasks_glc>96</ntasks_glc>
<ntasks_wav>96</ntasks_wav>
<ntasks_cpl>96</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
Expand Down
1 change: 0 additions & 1 deletion cime/cime_config/acme/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,6 @@
<batch_system MACH="cori-knl" type="slurm" version="x.y">
<directives>
<directive> --constraint=knl,quad,cache</directive>
<directive> --ntasks-per-core=4</directive>
</directives>
<queues>
<queue walltimemax="01:00:00" jobmin="1" jobmax="3000000" default="true">regular</queue>
Expand Down
4 changes: 2 additions & 2 deletions cime/cime_config/acme/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,8 +588,8 @@ for mct, etc.
</compiler>

<compiler COMPILER="intel" MACH="cori-knl">
<ADD_FFLAGS> -xMIC-AVX512 </ADD_FFLAGS>
<ADD_CFLAGS> -xMIC-AVX512 </ADD_CFLAGS>
<ADD_FFLAGS> -xMIC-AVX512 -diag-disable 10121</ADD_FFLAGS>
<ADD_CFLAGS> -axMIC-AVX512 -xCORE-AVX2 </ADD_CFLAGS>
<ADD_FFLAGS DEBUG="FALSE"> -O2 -qno-opt-dynamic-align</ADD_FFLAGS>
<ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
<ADD_CFLAGS compile_threaded="true"> -qopenmp </ADD_CFLAGS>
Expand Down
28 changes: 16 additions & 12 deletions cime/cime_config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,14 +254,15 @@
<command name="load">craype/2.5.7</command>
<command name="load">craype-haswell</command>

<command name="load">cray-mpich/7.4.0</command>
<command name="load">cray-mpich/7.4.4</command>
</modules>

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<command name="rm">intel</command>
<!--command name="load">intel/16.0.3.210</command-->
<command name="load">intel/17.0.0.098</command>
<!--command name="load">intel/17.0.0.098</command-->
<command name="load">intel/17.0.1.132</command>
</modules>

<modules compiler="cray">
Expand All @@ -270,12 +271,12 @@
</modules>
<modules compiler="gnu">
<command name="load">PrgEnv-gnu</command>
<command name="switch">gcc gcc/5.3.0</command>
<command name="switch">gcc gcc/6.2.0</command>
</modules>

<modules compiler="!intel">
<command name="rm">cray-libsci</command>
<command name="load">cray-libsci/16.06.1</command>
<command name="load">cray-libsci/16.09.1</command>
</modules>

<modules mpilib="mpi-serial">
Expand Down Expand Up @@ -307,10 +308,9 @@
<env name="OMP_PLACES">threads</env>

</environment_variables>

</machine>

<!-- Think of this as a stub entry. Will have access and can test soon. ndk -->
<!-- KNL nodes of Cori -->
<machine MACH="cori-knl">
<DESC>Cori. XC40 Cray system at NERSC. KNL partition. os is CNL, 68 pes/node (for now only use 64), batch system is SLURM</DESC>
<NODENAME_REGEX>cori-knl-haswell-is-default</NODENAME_REGEX>
Expand Down Expand Up @@ -383,8 +383,10 @@

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<!--command name="switch">intel intel/16.0.3.210</command-->
<command name="switch">intel intel/17.0.0.098</command>
<command name="rm">intel</command>
<!--command name="load">intel/16.0.3.210</command-->
<!--command name="load">intel/17.0.0.098</command-->
<command name="load">intel/17.0.1.132</command>
</modules>

<modules compiler="cray">
Expand All @@ -396,24 +398,26 @@
<command name="switch">gcc gcc/6.2.0</command>
</modules>
<modules compiler="!intel">
<command name="switch">cray-libsci/16.06.1</command>
<command name="switch">cray-libsci/16.09.1</command>
</modules>

<modules>
<command name="rm">craype</command>
<command name="load">craype/2.5.7</command>

<command name="rm">craype-haswell</command>
<command name="load">craype-mic-knl</command>
<command name="load">craype-haswell</command>
<!--command name="load">craype-mic-knl</command-->

<command name="load">cray-mpich/7.4.0</command>
<command name="load">cray-mpich/7.4.4</command>
</modules>

<modules mpilib="mpi-serial">
<command name="rm">cray-netcdf-hdf5parallel</command>
<command name="load">cray-hdf5/1.8.16</command>
<command name="load">cray-netcdf/4.4.0</command>
</modules>
<modules mpilib="!mpi-serial">
<command name="rm">cray-netcdf-hdf5parallel</command>
<command name="load">cray-netcdf-hdf5parallel/4.4.0</command>
<command name="load">cray-hdf5-parallel/1.8.16</command>
<command name="load">cray-parallel-netcdf/1.7.0</command>
Expand Down

0 comments on commit d35b226

Please sign in to comment.