Skip to content

Commit

Permalink
Merge branch 'ndk/machinefiles/cori-haswell-knl' into next (PR #1222)
Browse files Browse the repository at this point in the history 
Update modules and PE layouts for cori (haswell and KNL)
Does not pass all of the acme_developer tests yet.
  • Loading branch information
@ndkeen
ndkeen committed Jan 17, 2017
2 parents 428affa + d35b226 commit 58823d0
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 17 deletions.
115 changes: 113 additions & 2 deletions cime/cime_config/acme/allactive/config_pesall.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="edison">
<mach name="edison|cori-haswell">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
Expand Down Expand Up @@ -590,6 +590,43 @@
</pes>
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>256</ntasks_atm>
<ntasks_lnd>256</ntasks_lnd>
<ntasks_rof>256</ntasks_rof>
<ntasks_ice>256</ntasks_ice>
<ntasks_ocn>256</ntasks_ocn>
<ntasks_glc>256</ntasks_glc>
<ntasks_wav>256</ntasks_wav>
<ntasks_cpl>256</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>4</nthrds_atm>
<nthrds_lnd>4</nthrds_lnd>
<nthrds_rof>4</nthrds_rof>
<nthrds_ice>4</nthrds_ice>
<nthrds_ocn>4</nthrds_ocn>
<nthrds_glc>4</nthrds_glc>
<nthrds_wav>4</nthrds_wav>
<nthrds_cpl>4</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%ne30np4">
<mach name="eos">
<pes compset="any" pesize="any">
Expand Down Expand Up @@ -1552,6 +1589,43 @@
</pes>
</mach>
</grid>
<grid name="a%1.9x2.5">
<mach name="cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>192</ntasks_atm>
<ntasks_lnd>192</ntasks_lnd>
<ntasks_rof>192</ntasks_rof>
<ntasks_ice>192</ntasks_ice>
<ntasks_ocn>192</ntasks_ocn>
<ntasks_glc>192</ntasks_glc>
<ntasks_wav>192</ntasks_wav>
<ntasks_cpl>192</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%1.9x2.5">
<mach name="any">
<pes compset="CAM.+CLM.+DOCN." pesize="T">
Expand Down Expand Up @@ -1700,6 +1774,43 @@
</pes>
</mach>
</grid>
<grid name="a%0.9x1.25">
<mach name="cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
<ntasks_atm>64</ntasks_atm>
<ntasks_lnd>64</ntasks_lnd>
<ntasks_rof>64</ntasks_rof>
<ntasks_ice>64</ntasks_ice>
<ntasks_ocn>64</ntasks_ocn>
<ntasks_glc>64</ntasks_glc>
<ntasks_wav>64</ntasks_wav>
<ntasks_cpl>64</ntasks_cpl>
</ntasks>
<nthrds>
<nthrds_atm>1</nthrds_atm>
<nthrds_lnd>1</nthrds_lnd>
<nthrds_rof>1</nthrds_rof>
<nthrds_ice>1</nthrds_ice>
<nthrds_ocn>1</nthrds_ocn>
<nthrds_glc>1</nthrds_glc>
<nthrds_wav>1</nthrds_wav>
<nthrds_cpl>1</nthrds_cpl>
</nthrds>
<rootpe>
<rootpe_atm>0</rootpe_atm>
<rootpe_lnd>0</rootpe_lnd>
<rootpe_rof>0</rootpe_rof>
<rootpe_ice>0</rootpe_ice>
<rootpe_ocn>0</rootpe_ocn>
<rootpe_glc>0</rootpe_glc>
<rootpe_wav>0</rootpe_wav>
<rootpe_cpl>0</rootpe_cpl>
</rootpe>
</pes>
</mach>
</grid>
<grid name="a%0.9x1.25">
<mach name="any">
<pes compset="any" pesize="T">
Expand Down Expand Up @@ -1923,7 +2034,7 @@
</mach>
</grid>
<grid name="a%ne16np4">
<mach name="edison">
<mach name="edison|cori-haswell|cori-knl">
<pes compset="any" pesize="any">
<comment>none</comment>
<ntasks>
Expand Down
1 change: 0 additions & 1 deletion cime/cime_config/acme/machines/config_batch.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,6 @@
<batch_system MACH="cori-knl" type="slurm" version="x.y">
<directives>
<directive> --constraint=knl,quad,cache</directive>
<directive> --ntasks-per-core=4</directive>
</directives>
<queues>
<queue walltimemax="01:00:00" jobmin="1" jobmax="3000000" default="true">regular</queue>
Expand Down
4 changes: 2 additions & 2 deletions cime/cime_config/acme/machines/config_compilers.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -586,8 +586,8 @@ for mct, etc.
</compiler>

<compiler COMPILER="intel" MACH="cori-knl">
<ADD_FFLAGS> -xMIC-AVX512 </ADD_FFLAGS>
<ADD_CFLAGS> -xMIC-AVX512 </ADD_CFLAGS>
<ADD_FFLAGS> -xMIC-AVX512 -diag-disable 10121</ADD_FFLAGS>
<ADD_CFLAGS> -axMIC-AVX512 -xCORE-AVX2 </ADD_CFLAGS>
<ADD_FFLAGS DEBUG="FALSE"> -O2 -qno-opt-dynamic-align</ADD_FFLAGS>
<ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
<ADD_CFLAGS compile_threaded="true"> -qopenmp </ADD_CFLAGS>
Expand Down
28 changes: 16 additions & 12 deletions cime/cime_config/acme/machines/config_machines.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,14 +254,15 @@
<command name="load">craype/2.5.7</command>
<command name="load">craype-haswell</command>

<command name="load">cray-mpich/7.4.0</command>
<command name="load">cray-mpich/7.4.4</command>
</modules>

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<command name="rm">intel</command>
<!--command name="load">intel/16.0.3.210</command-->
<command name="load">intel/17.0.0.098</command>
<!--command name="load">intel/17.0.0.098</command-->
<command name="load">intel/17.0.1.132</command>
</modules>

<modules compiler="cray">
Expand All @@ -270,12 +271,12 @@
</modules>
<modules compiler="gnu">
<command name="load">PrgEnv-gnu</command>
<command name="switch">gcc gcc/5.3.0</command>
<command name="switch">gcc gcc/6.2.0</command>
</modules>

<modules compiler="!intel">
<command name="rm">cray-libsci</command>
<command name="load">cray-libsci/16.06.1</command>
<command name="load">cray-libsci/16.09.1</command>
</modules>

<modules mpilib="mpi-serial">
Expand Down Expand Up @@ -307,10 +308,9 @@
<env name="OMP_PLACES">threads</env>

</environment_variables>

</machine>

<!-- Think of this as a stub entry. Will have access and can test soon. ndk -->
<!-- KNL nodes of Cori -->
<machine MACH="cori-knl">
<DESC>Cori. XC40 Cray system at NERSC. KNL partition. os is CNL, 68 pes/node (for now only use 64), batch system is SLURM</DESC>
<NODENAME_REGEX>cori-knl-haswell-is-default</NODENAME_REGEX>
Expand Down Expand Up @@ -383,8 +383,10 @@

<modules compiler="intel">
<command name="load">PrgEnv-intel</command>
<!--command name="switch">intel intel/16.0.3.210</command-->
<command name="switch">intel intel/17.0.0.098</command>
<command name="rm">intel</command>
<!--command name="load">intel/16.0.3.210</command-->
<!--command name="load">intel/17.0.0.098</command-->
<command name="load">intel/17.0.1.132</command>
</modules>

<modules compiler="cray">
Expand All @@ -396,24 +398,26 @@
<command name="switch">gcc gcc/6.2.0</command>
</modules>
<modules compiler="!intel">
<command name="switch">cray-libsci/16.06.1</command>
<command name="switch">cray-libsci/16.09.1</command>
</modules>

<modules>
<command name="rm">craype</command>
<command name="load">craype/2.5.7</command>

<command name="rm">craype-haswell</command>
<command name="load">craype-mic-knl</command>
<command name="load">craype-haswell</command>
<!--command name="load">craype-mic-knl</command-->

<command name="load">cray-mpich/7.4.0</command>
<command name="load">cray-mpich/7.4.4</command>
</modules>

<modules mpilib="mpi-serial">
<command name="rm">cray-netcdf-hdf5parallel</command>
<command name="load">cray-hdf5/1.8.16</command>
<command name="load">cray-netcdf/4.4.0</command>
</modules>
<modules mpilib="!mpi-serial">
<command name="rm">cray-netcdf-hdf5parallel</command>
<command name="load">cray-netcdf-hdf5parallel/4.4.0</command>
<command name="load">cray-hdf5-parallel/1.8.16</command>
<command name="load">cray-parallel-netcdf/1.7.0</command>
Expand Down

0 comments on commit 58823d0

Please sign in to comment.