Merge branch 'singhbalwinder/scripts/mach-files-for-constance' (PR #398)

This commit adds machine files for a new PNNL cluster, Constance [BFB]
E3SM-Project · Oct 29, 2015 · 5868c7d · 5868c7d
2 parents a478ed6 + 732ad3d
commit 5868c7d
Show file tree

Hide file tree

Showing 4 changed files with 129 additions and 0 deletions.
diff --git a/cime/machines-acme/config_batch.xml b/cime/machines-acme/config_batch.xml
@@ -145,6 +145,18 @@
      </directives>
    </batch_system>
 
+   <!-- Constance is slurm -->
+
+   <batch_system MACH="constance" version="x.y">
+    <directives>
+       <directive>-A  climate</directive>
+       <directive>--mail-type=END</directive>
+       <directive>--mail-user=email@pnnl.gov</directive>
+       <directive>--output=slurm.out</directive>
+       <directive>--error=slurm.err</directive>
+    </directives>
+   </batch_system>
+
    <batch_system MACH="redsky" version="x.y">
      <directives>
        <directive>--account={{ project }}</directive>

diff --git a/cime/machines-acme/config_compilers.xml b/cime/machines-acme/config_compilers.xml
@@ -918,6 +918,31 @@ for mct, etc.
  <ADD_FFLAGS DEBUG="TRUE" MODEL="cam">  -gline  -C=all  -g  -nan -O0 -v  </ADD_FFLAGS>
  <ADD_SLIBS> -L$(NETCDF_ROOT)/lib -lnetcdf -lnetcdff -L$(MKL_PATH) -lmkl_rt</ADD_SLIBS>
 </compiler>
+
+<compiler COMPILER="pgi" MACH="constance">
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+  <NETCDF_PATH> $(NETCDF_HOME)</NETCDF_PATH>
+  <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
+  <ADD_FFLAGS DEBUG="TRUE">-C -Mbounds -traceback -Mchkfpstk -Mchkstk -Mdalign  -Mdepchk  -Mextend -Miomutex -Mrecursive -Msave -Ktrap=fp -O0 -g -byteswapio -Meh_frame</ADD_FFLAGS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH) -lmkl_rt -L$(MPI_LIB) -lmpich</SLIBS>
+</compiler>
+
+<compiler COMPILER="intel" MACH="constance">
+  <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>
+  <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+  <NETCDF_PATH> $(NETCDF_HOME)</NETCDF_PATH>
+  <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
+  <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
+  <PIO_FILESYSTEM_HINTS>lustre</PIO_FILESYSTEM_HINTS>
+  <ADD_CPPDEFS> -DLINUX </ADD_CPPDEFS>
+  <ADD_FFLAGS DEBUG="TRUE"> -g -traceback  -O0 -fpe0 -check  all -check noarg_temp_created -ftrapuv </ADD_FFLAGS>
+  <SLIBS> -L$(NETCDF_PATH)/lib -lnetcdf -lnetcdff -lpmi -L$(MKL_PATH) -lmkl_rt</SLIBS>
+</compiler>
+
 
 <compiler COMPILER="pgi" MACH="hopper">
   <ADD_CFLAGS DEBUG="FALSE"> -O2 </ADD_CFLAGS>

diff --git a/cime/machines-acme/config_machines.xml b/cime/machines-acme/config_machines.xml
@@ -576,6 +576,49 @@
          <MAX_TASKS_PER_NODE>16</MAX_TASKS_PER_NODE>
 </machine>
 
+<machine MACH="constance">
+         <DESC>PNL Haswell cluster, OS is Linux, batch system is SLURM</DESC>
+         <OS>LINUX</OS>
+         <COMPILERS>pgi,intel</COMPILERS>
+         <MPILIBS>mpich</MPILIBS>
+         <RUNDIR>/pic/scratch/$CCSMUSER/csmruns/$CASE/run</RUNDIR>
+         <EXEROOT>/pic/scratch/$CCSMUSER/csmruns/$CASE/bld</EXEROOT>
+         <CESMSCRATCHROOT>/pic/scratch/$USER</CESMSCRATCHROOT>
+         <DIN_LOC_ROOT>/pic/projects/climate/csmdata/</DIN_LOC_ROOT>
+         <DIN_LOC_ROOT_CLMFORC>/pic/projects/climate/csmdata/atm/datm7</DIN_LOC_ROOT_CLMFORC>
+         <DOUT_S_ROOT>/pic/scratch/$CCSMUSER/archive/$CASE</DOUT_S_ROOT>
+         <DOUT_L_MSROOT>UNSET</DOUT_L_MSROOT>
+         <CCSM_BASELINE>/pic/projects/climate/acme_baselines</CCSM_BASELINE>
+         <CCSM_CPRNC>/pic/projects/climate/acme_baselines/cprnc</CCSM_CPRNC>
+         <BATCHQUERY>squeue</BATCHQUERY>
+         <BATCHSUBMIT>sbatch</BATCHSUBMIT>
+	 <BATCHREDIRECT></BATCHREDIRECT>
+         <SUPPORTED_BY>balwinder.singh -at- pnnl.gov</SUPPORTED_BY>
+         <GMAKE_J>8</GMAKE_J>
+	 <MAX_TASKS_PER_NODE>24</MAX_TASKS_PER_NODE>
+	 <batch_system type="slurm" version="x.y">
+	   <queues>
+	     <queue jobmin="1" jobmax="9999" default="true">batch</queue>
+	   </queues>
+	   <walltimes>
+	     <walltime default="true">0:59:00</walltime>
+	   </walltimes>
+	 </batch_system>
+	 <mpirun mpilib="mpich">
+	   <executable>srun</executable>
+	   <arguments>
+	     <arg name="mpinone"> --mpi=none</arg>
+	     <arg name="num_tasks"> --ntasks={{ num_tasks }}</arg>
+	     <arg name="cpubind"> --cpu_bind=sockets</arg>
+	     <arg name="cpubind"> --cpu_bind=verbose</arg>
+             <arg name="killonbadexit"> --kill-on-bad-exit</arg>
+	   </arguments>
+	 </mpirun>
+	 <mpirun mpilib="mpi-serial">
+           <executable></executable>
+	 </mpirun>
+
+</machine>
 <machine MACH="oic2">
          <DESC>ORNL XK6, os is Linux, 8 pes/node, batch system is PBS</DESC>
          <NODENAME_REGEX>oic2</NODENAME_REGEX>

diff --git a/cime/machines-acme/env_mach_specific.constance b/cime/machines-acme/env_mach_specific.constance
@@ -0,0 +1,49 @@
+#! /bin/csh -f
+#--- set modules
+source /etc/profile.d/modules.csh
+
+#--- unload all modules
+module purge
+unsetenv LD_LIBRARY_PATH
+module load perl/5.20.0
+module load cmake/3.3.0
+
+
+if ( $COMPILER == "intel") then
+    module load intel/15.0.1
+    module load mvapich2/2.1
+    module load netcdf/4.3.2
+
+    setenv MPI_LIB $MPI_ROOT/lib
+
+    module load pnetcdf/1.6.1
+    setenv PNETCDFROOT $PNETCDF_LIB/../
+
+    module load mkl/15.0.1
+    setenv MKL_PATH $MLIB_LIB
+    setenv NETCDF_HOME $NETCDF_LIB/../
+endif
+
+if ( $COMPILER == "pgi") then
+    module load pgi/14.10
+    module load mvapich2/2.1
+    module load netcdf/4.3.2
+
+    module load pnetcdf/1.6.1
+    setenv PNETCDFROOT $NETCDF_LIB/../
+
+    module load mkl/15.0.1
+    setenv MKL_PATH $MLIB_LIB
+    setenv NETCDF_HOME $NETCDF_LIB/../
+endif
+
+#-------------------------------------------------------------------------------
+
+limit coredumpsize 1000000
+limit stacksize unlimited
+
+if ( $?PERL ) then
+  printenv
+endif
+
+