Merge branch 'rljacob/cime5.1/update-acme-mach' into maint-cime5.1 (PR …

…#806)

Update several ACME machine files with changes through ACME v1alpha.9

All the Depends.intel* files And mira, cetus, titan, melvin compilers
and config. Add compiler entries for mesabi and itasca

Also remove all the old env_mach_specific.* files to remove clutter.

* rljacob/cime5.1/update-acme-mach:
  Update Depends.intel files for acme
  Add intel entries for mesabi and itasca for acme
  Update melvin machine settings for acme
  Update cetus/mira for acme
  Update titan for acme machines
  Remove old env_mach_specific files from acme

cime_config/acme/machines/Depends.cetus

@@ -1,19 +1,61 @@
+# These routines have problems with stacksize when omp is invoked add -qsmallstack to resolve
+SSOBJS = mo_sethet.o mo_drydep.o time_management.o
+
+$(SSOBJS): %.o: %.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -qsmallstack $<
+
 QSMPFLAGS:=
 ifeq ($(compile_threaded), true)
   QSMPFLAGS += -qsmp=noauto:noomp
 endif
 shr_reprosum_mod.o: shr_reprosum_mod.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(QSMPFLAGS) $<
 
-mo_sethet.o: mo_sethet.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(QSMPFLAGS) $<
-mo_drydep.o: mo_drydep.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(QSMPFLAGS) $<
-time_management.o: time_management.F90
-	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -qsmp=noauto:noomp $<
+# These routines benefit from -qnostrict without violating the bfb test
+PERFOBJS=\
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
+prim_advance_mod.o \
+uwshcu.o \
+wetdep.o
+
+
+ifeq ($(DEBUG),FALSE)
+  $(PERFOBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qnostrict  $<
+endif
+
+### These files take long time to compile with default optimization flags.
+### Reducing optimization gives <1min build-times and little impact on model run time.
+### begin
+## atm files taking more than a minute to compile
+# this takes 9 mins to compile at default -O3 level
+buffer.o: buffer.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $<
+
+## lnd files taking more than a minute to compile
+# this takes 4 mins to compile with -O3 -qsmp=omp
+BiogeophysRestMod.o: BiogeophysRestMod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -qsmp=noopt $<
+
+# this takes 17 mins to compile with -O3 -qsmp=omp
+CNrestMod.o: CNrestMod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -qsmp=noopt $<
+
+# this takes 4 mins to compile with -qsmp=omp
+clmtypeInitMod.o: clmtypeInitMod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qsmp=noopt $<
 
+# this takes 2 mins to compile with -qsmp=omp
+clmtype.o: clmtype.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qsmp=noopt $<
+
 # disable inlining (some issues with pure functions therein)
 advance_xm_wpxp_module.o: advance_xm_wpxp_module.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Q! $<
 advance_wp2_wp3_module.o: advance_wp2_wp3_module.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Q! $<
+### end

cime_config/acme/machines/Depends.intel

@@ -1,9 +1,10 @@
 #
 PERFOBJS=\
-prim_advection_mod.o \
-edge_mod.o \
-derivative_mod.o \
-bndry_mod.o \
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
 prim_advance_mod.o \
 uwshcu.o
 

cime_config/acme/machines/Depends.intel14

@@ -6,23 +6,23 @@
 #ecosys_mod.o: ecosys_mod.F90
 #	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -CU $<
 
-prim_advection_mod.o: prim_advection_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3  -no-prec-div  $<
 
-edge_mod.o: edge_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
 
-derivative_mod.o: derivative_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
+PERFOBJS=\
+prim_advection_mod_base.o \
+vertremap_mod_base.o \
+edge_mod_base.o \
+derivative_mod_base.o \
+bndry_mod_base.o \
+prim_advance_mod.o \
+uwshcu.o \
+wetdep.o
 
-bndry_mod.o: bndry_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-prim_advance_mod.o: prim_advance_mod.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-uwshcu.o: uwshcu.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
-
-wetdep.o: wetdep.F90
-		      $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div  $<
+ifeq ($(DEBUG),FALSE)
+  $(PERFOBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div $<
+  $(REDUCED_OPT_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+  $(REDUCED_PRECISION_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $<
+endif

cime_config/acme/machines/Depends.intelmic

@@ -1,6 +1,6 @@
 
-derivative_mod.o: derivative_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $<
+#derivative_mod_base.o: derivative_mod_base.F90
+#	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $<
 
 shr_ncread_mod.o: shr_ncread_mod.F90
 	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<

cime_config/acme/machines/Depends.intelmic14

@@ -1,6 +1,6 @@
 
-derivative_mod.o: derivative_mod.F90
-	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $<
+#derivative_mod_base.o: derivative_mod_base.F90
+#	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) -O1 $<
 
 shr_ncread_mod.o: shr_ncread_mod.F90
 	 $(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS_NOOPT) $<

cime_config/acme/machines/Depends.mira

@@ -26,6 +26,11 @@ wetdep.o
 ifeq ($(DEBUG),FALSE)
   $(PERFOBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qnostrict  $<
+#Model crashes if these files are compiled with O3(default) optimizations
+  seasalt_model.o: seasalt_model.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O2 $<
+  linoz_data.o: linoz_data.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O2 $<
 endif
 
 ### These files take long time to compile with default optimization flags.

cime_config/acme/machines/Depends.titan.pgi_acc

@@ -1,6 +1,54 @@
 dyn_comp.o: dyn_comp.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 
+microp_aero.o: microp_aero.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
+
+
+
+bndry_mod.o: bndry_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+derivative_mod.o: derivative_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+edge_mod.o: edge_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+element_mod.o: element_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+openacc_utils_mod.o: openacc_utils_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+prim_advance_mod.o: prim_advance_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+prim_advection_mod.o: prim_advection_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+prim_si_mod.o: prim_si_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+solver_init_mod.o: solver_init_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+vertremap_mod.o: vertremap_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+viscosity_mod.o: viscosity_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+prim_driver_mod.o: prim_driver_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+physics_mod.o: physics_mod.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+physconst.o: physconst.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) -DUSE_OPENACC=1 -acc -ta=tesla,pin,cc35,cuda7.5,ptxinfo -Minfo=accel $(FREEFLAGS) $<
+
+
 #uwshcu.o: uwshcu.F90
 #	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS_NOOPT) $(FREEFLAGS) $<
 

cime_config/acme/machines/config_batch.xml

@@ -208,9 +208,7 @@
         <queue walltimemax="06:00:00" jobmin="1" jobmax="786432" default="true">default</queue>
       </queues>
       <walltimes>
-        <walltime default="true">00:30:00</walltime>
-        <walltime ccsm_estcost="-3">01:00:00</walltime>
-        <walltime ccsm_estcost="0">01:00:00</walltime>
+        <walltime default="true">03:00:00</walltime>
       </walltimes>
     </batch_system>
 

cime_config/acme/machines/config_compilers.xml

@@ -239,7 +239,7 @@ for mct, etc.
   <FC_AUTO_R8> -r8 </FC_AUTO_R8>
 
   <FFLAGS>  -i4 -Mlist  -time -Mstack_arrays -Mextend -byteswapio -Mflushz -Kieee  </FFLAGS>
-  <ADD_FFLAGS MODEL="cam"> -acc -ta=tesla,pin,cuda7.0,cc35,ptxinfo -Minfo=accel -DUSE_OPENACC=1 </ADD_FFLAGS>
+  <ADD_FFLAGS MODEL="cam"> </ADD_FFLAGS>
   <FFLAGS_NOOPT> -O0 -g -Ktrap=fp -Mbounds -Kieee  </FFLAGS_NOOPT>
   <ADD_FFLAGS_NOOPT compile_threaded="false"> </ADD_FFLAGS_NOOPT>
   <ADD_FFLAGS_NOOPT compile_threaded="true"> -mp </ADD_FFLAGS_NOOPT>
@@ -251,7 +251,7 @@ for mct, etc.
   <ADD_FFLAGS MODEL="dwav"> -Mnovect </ADD_FFLAGS>
   <ADD_FFLAGS MODEL="dice"> -Mnovect </ADD_FFLAGS>
   <ADD_FFLAGS MODEL="docn"> -Mnovect </ADD_FFLAGS>
-  <LDFLAGS> -time -Wl,--allow-multiple-definition -acc -ta=tesla,pin,cuda7.0,cc35,ptxinfo </LDFLAGS>
+  <LDFLAGS> -time -Wl,--allow-multiple-definition -acc -ta=tesla,pin,cuda7.5,cc35 </LDFLAGS>
   <SCC> pgcc </SCC>
   <SFC> pgf95 </SFC>
   <SCXX> pgc++ </SCXX>
@@ -1217,6 +1217,61 @@ for mct, etc.
 	<GPTL_CPPDEFS> -DHAVE_VPRINTF -DHAVE_GETTIMEOFDAY </GPTL_CPPDEFS>
 </compiler>
 
+
+<compiler COMPILER="intel" MACH="mesabi">
+        <ADD_CPPDEFS> -DFORTRANUNDERSCORE -DNO_R16</ADD_CPPDEFS>
+        <ADD_CFLAGS compile_threaded="true"> -openmp </ADD_CFLAGS>
+        <ADD_FFLAGS compile_threaded="true"> -openmp </ADD_FFLAGS>
+        <ADD_LDFLAGS compile_threaded="true"> -openmp </ADD_LDFLAGS>
+        <FREEFLAGS> -free </FREEFLAGS>
+        <FIXEDFLAGS> -fixed -132 </FIXEDFLAGS>
+        <ADD_FFLAGS DEBUG="TRUE"> -O0 -g -check uninit -check bounds -check pointers -fpe0 </ADD_FFLAGS>
+        <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+        <FFLAGS> -fp-model source -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -I/soft/i
+ntel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </FFLAGS>
+        <CFLAGS> -O2 -fp-model precise -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/inc
+lude </CFLAGS>
+        <FFLAGS_NOOPT> -O0 </FFLAGS_NOOPT>
+        <FC_AUTO_R8> -r8 </FC_AUTO_R8>
+        <SFC> ifort </SFC>        <SCC> icc </SCC>        <SCXX> icpc </SCXX>
+        <MPIFC> mpiifort </MPIFC>
+        <MPICC> mpiicc  </MPICC>
+        <MPICXX> mpiicpc </MPICXX>
+        <CXX_LINKER>FORTRAN</CXX_LINKER>
+        <CXX_LDFLAGS> -cxxlib </CXX_LDFLAGS>
+        <ADD_CPPDEFS> -DCPRINTEL </ADD_CPPDEFS>
+        <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+        <ADD_LDFLAGS> -lnetcdff </ADD_LDFLAGS>
+        <ADD_SLIBS>-L/soft/netcdf/fortran-4.4-intel-sp1-update3-parallel/lib -lnetcdff -L/soft/hdf5/hdf5-1.8.13-intel-2013-sp1-update3-impi-5.0.0.028/lib -openmp -fPIC -lnetcdf -lnetcdf -L/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -lm </ADD_SLIBS>
+</compiler>
+
+<compiler COMPILER="intel" MACH="itasca">
+        <ADD_CPPDEFS> -DFORTRANUNDERSCORE -DNO_R16</ADD_CPPDEFS>
+        <ADD_CFLAGS compile_threaded="true"> -openmp </ADD_CFLAGS>
+        <ADD_FFLAGS compile_threaded="true"> -openmp </ADD_FFLAGS>
+        <ADD_LDFLAGS compile_threaded="true"> -openmp </ADD_LDFLAGS>
+        <FREEFLAGS> -free </FREEFLAGS>
+        <FIXEDFLAGS> -fixed -132 </FIXEDFLAGS>
+        <ADD_FFLAGS DEBUG="TRUE"> -O0 -g -check uninit -check bounds -check pointers -fpe0 </ADD_FFLAGS>
+        <ADD_FFLAGS DEBUG="FALSE"> -O2 </ADD_FFLAGS>
+        <FFLAGS> -fp-model source -convert big_endian -assume byterecl -ftz -traceback -assume realloc_lhs -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </FFLAGS>
+        <CFLAGS> -O2 -fp-model precise -I/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/include </CFLAGS>
+        <FFLAGS_NOOPT> -O0 </FFLAGS_NOOPT>
+        <FC_AUTO_R8> -r8 </FC_AUTO_R8>
+        <SFC> ifort </SFC>
+        <SCC> icc </SCC>
+        <SCXX> icpc </SCXX>
+        <MPIFC> mpiifort </MPIFC>
+        <MPICC> mpiicc  </MPICC>
+        <MPICXX> mpiicpc </MPICXX>
+        <CXX_LINKER>FORTRAN</CXX_LINKER>
+        <CXX_LDFLAGS> -cxxlib </CXX_LDFLAGS>
+        <ADD_CPPDEFS> -DCPRINTEL </ADD_CPPDEFS>
+        <SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
+        <ADD_LDFLAGS> -lnetcdff </ADD_LDFLAGS>
+        <ADD_SLIBS>-L/soft/netcdf/fortran-4.4-intel-sp1-update3-parallel/lib -lnetcdff -L/soft/hdf5/hdf5-1.8.13-intel-2013-sp1-update3-impi-5.0.0.028/lib -openmp -fPIC -lnetcdf -lnetcdf -L/soft/intel/x86_64/2013/composer_xe_2013/composer_xe_2013_sp1.3.174/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_core -lmkl_intel_thread -lpthread -lm </ADD_SLIBS>
+</compiler>
+
 <compiler COMPILER="gnu">
 	<ADD_CFLAGS MODEL="csm_share"> -std=c99 </ADD_CFLAGS>
 </compiler>