Merge branch 'develop' into atmosphere/develop

This merge introduces fixes from the v5.1 release that were merged to 'develop', and it brings the 'atmosphere/develop' up to date with other infrastructure changes, most notably the switch from 'write(0,*)' statements to 'call mpas_log_write' for writing log messages. * develop: (98 commits) Initialize garbage elements for ke_edge and ke_vertex in atm_mpas_init_block Convert several 'write' statements introduced in v5.1 to mpas_log_write calls Note in description of config_len_disp that it is also used by 3-d div damping Threading change to mpas_vector_reconstruction by Phil Jones Framework only: bit-for-bit optimization for OpenMP by Abhinav Make a warning if a stream has duplicate contents Minor wording changes to threading level message in mpas_dmpar_init Only print thread level message if threadLevel /= desiredThreadLevel Clarify output message on MPI_THREAD_MULTIPLE Introduce a new, scale-aware 3-d divergence damping scheme for acoustic modes Introduced scaling of the gravity-wave absorbing layer coefficient Change G0 format to Gw Correct incorrect description in log_expand_string Change log write back to write Rename OMP CRITICAL section in mpas_log_write Remove old logging system Comment-out code in MYNN surface-layer scheme to bound 2-m theta Expand "$l" to either T/F (rather than TRUE/FALSE) in log module Convert atmosphere core to use new logging module Enable new log files for atmosphere core ... Conflicts: src/core_atmosphere/physics/mpas_atmphys_driver_lsm.F src/core_atmosphere/physics/mpas_atmphys_driver_radiation_lw.F src/core_atmosphere/physics/mpas_atmphys_driver_radiation_sw.F src/core_atmosphere/physics/mpas_atmphys_manager.F
MPAS-Dev · May 16, 2017 · eb49435 · eb49435
2 parents ee4b28b + e86d115
commit eb49435
Show file tree

Hide file tree

Showing 156 changed files with 4,962 additions and 3,169 deletions.
diff --git a/INSTALL b/INSTALL
@@ -12,7 +12,11 @@ and the gfortran compiler. The gfortran compiler can be installed using homebrew
 (http://brew.sh), or using pre-compiled binaries from the MacOSX HPC website
 (http://hpc.soureforge.net), or it can be compiled by the user from the GNU sources.
 
-The standard clang compiler does not support OpenMP. Users wanting to compile MPAS
+Caveats:
+
+(1) MPAS cannot be compiled with gfortran-clang if GEN_F90=true.
+
+(2) The standard clang compiler does not support OpenMP. Users wanting to compile MPAS
 with OpenMP support on MacOSX will have to install the LLVM clang compiler, which is
 accomplished easiest with homebrew. Since this alternative clang compiler is not in
 the standard search/library path, the user will have to modify the call to the clang
@@ -57,3 +61,10 @@ would become
 ...
 
 assuming that the LLVM clang compiler is installed in /usr/local/opt/llvm.
+
+
+bluegene: Compiling MPAS on IBM Bluegene using the xl compilers
+----------
+All MPAS cores except the ocean compile on IBM Bluegene using the xl compilers. The ocean
+core currently does not work on IBM Bluegene. Known limitations: OPENMP must be disabled
+(OPENMP=false) for compiling, since the xl compilers do not support nested OpenMP directives.
diff --git a/Makefile b/Makefile
@@ -509,7 +509,6 @@ ifeq "$(OPENMP)" "true"
 endif #OPENMP IF
 
 ifeq "$(PRECISION)" "single"
-	FFLAGS += "-DSINGLE_PRECISION"
 	CFLAGS += "-DSINGLE_PRECISION"
 	CXXFLAGS += "-DSINGLE_PRECISION"
 	override CPPFLAGS += "-DSINGLE_PRECISION"

diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-MPAS-v5.0
+MPAS-v5.1
 ====
 
 The Model for Prediction Across Scales (MPAS) is a collaborative project for

diff --git a/src/Makefile.in.ACME b/src/Makefile.in.ACME
@@ -13,7 +13,7 @@ else
 endif
 # End duplicated logic
 
-include $(CASEROOT)/Macros
+include $(CASEROOT)/Macros.make
 
 ifneq ($(wildcard core_$(CORE)/build_options.mk), ) # Check for build_options.mk
     include core_$(CORE)/build_options.mk
@@ -44,14 +44,14 @@ CC=$(MPICC)
 CXX=$(MPICXX)
 NETCDF=$(NETCDF_PATH)
 PNETCDF=$(PNETCDF_PATH)
-PIO=$(EXEROOT)/pio
+PIO=$(INSTALL_SHAREDPATH)/pio
 FILE_OFFSET = -DOFFSET64BIT
 override CFLAGS += -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -DMPAS_PERF_MOD_TIMERS
 override FFLAGS += -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -DMPAS_PERF_MOD_TIMERS
 override CPPFLAGS += $(CPPDEFS) $(MODEL_FORMULATION) $(FILE_OFFSET) $(ZOLTAN_DEFINE) -DMPAS_NO_LOG_REDIRECT -DMPAS_NO_ESMF_INIT -DMPAS_ESM_SHR_CONST -D_MPI -DMPAS_NAMELIST_SUFFIX=$(NAMELIST_SUFFIX) -DMPAS_EXE_NAME=$(EXE_NAME) -DMPAS_PERF_MOD_TIMERS
-override CPPINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(SHAREDPATH)/include -I$(SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include
-override FCINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(SHAREDPATH)/include -I$(SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include
-LIBS += -L$(PIO) -L$(PNETCDF)/lib -L$(NETCDF)/lib -L$(LIBROOT) -L$(SHAREDPATH)/lib -lpio -lpnetcdf -lnetcdf
+override CPPINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(INSTALL_SHAREDPATH)/include -I$(INSTALL_SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include
+override FCINCLUDES += -I$(EXEROOT)/$(COMPONENT)/source/inc -I$(INSTALL_SHAREDPATH)/include -I$(INSTALL_SHAREDPATH)/$(COMP_INTERFACE)/$(ESMFDIR)/$(NINST_VALUE)/csm_share -I$(NETCDF)/include -I$(PIO) -I$(PNETCDF)/include
+LIBS += -L$(PIO) -L$(PNETCDF)/lib -L$(NETCDF)/lib -L$(LIBROOT) -L$(INSTALL_SHAREDPATH)/lib -lpio -lpnetcdf -lnetcdf
 
 ifneq (,$(findstring FORTRANUNDERSCORE, $(CPPFLAGS)))
 ifeq (,$(findstring DUNDERSCORE, $(CPPFLAGS)))

diff --git a/src/Makefile.in.CESM b/src/Makefile.in.CESM
@@ -13,7 +13,7 @@ else
 endif
 # End duplicated logic
 
-include $(CASEROOT)/Macros
+include $(CASEROOT)/Macros.make
 
 ifneq ($(wildcard core_$(CORE)/build_options.mk), ) # Check for build_options.mk
     include core_$(CORE)/build_options.mk

diff --git a/src/core_atmosphere/Registry.xml b/src/core_atmosphere/Registry.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0"?>
-<registry model="mpas" core="atmosphere" core_abbrev="atm" version="5.0">
+<registry model="mpas" core="atmosphere" core_abbrev="atm" version="5.1">
 
 <!-- **************************************************************************************** -->
 <!-- ************************************** Dimensions ************************************** -->
@@ -141,7 +141,7 @@
 
                 <nml_option name="config_len_disp" type="real" default_value="120000.0"
                      units="m"
-                     description="Horizontal length scale for Smagorinsky formulation of horizontal diffusion"
+                     description="Horizontal length scale, used by the Smagorinsky formulation of horizontal diffusion and by 3-d divergence damping"
                      possible_values="Positive real values"/>
 
                 <nml_option name="config_visc4_2dsmag" type="real" default_value="0.05"
@@ -224,16 +224,11 @@
                      description="Off-centering parameter for the vertically implicit acoustic integration"
                      possible_values="Positive real values"/>
 
-                <nml_option name="config_smdiv" type="real" default_value="0.025"
+                <nml_option name="config_smdiv" type="real" default_value="0.1"
                      units="-"
                      description="3-d divergence damping coefficient"
                      possible_values="Positive real values"/>
 
-                <nml_option name="config_smdiv_p_forward" type="real" default_value="0.1"
-                     units="-"
-                     description="Fractional forward weighting of the pressure during the acoustic step --- a form of 3D divergence damping"
-                     possible_values="Real values typically in the range 0 to 0.2"/>
-
                 <nml_option name="config_apvm_upwinding" type="real" default_value="0.5" in_defaults="false"
                      units="-"
                      description="Amount of upwinding in APVM"
@@ -1588,9 +1583,6 @@
                 <var name="rho_p_save" type="real" dimensions="nVertLevels nCells Time" units="kg m^{-3}"
                      description="predicted value rho_p, saved before acoustic steps"/>
 
-                <var name="divergence_3d" type="real" dimensions="nVertLevels nCells Time" units="kg m^{-3} s^{-1}"
-                     description="3D divergence used for acoustic filtering"/>
-
                 <var name="kdiff" type="real" dimensions="nVertLevels nCells Time" units="m^2 s^{-1}"
                      description="Smagorinsky horizontal eddy viscosity"/>
 

diff --git a/src/core_atmosphere/diagnostics/convective_diagnostics.F b/src/core_atmosphere/diagnostics/convective_diagnostics.F
@@ -7,8 +7,9 @@
 !
 module convective_diagnostics
 
-    use mpas_derived_types, only : MPAS_pool_type, MPAS_clock_type
+    use mpas_derived_types, only : MPAS_pool_type, MPAS_clock_type, MPAS_LOG_ERR, MPAS_LOG_CRIT
     use mpas_kind_types, only : RKIND
+    use mpas_log, only : mpas_log_write
 
     type (MPAS_pool_type), pointer :: mesh
     type (MPAS_pool_type), pointer :: state
@@ -748,7 +749,7 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
         endif
       enddo
     ENDIF
-    if(debug_level.ge.100) print *,'  kmax,maxthe = ',kmax,maxthe
+    if(debug_level.ge.100) call mpas_log_write('  kmax,maxthe = $i $r', intArgs=(/kmax/), realArgs=(/maxthe/))
 
   ELSEIF(source.eq.3)THEN
     ! use mixed layer
@@ -775,13 +776,12 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
       avgth = 0.0
       avgqv = 0.0
       k = 2
-      if(debug_level.ge.100) print *,'  ml_depth = ',ml_depth
-      if(debug_level.ge.100) print *,'  k,z,th,q:'
-      if(debug_level.ge.100) print *,1,z(1),th(1),q(1)
+      if(debug_level.ge.100) call mpas_log_write('  ml_depth = $r', realArgs=(/ml_depth/))
+      if(debug_level.ge.100) call mpas_log_write('  k,z,th,q: $i $r $r $r', intArgs=(/1/), realArgs=(/z(1),th(1),q(1)/))
 
       do while( (z(k).le.ml_depth) .and. (k.le.nk) )
 
-        if(debug_level.ge.100) print *,k,z(k),th(k),q(k)
+        if(debug_level.ge.100) call mpas_log_write('$i $r $r $r', intArgs=(/k/), realArgs=(/z(k),th(k),q(k)/))
 
         avgth = avgth + 0.5*(z(k)-z(k-1))*(th(k)+th(k-1))
         avgqv = avgqv + 0.5*(z(k)-z(k-1))*(q(k)+q(k-1))
@@ -793,12 +793,12 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
       th2 = th(k-1)+(th(k)-th(k-1))*(ml_depth-z(k-1))/(z(k)-z(k-1))
       qv2 =  q(k-1)+( q(k)- q(k-1))*(ml_depth-z(k-1))/(z(k)-z(k-1))
 
-      if(debug_level.ge.100) print *,999,ml_depth,th2,qv2
+      if(debug_level.ge.100) call mpas_log_write('999 $r $r $r', realArgs=(/ml_depth,th2,qv2/))
 
       avgth = avgth + 0.5*(ml_depth-z(k-1))*(th2+th(k-1))
       avgqv = avgqv + 0.5*(ml_depth-z(k-1))*(qv2+q(k-1))
 
-      if(debug_level.ge.100) print *,k,z(k),th(k),q(k)
+      if(debug_level.ge.100) call mpas_log_write('$i $r $r $r', intArgs=(/k/), realArgs=(/z(k),th(k),q(k)/))
 
       avgth = avgth/ml_depth
       avgqv = avgqv/ml_depth
@@ -807,17 +807,16 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
 
     ENDIF
 
-    if(debug_level.ge.100) print *,avgth,avgqv
+    if(debug_level.ge.100) call mpas_log_write('$r $r', realArgs=real((/avgth,avgqv/),kind=RKIND))
 
   ELSE
 
-!    print *
-!    print *,'  Unknown value for source'
-!    print *
-!    print *,'  source = ',source
-!    print *
-!    stop
-    write(0,*) 'getcape: unknown value for source'
+!    call mpas_log_write('')
+!    call mpas_log_write('  Unknown value for source')
+!    call mpas_log_write('')
+!    call mpas_log_write('  source = $i', intArgs=(/source/))
+!    call mpas_log_write('', messageType=MPAS_LOG_CRIT)
+    call mpas_log_write('getcape: unknown value for source', messageType=MPAS_LOG_ERR)
     return
 
   ENDIF
@@ -862,13 +861,13 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
     endif
 
       the = getthe(p2,t2,t2,qv2)
-      if(debug_level.ge.100) print *,'  the = ',the
+      if(debug_level.ge.100) call mpas_log_write('  the = $r', realArgs=(/the/))
 
 !---- begin ascent of parcel ----!
 
       if(debug_level.ge.100)then
-        print *,'  Start loop:'
-        print *,'  p2,th2,qv2 = ',p2,th2,qv2
+        call mpas_log_write('  Start loop:')
+        call mpas_log_write('  p2,th2,qv2 = $r $r $r', realArgs=(/p2,th2,qv2/))
       endif
 
     do while( doit .and. (k.lt.nk) )
@@ -932,16 +931,16 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
                        +lhs*(qi2-qi1)/(cpm*tbar)     &
                        +(rm/cpm-rd/cp)*alog(p2/p1) )
 
-          if(i .gt. 90 .and. debug_level .gt. 0) print *,i,th2,thlast,th2-thlast
+          if(i .gt. 90 .and. debug_level .gt. 0) call mpas_log_write('$i $r $r $r', intArgs=(/i/), realArgs=(/th2,thlast,th2-thlast/))
           if(i .gt. 100)then
-!            print *
-!            print *,'  Error:  lack of convergence'
-!            print *
-!            print *,'  ... stopping iteration '
-!            print *
+!            call mpas_log_write('')
+!            call mpas_log_write('  Error:  lack of convergence')
+!            call mpas_log_write('')
+!            call mpas_log_write('  ... stopping iteration ')
+!            call mpas_log_write('')
 !            stop 1001
             if (debug_level .gt. 0) then
-               write(0,*) 'getcape: lack of convergence'
+               call mpas_log_write('getcape: lack of convergence', messageType=MPAS_LOG_ERR)
             end if
             return
           endif
@@ -963,11 +962,11 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
           ql2 = 0.0
           qi2 = 0.0
         ELSEIF(adiabat.le.0.or.adiabat.ge.5)THEN
-!          print *
-!          print *,'  Undefined adiabat'
-!          print *
+!          call mpas_log_write('')
+!          call mpas_log_write('  Undefined adiabat')
+!          call mpas_log_write('')
 !          stop 10000
-          write(0,*) 'getcape: Undefined adiabat'
+          call mpas_log_write('getcape: Undefined adiabat', messageType=MPAS_LOG_ERR)
           return
         ENDIF
 
@@ -988,11 +987,11 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
         parea =  0.5*b2*dz*frac
         narea = narea-0.5*b1*dz*(1.0-frac)
         if(debug_level.ge.200)then
-          print *,'      b1,b2 = ',b1,b2
-          print *,'      p1,ps,p2 = ',p(k-1),ps,p(k)
-          print *,'      frac = ',frac
-          print *,'      parea = ',parea
-          print *,'      narea = ',narea
+          call mpas_log_write('      b1,b2 = $r $r', realArgs=(/b1,b2/))
+          call mpas_log_write('      p1,ps,p2 = $r $r $r', realArgs=(/p(k-1),ps,p(k)/))
+          call mpas_log_write('      frac = $r', realArgs=(/frac/))
+          call mpas_log_write('      parea = $r', realArgs=(/parea/))
+          call mpas_log_write('      narea = $r', realArgs=(/narea/))
         endif
         cin  = cin  + narea
         narea = 0.0
@@ -1003,11 +1002,11 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
         parea =  0.5*b1*dz*frac
         narea = -0.5*b2*dz*(1.0-frac)
         if(debug_level.ge.200)then
-          print *,'      b1,b2 = ',b1,b2
-          print *,'      p1,ps,p2 = ',p(k-1),ps,p(k)
-          print *,'      frac = ',frac
-          print *,'      parea = ',parea
-          print *,'      narea = ',narea
+          call mpas_log_write('      b1,b2 = $r $r', realArgs=(/b1,b2/))
+          call mpas_log_write('      p1,ps,p2 = $r $r $r', realArgs=(/p(k-1),ps,p(k)/))
+          call mpas_log_write('      frac = $r', realArgs=(/frac/))
+          call mpas_log_write('      parea = $r', realArgs=(/parea/))
+          call mpas_log_write('      narea = $r', realArgs=(/narea/))
         endif
       elseif( b2.lt.0.0 )then
         ! still collecting negative buoyancy
@@ -1022,8 +1021,7 @@ subroutine getcape( nk , p_in , t_in , td_in, cape , cin )
       cape = cape + max(0.0,parea)
 
       if(debug_level.ge.200)then
-        write(6,102) p2,b1,b2,cape,cin,cloud
-102     format(5(f13.4),2x,l1)
+        call mpas_log_write('$r $r $r $r $r $l', realArgs=(/p2,b1,b2,cape,cin/), logicArgs=(/cloud/))
       endif
 
       if( (p(k).le.10000.0).and.(b2.lt.0.0) )then