Merge branch 'jgfouca/homme/bring_in_as_subtree_attempt_2' into maste…

…r (PR #362)

This branch brings in the full HOMME dycore into ACME. The full HOMME
model is included under models/homme, including the ability to run
standalone HOMME. HOMME Is brought in as a subtree, with history going
back to HOMME1_3_1. The dycore should be roundoff with the 1.5 year
old version of HOMME currently in ACME (which is about 1.5 years out
of date). But because of these roundoff changes, all comparison tests
should fail.

[Non-BFB]

* jgfouca/homme/bring_in_as_subtree_attempt_2: (22 commits)
  Updating edgeVpack/edgeVunpack in HOMME
  Fixing XLF compile issues on Mira
  Remove a C comment from Fortran source file
  Build standalone HOMME on Mira
  Protect nested OMP on Mira
  IBM compiler updates
  Compiler flags for OpenMP on Mira
  Replace _BGL with __bg__
  Updated to cuda7.0 on Titan
  Updating paths for CIME directories
  Improved PCI-e efficiency
  Getting rid of excess PCI-e copies
  Improved scripts for thatchroof profiling
  Adding scripts for thatchroof machine
  edgeVunpack using receive buffer now
  OACC now compatable with new HOMME
  Update to compile paths
  Compilation files for HOMME titan openacc
  PGI 15.10 bug fixes,undoing edge_mod workarounds
  OpenACC now machine precision same w/ CPU!
  ...

cime/externals/CMake/FindNETCDF.cmake

@@ -1,39 +1,35 @@
-# - Try to find Netcdf
-# Once done this will define
-#  NETCDF_FOUND - System has Netcdf
-#  NETCDF_INCLUDE_DIRS - The Netcdf include directories
-# NETCDF_C_LIBRARIES - The C libraries needed to use Netcdf
-# NETCDF_Fortran_LIBRARIES - The Fortran libraries needed to use Netcdf
-#  NETCDF_LIBRARIES - All the libraries needed to use Netcdf
-#  NETCDF_DEFINITIONS - Compiler switches required for using Netcdf
-
-# If we weren't given a hint via a CMake variable, check the environment.
-if(NOT NETCDF_DIR)
-  set(NETCDF_DIR $ENV{NETCDF})
-endif()
 
-find_path(NETCDF_INCLUDE_DIR netcdf.h
-          HINTS ${NETCDF_DIR}/include )
 
-find_path(NETCDF_LIB_DIR NAMES libnetcdf.a libnetcdf.so
-          HINTS ${NETCDF_DIR}/lib ${NETCDF_DIR}/lib64 )
+find_path(Netcdf_INCLUDE_DIR
+          NAMES netcdf.h
+          PATHS ${NETCDF_DIR}
+          PATH_SUFFIXES include
+          NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+
+find_library(Netcdf_LIBRARY
+             NAMES libnetcdf.a netcdf
+             HINTS ${Netcdf_INCLUDE_DIR}/../lib
+             NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+
+find_library(NetcdfF_LIBRARY
+             NAMES libnetcdff.a netcdff
+             HINTS ${Netcdf_INCLUDE_DIR}/../lib
+             NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
 
-find_path(NETCDF_FORTRAN_LIB_DIR NAMES libnetcdff.a libnetcdff.so
-          HINTS ${NETCDF_DIR}/lib ${NETCDF_DIR}/lib64 )
+find_library(Pnetcdf_LIBRARY
+             NAMES libpnetcdf.a pnetcdf
+             HINTS ${PNETCDF_DIR}/lib
+             NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
 
+find_path(Netcdf_NC_CONFIG_BIN
+          NAMES nc-config
+          HINTS ${Netcdf_INCLUDE_DIR}/../bin
+          NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
 
 find_file(NETCDF4_PAR_H netcdf_par.h
-          HINTS ${NETCDF_INCLUDE_DIR}
+          HINTS ${Netcdf_INCLUDE_DIR}
           NO_DEFAULT_PATH )
 
-#MESSAGE("PAR_H: ${NETCDF4_PAR_H}")
-find_library(NETCDF_C_LIBRARY NAMES libnetcdf.a netcdf HINTS ${NETCDF_LIB_DIR})
-
-if(NOT NETCDF_FORTRAN_LIB_DIR)
-  MESSAGE(WARNING "Did not find netCDF Fortran library.")
-else()
-  find_library(NETCDF_Fortran_LIBRARY NAMES libnetcdff.a netcdff HINTS ${NETCDF_FORTRAN_LIB_DIR})
-endif()
 if(NOT NETCDF4_PAR_H)
   set(NETCDF4_PARALLEL "no")
   MESSAGE("NETCDF built without MPIIO")
@@ -42,45 +38,191 @@ else()
   MESSAGE("NETCDF built with hdf5 MPIIO support")
 endif()
 
-set(NETCDF_INCLUDE_DIRS ${NETCDF_INCLUDE_DIR} )
+# Store libraries in Netcdf_LIBRARIES
+set(Netcdf_LIBRARIES ${NetcdfF_LIBRARY} ${Netcdf_LIBRARY})
 
-FIND_PACKAGE(HDF5 COMPONENTS C HL CONFIG)
-
-if(${HDF5_FOUND})
-  MESSAGE(STATUS "Adding hdf5 libraries ")
- set(NETCDF_C_LIBRARY ${NETCDF_C_LIBRARY} ${HDF5_LIBRARIES}
-                      ${SZIP_LIBRARIES} ${ZLIB_LIBRARIES})
+if(NOT Pnetcdf_LIBRARY)
+  set(HAS_PNETCDF "no")
+  MESSAGE("NETCDF built without Pnetcdf")
+else()
+  set(HAS_PNETCDF "yes")
+  MESSAGE("NETCDF built with Pnetcdf")
+  set(Netcdf_LIBRARIES ${NetcdfF_LIBRARY} ${Netcdf_LIBRARY} ${Pnetcdf_LIBRARY})
+  set(Netcdf_INCLUDE_DIR ${Netcdf_INCLUDE_DIR} ${PNETCDF_DIR}/include)
 endif()
 
-# If netCDF was configured with DAP, it depends on libcurl.
-find_program(NETCDF_NC_CONFIG nc-config HINTS ${NETCDF_INCLUDE_DIR}/../bin)
-if(NETCDF_NC_CONFIG)
-  execute_process(COMMAND ${NETCDF_NC_CONFIG} --has-dap
-    OUTPUT_VARIABLE nc_config_output)
-  if(nc_config_output MATCHES yes)
-    find_package(CURL)
-    if(CURL_FOUND)
-      MESSAGE(STATUS "Adding curl libraries for netCDF DAP.")
-      set(NETCDF_C_LIBRARY ${NETCDF_C_LIBRARY} ${CURL_LIBRARIES})
+IF (NOT ${Netcdf_NC_CONFIG_BIN} STREQUAL Netcdf_NC_CONFIG_BIN-NOTFOUND)
+
+  # Probe nc-config to determine dependencies of Netcdf
+  MESSAGE(STATUS "nc-config found at ${Netcdf_NC_CONFIG_BIN}")
+
+  # use nc-confg --has-nc4 to determine if Netcdf depends upon HDF5
+  EXECUTE_PROCESS(COMMAND ${Netcdf_NC_CONFIG_BIN}/nc-config --has-nc4
+    RESULT_VARIABLE NCCONFIG_RESULT
+    OUTPUT_VARIABLE NCCONFIG_OUTPUT
+    ERROR_VARIABLE NCCONFIG_ERROR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  IF (${NCCONFIG_ERROR})
+    MESSAGE(FATAL_ERROR "${Netcdf_NC_CONFIG_BIN}/nc-config --has-nc4 produced an error")
+  ELSE ()
+    IF (${NCCONFIG_OUTPUT} STREQUAL yes)
+      SET (NETCDF_REQUIRE_HDF5 TRUE)
+      MESSAGE(STATUS "nc-config: Netcdf depends upon HDF5")
+    ELSE ()
+      SET (NETCDF_REQUIRE_HDF5 FALSE)
+      MESSAGE(STATUS "nc-config: Netcdf does not depend upon HDF5")
+    ENDIF ()
+  ENDIF ()
+
+  # use nc-confg --has-dap to determine if Netcdf depends upon CURL
+  EXECUTE_PROCESS(COMMAND ${Netcdf_NC_CONFIG_BIN}/nc-config --has-dap
+    RESULT_VARIABLE NCCONFIG_RESULT
+    OUTPUT_VARIABLE NCCONFIG_OUTPUT
+    ERROR_VARIABLE NCCONFIG_ERROR
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+  )
+  IF (${NCCONFIG_ERROR})
+    MESSAGE(FATAL_ERROR "${Netcdf_NC_CONFIG_BIN}/nc-config --has-dap produced an error")
+  ELSE ()
+    IF (${NCCONFIG_OUTPUT} STREQUAL yes)
+      SET (NETCDF_REQUIRE_CURL TRUE)
+      MESSAGE(STATUS "nc-config: Netcdf depends upon CURL")
+    ELSE ()
+      SET (NETCDF_REQUIRE_CURL FALSE)
+      MESSAGE(STATUS "nc-config: Netcdf does not depend upon CURL")
+    ENDIF ()
+  ENDIF ()
+
+ELSE ()
+  SET (NETCDF_REQUIRE_HDF5 TRUE)
+  SET (NETCDF_REQUIRE_CURL TRUE)
+  MESSAGE(STATUS "nc-config not found assuming hdf5 and curl dependencies")
+ENDIF ()
+
+IF (${NETCDF_REQUIRE_CURL})
+
+  # For some reasone CURL uses CURL_ROOT rather than CURL_DIR
+  #   - change the variable for consistency
+  SET(CURL_ROOT ${CURL_DIR})
+  find_package(CURL)
+
+  IF (${CURL_FOUND})
+    MESSAGE(STATUS "Found CURL: ${CURL_LIBRARY}")
+    set(Netcdf_LIBRARIES ${Netcdf_LIBRARIES} ${CURL_LIBRARY})
+  ELSE ()
+    MESSAGE(FATAL_ERROR "CURL Not found")
+  ENDIF ()
+ENDIF ()
+
+IF (${NETCDF_REQUIRE_HDF5})
+
+  IF (HDF5_DIR)
+    find_path(HDF5_INCLUDE_DIR
+              hdf5.h
+              PATHS ${HDF5_DIR}
+              PATH_SUFFIXES include
+              NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+
+    find_library(HDF5_LIBRARY
+                 NAMES libhdf5.a hdf5
+                 PATHS ${HDF5_DIR}
+                 PATH_SUFFIXES lib lib64
+                 NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+
+    find_library(HDF5hl_LIBRARY
+                 NAMES libhdf5_hl.a hdf5_hl
+                 PATHS ${HDF5_DIR}
+                 PATH_SUFFIXES lib lib64
+                 NO_SYSTEM_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH)
+
+
+    if(${HDF5_LIBRARY} STREQUAL "HDF5_LIBRARY-NOTFOUND" OR ${HDF5hl_LIBRARY} STREQUAL "HDF5hl_LIBRARY-NOTFOUND")
+      set(HDF5_FOUND OFF)
+      MESSAGE(FATAL_ERROR "HDF5 not found, set HDF5_DIR to appropriate installation")
     else()
-      MESSAGE(WARNING "netCDF DAP appears enabled, but libcurl was not found.")
+      set(HDF5_FOUND ON)
+      MESSAGE(STATUS "Found HDF5: ${HDF5hl_LIBRARY} ${HDF5_LIBRARY}")
+      set(Netcdf_LIBRARIES ${Netcdf_LIBRARIES} ${HDF5hl_LIBRARY} ${HDF5_LIBRARY})
     endif()
-  endif()
-endif()
 
-set(NETCDF_LIBRARIES ${NETCDF_Fortran_LIBRARY} ${NETCDF_C_LIBRARY})
+  else()
+    FIND_PACKAGE(HDF5 COMPONENTS C HL)
 
-# Export variables so other projects can use them as well
-#  ie. if pio is added with add_subdirectory
-SET(NETCDF_INCLUDE_DIR ${NETCDF_INCLUDE_DIR} CACHE STRING "Location of NetCDF include files.")
-SET(NETCDF_LIBRARIES ${NETCDF_LIBRARIES} CACHE STRING "Link line for NetCDF.")
+    if(${HDF5_FOUND})
+      MESSAGE(STATUS "Adding hdf5 libraries ")
+    set(NETCDF_C_LIBRARY ${NETCDF_C_LIBRARY} ${HDF5_LIBRARIES})
+    endif()
+  endif()
 
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set NETCDF_FOUND to TRUE
-# if all listed variables are TRUE
-# (Note that the Fortran interface is not always a separate library, so
-# don't require it to be found.)
-find_package_handle_standard_args(NETCDF  DEFAULT_MSG NETCDF_LIBRARIES
-                                  NETCDF_C_LIBRARY NETCDF_INCLUDE_DIR)
 
-mark_as_advanced(NETCDF_INCLUDE_DIR NETCDF_LIBRARIES NETCDF_C_LIBRARY NETCDF_Fortran_LIBRARY NETCDF4_PARALLEL )
+  # Check to see which dependencies (ZLIB, SZIP) hdf5 has
+  INCLUDE(CheckSymbolExists)
+  CHECK_SYMBOL_EXISTS(H5_HAVE_SZLIB_H "${HDF5_INCLUDE_DIR}/H5pubconf.h" HDF5_REQUIRE_SZ)
+  CHECK_SYMBOL_EXISTS(H5_HAVE_ZLIB_H "${HDF5_INCLUDE_DIR}/H5pubconf.h" HDF5_REQUIRE_ZLIB)
+
+  IF (${HDF5_REQUIRE_ZLIB})
+
+    MESSAGE(STATUS "This HDF5 library requires ZLIB")
+    # Find package always finds the shared object
+    #find_package(ZLIB REQUIRED)
+    find_library(ZLIB_LIBRARY
+                 NAMES libz.a z
+                 PATHS ${ZLIB_DIR}
+                 PATH_SUFFIXES lib lib64
+                 NO_SYSTEM_ENVIRONMENT_PATH)
+
+    IF(${ZLIB_LIBRARY} STREQUAL "ZLIB_LIBRARY-NOTFOUND")
+      SET(ZLIB_FOUND OFF)
+      MESSAGE(FATAL_ERROR "ZLIB Not found")
+    ELSE()
+      SET(ZLIB_FOUND ON)
+    ENDIF ()
+    MESSAGE(STATUS "Found ZLIB_LIBRARY: ${ZLIB_LIBRARY}")
+    MESSAGE(STATUS "ZLIB_LIBRARIES: ${ZLIB_LIBRARIES}")
+    IF (${ZLIB_FOUND})
+      MESSAGE(STATUS "Found ZLIB: ${ZLIB_LIBRARY}")
+      set(Netcdf_LIBRARIES ${Netcdf_LIBRARIES} ${ZLIB_LIBRARY})
+    ELSE ()
+      MESSAGE(FATAL_ERROR "ZLIB Not found, set ZLIB_DIR to appropriate installation")
+    ENDIF ()
+  ENDIF ()
+
+  IF (${HDF5_REQUIRE_SZ})
+    MESSAGE(STATUS "This HDF5 library requires SZIP")
+
+    find_library(SZIP_LIBRARY
+                 NAMES libsz.a sz
+                 PATHS ${SZIP_DIR}
+                 PATH_SUFFIXES lib lib64
+                 NO_SYSTEM_ENVIRONMENT_PATH)
+
+    IF(${SZIP_LIBRARY} STREQUAL "SZIP_LIBRARY-NOTFOUND")
+      SET(SZIP_FOUND OFF)
+      MESSAGE(FATAL_ERROR "SZIP Not found, set SZIP_DIR to appropriate installation")
+    ELSE()
+      SET(SZIP_FOUND ON)
+      MESSAGE(STATUS "Found SZIP: ${SZIP_LIBRARY}")
+      SET(Netcdf_LIBRARIES ${Netcdf_LIBRARIES} ${SZIP_LIBRARY})
+    ENDIF()
+  ENDIF ()
+
+ENDIF()
+
+# If we haven't had a FATAL_ERROR then we have found Netcdf and all dependencies
+set(Netcdf_FOUND ON)
+
+# Set all caps vars too
+set(NETCDF_FOUND ON)
+set(NETCDF_INCLUDE_DIR ${Netcdf_INCLUDE_DIR})
+set(NETCDF_LIBRARIES ${Netcdf_LIBRARIES})
+
+MESSAGE(STATUS "Found Netcdf:")
+MESSAGE(STATUS "  Libraries: ${Netcdf_LIBRARIES}")
+MESSAGE(STATUS "  Includes:  ${Netcdf_INCLUDE_DIR}")
+
+IF(Netcdf_FIND_REQUIRED AND NOT Netcdf_FOUND)
+  MESSAGE(FATAL_ERROR "Did not find required library Netcdf")
+ELSEIF(Netcdf_FOUND)
+  MESSAGE(STATUS "Found Netcdf and all dependencies")
+ENDIF()

cime/externals/pio/pio/CMakeLists.txt

@@ -1,5 +1,5 @@
 IF( NOT GENF90_PATH) 
-  SET (GENF90_PATH ${CMAKE_CURRENT_SOURCE_DIR}/bin)
+  SET (GENF90_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../genf90)
 ENDIF()
 
 PROJECT(PIO C Fortran)

components/cam/bld/configure

@@ -2737,7 +2737,7 @@ sub write_filepath
 
     print $fh "$camsrcdir/cam/src/dynamics/$dyn\n";
     if($dyn eq 'se') {
-	print $fh "$camsrcdir/cam/src/dynamics/se/share\n";
+	print $fh "$camsrcdir/homme/src/share\n";
     }
 
     # Parallelization utilies

components/cam/src/dynamics/se/dyn_comp.F90

@@ -158,7 +158,7 @@ subroutine dyn_init1(fh, NLFileName, dyn_in, dyn_out)
        write(iulog,*) "dyn_init1: number of OpenMP threads = ", nthreads
        write(iulog,*) " "
     endif
-#if defined (ELEMENT_OPENMP)
+#ifdef COLUMN_OPENMP
     if(par%masterproc) then
        write(iulog,*) " "
        write(iulog,*) "dyn_init1: using OpenMP within element instead of across elements"
@@ -284,7 +284,7 @@ subroutine dyn_init2(dyn_in)
     end do
     if(iam < par%nprocs) then
 
-#if (! defined ELEMENT_OPENMP)
+#ifdef HORIZ_OPENMP
        !$OMP PARALLEL DEFAULT(SHARED), PRIVATE(ie,ithr,nets,nete,hybrid)
 #endif
        ithr=omp_get_thread_num()
@@ -347,7 +347,7 @@ subroutine dyn_init2(dyn_in)
        ! This subroutine is used to create nc_topo files, if requested
        ! 
        call nctopo_util_driver(elem,hybrid,nets,nete)
-#if (! defined ELEMENT_OPENMP)
+#ifdef HORIZ_OPENMP
        !$OMP END PARALLEL 
 #endif
     end if
@@ -387,7 +387,7 @@ subroutine dyn_run( dyn_state, rc )
     ! !DESCRIPTION:
     !
     if(iam < par%nprocs) then
-#if (! defined ELEMENT_OPENMP)
+#ifdef HORIZ_OPENMP
        !$OMP PARALLEL DEFAULT(SHARED), PRIVATE(ithr,nets,nete,hybrid,n)
 #endif
        ithr=omp_get_thread_num()
@@ -402,7 +402,7 @@ subroutine dyn_run( dyn_state, rc )
        end do
 
 
-#if (! defined ELEMENT_OPENMP)
+#ifdef HORIZ_OPENMP
        !$OMP END PARALLEL
 #endif
     end if

components/cam/src/dynamics/se/gravity_waves_sources.F90

@@ -1,7 +1,7 @@
 module gravity_waves_sources
   use derivative_mod, only : derivative_t
   use dimensions_mod, only : np,nlev
-  use edge_mod, only       : EdgeBuffer_t
+  use edgetype_mod, only       : EdgeBuffer_t
   use element_mod, only    : element_t
   use hybrid_mod, only     : hybrid_t
   use kinds, only          : real_kind
@@ -29,14 +29,17 @@ module gravity_waves_sources
 CONTAINS
 !----------------------------------------------------------------------
 
-  subroutine gws_init
+  subroutine gws_init(elem)
     use edge_mod, only       : initEdgeBuffer
     use hycoef, only         : hypi
     use pmgrid, only         : plev
+    use parallel_mod, only    : par
     implicit none
+    type (element_t), intent(inout), dimension(:) :: elem
+
 
     ! Set up variables similar to dyn_comp and prim_driver_mod initializations
-    call initEdgeBuffer(edge3,3*nlev)
+    call initEdgeBuffer(par,edge3,elem,3*nlev)
     allocate(deriv(0:Nthreads-1))
 
     psurf_ref = hypi(plev+1)
@@ -144,13 +147,13 @@ subroutine compute_frontogenesis(frontgf,frontga,tl,elem,ederiv,hybrid,nets,nete
 
        enddo
        ! pack
-       call edgeVpack(edge3, frontgf(:,:,:,ie),nlev,0,elem(ie)%desc)
-       call edgeVpack(edge3, gradth(:,:,:,:,ie),2*nlev,nlev,elem(ie)%desc)
+       call edgeVpack(edge3, frontgf(:,:,:,ie),nlev,0,ie)
+       call edgeVpack(edge3, gradth(:,:,:,:,ie),2*nlev,nlev,ie)
     enddo
     call bndry_exchangeV(hybrid,edge3)
     do ie=nets,nete
-       call edgeVunpack(edge3, frontgf(:,:,:,ie),nlev,0,elem(ie)%desc)
-       call edgeVunpack(edge3, gradth(:,:,:,:,ie),2*nlev,nlev,elem(ie)%desc)
+       call edgeVunpack(edge3, frontgf(:,:,:,ie),nlev,0,ie)
+       call edgeVunpack(edge3, gradth(:,:,:,:,ie),2*nlev,nlev,ie)
        ! apply inverse mass matrix,
        do k=1,nlev
           gradth(:,:,1,k,ie)=gradth(:,:,1,k,ie)*elem(ie)%rspheremp(:,:)