Skip to content

Commit

Permalink
Merge pull request #1396 from billsacks/compiler_from_macros
Browse files Browse the repository at this point in the history
Get unit test build and run working with serial or parallel pFUnit
  • Loading branch information
jedwards4b authored Apr 21, 2017
2 parents af32abd + a5cc0b0 commit 962b20b
Show file tree
Hide file tree
Showing 15 changed files with 183 additions and 65 deletions.
32 changes: 28 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@ cmake_minimum_required(VERSION 2.8)
include(ExternalProject)
set(CIME_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")

list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
include(CIME_initial_setup)

project(cime_tests Fortran C)

# We rely on pio for cmake utilities like findnetcdf.cmake, so that we don't
# need to duplicate this cmake code
list(APPEND CMAKE_MODULE_PATH "${CIME_ROOT}/src/externals/pio2/cmake")

list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
include(CIME_utils)
find_package(NetCDF COMPONENTS C Fortran)
include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS})
Expand All @@ -21,14 +23,21 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS})
# ------------------------------------------------------------------------
set(MCT_ROOT "${CIME_ROOT}/src/externals/mct")

if (USE_MPI_SERIAL)
set(ENABLE_MPI_SERIAL "--enable-mpiserial")
else()
set(ENABLE_MPI_SERIAL "")
endif()

ExternalProject_add(mct_project
PREFIX ${CMAKE_CURRENT_BINARY_DIR}
SOURCE_DIR ${MCT_ROOT}
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/mct
CONFIGURE_COMMAND ${MCT_ROOT}/configure --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g"
BUILD_COMMAND $(MAKE)
CONFIGURE_COMMAND ${MCT_ROOT}/configure ${ENABLE_MPI_SERIAL} --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g"
BUILD_COMMAND $(MAKE) SRCDIR=${MCT_ROOT}
# Leave things in <BINARY_DIR> rather than "installing", because we have
# no need to move things around inside of the CMake binary directory.
# no need to move things around inside of the CMake binary directory. Also,
# mpi-serial doesn't install properly in the out-of-source build
INSTALL_COMMAND :
)
# This copy_makefiles step is needed because mct currently doesn't support an
Expand All @@ -43,12 +52,27 @@ ExternalProject_add_step(mct_project copy_makefiles
COMMAND mkdir -p mpeu
COMMAND cp -p <SOURCE_DIR>/mpeu/Makefile mpeu/
)
if (USE_MPI_SERIAL)
ExternalProject_add_step(mct_project copy_mpi_serial_files
DEPENDEES configure
DEPENDERS build
WORKING_DIRECTORY <BINARY_DIR>
COMMAND mkdir -p mpi-serial
COMMAND cp -p <SOURCE_DIR>/mpi-serial/Makefile mpi-serial/
COMMAND cp <SOURCE_DIR>/mpi-serial/mpif.h mpi-serial/
COMMAND cp <SOURCE_DIR>/mpi-serial/mpi.h mpi-serial/
)
endif()

# Tell cmake to look for libraries & mod files here, because this is where we built libraries
include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu)
link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct)
link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu)
if (USE_MPI_SERIAL)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial)
link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial)
endif()

# ------------------------------------------------------------------------
# Done MCT build
Expand Down
5 changes: 2 additions & 3 deletions README.unit_testing
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# To run all CIME unit tests on caldera, run the following command:
# (Note that this must be done from an interactive caldera session, not from yellowstone)
# Note also that this requires module load all-python-libs
# To run all the CIME Fortran unit tests, run the following command:
# On yellowstone, this requires module load all-python-libs
#
# The creation of a temporary directory ensures that you are doing a completely
# clean build of the unit tests. (The use of the --clean flag to run_tests.py
Expand Down
3 changes: 0 additions & 3 deletions config/acme/machines/config_compilers.xml
Original file line number Diff line number Diff line change
Expand Up @@ -811,7 +811,6 @@ for mct, etc.
-L$(LAPACK_LIBDIR) -Wl,-rpath=$(LAPACK_LIBDIR) \
-L$(BLAS_LIBDIR) -Wl,-rpath=$(BLAS_LIBDIR)
</ADD_LDFLAGS>
<PFUNIT_PATH></PFUNIT_PATH>
</compiler>

<compiler COMPILER="intel" MACH="titan">
Expand Down Expand Up @@ -920,7 +919,6 @@ for mct, etc.
<LD> mpixlf77_r </LD>
<NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
<PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.0/cnk-xl/current/</PNETCDF_PATH>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
<HDF5_PATH>/soft/libraries/hdf5/1.8.14/cnk-xl/current/</HDF5_PATH>
<ADD_SLIBS>-L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib </ADD_SLIBS>
<ADD_SLIBS compile_threaded="true"> -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp </ADD_SLIBS>
Expand All @@ -943,7 +941,6 @@ for mct, etc.
<LD> mpixlf77_r </LD>
<NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
<PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.0/cnk-xl/current/</PNETCDF_PATH>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
<HDF5_PATH>/soft/libraries/hdf5/1.8.14/cnk-xl/current/</HDF5_PATH>
<ADD_SLIBS>-L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf -L$(HDF5_PATH)/lib -lhdf5_hl -lhdf5 -L/soft/libraries/alcf/current/xl/ZLIB/lib -lz -L/soft/libraries/alcf/current/xl/LAPACK/lib -llapack -L/soft/libraries/alcf/current/xl/BLAS/lib -lblas -L/bgsys/drivers/ppcfloor/comm/sys/lib </ADD_SLIBS>
<ADD_SLIBS compile_threaded="true"> -L$(IBM_MAIN_DIR)/xlf/bg/14.1/bglib64 -lxlfmath -lxlf90_r -lxlopt -lxl -L$(IBM_MAIN_DIR)/xlsmp/bg/3.1/bglib64 -lxlsmp </ADD_SLIBS>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
<NETCDF_PATH>/usr/local</NETCDF_PATH>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
<PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
</compiler>

<compiler COMPILER="gnu" MACH="example-osx-macports">
Expand Down Expand Up @@ -57,6 +57,6 @@
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
<NETCDF_PATH>/opt/local</NETCDF_PATH>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
<PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
</compiler>
</config_compilers>
11 changes: 5 additions & 6 deletions config/cesm/machines/config_compilers.xml
Original file line number Diff line number Diff line change
Expand Up @@ -687,10 +687,10 @@ using a fortran linker.
<append> -Wl,-rpath,$ENV{MPI_PATH}/lib</append>
<append> -lifcore</append>
</LDFLAGS>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_Intel_3_0</PFUNIT_PATH>
<SLIBS>
<append MPILIB="mvapich2"> -mkl=cluster </append>
</SLIBS>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">/fs/cgd/csm/tools/pFUnit/pFUnit3.2.8_hobart_Intel15.0.2_noMPI_noOpenMP</PFUNIT_PATH>
</compiler>

<compiler MACH="hobart" COMPILER="nag">
Expand All @@ -701,7 +701,6 @@ using a fortran linker.
<LDFLAGS>
<append> -lpthread</append>
</LDFLAGS>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_NAG_3_0</PFUNIT_PATH>
</compiler>

<compiler MACH="hobart" COMPILER="pgi">
Expand Down Expand Up @@ -767,7 +766,6 @@ using a fortran linker.
<MPICC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r </MPICC>
<MPIFC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlf2003_r </MPIFC>
<NETCDF_PATH>/soft/libraries/netcdf/4.3.3-f4.4.1/cnk-xl/current/</NETCDF_PATH>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
<PNETCDF_PATH>/soft/libraries/pnetcdf/1.6.1/cnk-xl/current/</PNETCDF_PATH>
<SCC> /home/pkcoff/mpich-sandboxes/master/install-production/bin/mpixlc_r </SCC>
Expand Down Expand Up @@ -930,7 +928,6 @@ using a fortran linker.
<MPICC> mpixlc_r </MPICC>
<MPIFC> mpixlf2003_r </MPIFC>
<NETCDF_PATH>/soft/libraries/netcdf/4.3.0-f4.2/cnk-xl/V1R2M0-20131211/</NETCDF_PATH>
<PFUNIT_PATH>/home/santos/pFUnit/pFUnit_IBM</PFUNIT_PATH>
<PIO_FILESYSTEM_HINTS>gpfs </PIO_FILESYSTEM_HINTS>
<PNETCDF_PATH>/soft/libraries/pnetcdf/1.3.1/cnk-xl/current/</PNETCDF_PATH>
<SCC> mpixlc_r </SCC>
Expand Down Expand Up @@ -992,7 +989,8 @@ using a fortran linker.
<CMAKE_OPTS>
<append DEBUG="TRUE"> -DPIO_ENABLE_LOGGING=ON </append>
</CMAKE_OPTS>
<PFUNIT_PATH>$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_MPI_openMP</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_noMPI_noOpenMP</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpt" compile_threaded="true">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.2.8_cheyenne_Intel17.0.1_MPI_openMP</PFUNIT_PATH>
<!-- Bug in the intel/17.0.1 compiler requires this, remove this line when compiler is updated -->
<HAS_F2008_CONTIGUOUS>FALSE</HAS_F2008_CONTIGUOUS>
</compiler>
Expand Down Expand Up @@ -1037,7 +1035,8 @@ using a fortran linker.
<MPICXX MPILIB="mpich2">mpiicpc</MPICXX>
<PAPI_INC MPILIB="mpich2"> /glade/apps/opt/papi/5.3.0/intel/12.1.5/include/</PAPI_INC>
<PAPI_LIB MPILIB="mpich2">/glade/apps/opt/papi/5.3.0/intel/12.1.5/lib64 </PAPI_LIB>
<PFUNIT_PATH>$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_yellowstone_Intel15.0.1_noMPI_noOpenMP</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpich2" compile_threaded="true">$ENV{CESMDATAROOT}/tools/pFUnit/pFUnit3.1_Intel15.0.1_MPI</PFUNIT_PATH>
<!-- Needed due to the way that netcdf is loaded on yellowstone -->
<SCC MPILIB="mpi-serial">icc</SCC>
<SFC MPILIB="mpi-serial">ifort</SFC>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
<NETCDF_PATH>/usr/local</NETCDF_PATH>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
<PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
</compiler>

<compiler COMPILER="gnu" MACH="example-osx-macports">
Expand Down Expand Up @@ -57,6 +57,6 @@
<SUPPORTS_CXX>TRUE</SUPPORTS_CXX>
<NETCDF_PATH>/opt/local</NETCDF_PATH>
<ADD_SLIBS>$(shell $(NETCDF_PATH)/bin/nf-config --flibs) -framework Accelerate</ADD_SLIBS>
<PFUNIT_PATH>$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
<PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$ENV{HOME}/local/pfunit/pfunit-sf.git.ae92605e8e</PFUNIT_PATH>
</compiler>
</config_compilers>
22 changes: 21 additions & 1 deletion scripts/lib/CIME/BuildTools/cmakemacroswriter.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ def set_variable(self, name, value):
>>> s.getvalue()
u'set(foo "bar")\\n'
"""
self.write_line("set(" + name + ' "' + value + '")')
value_transformed = self._transform_value(value)
self.write_line("set(" + name + ' "' + value_transformed + '")')

def start_ifeq(self, left, right):
"""Write out a statement to start a conditional block.
Expand Down Expand Up @@ -117,3 +118,22 @@ def end_ifeq(self):
"""
self.indent_left()
self.write_line("endif()")

def _transform_value(self, value):
"""Some elements need their values transformed in some way for CMake to handle them properly.
This method does those transformations.
Args:
- value (str): value of element
Returns transformed value
"""

# Not all variables need leading & trailing whitespace removed, but some
# do. In particular, compiler variables (MPICC, MPICXX, MPIFC, SCC,
# SCXX, SFC) are only handled correctly if leading & trailing whitespace
# is removed. It doesn't seem to hurt to remove whitespace from all
# variables.
value_transformed = value.strip()

return value_transformed
4 changes: 3 additions & 1 deletion scripts/tests/scripts_regression_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ def setUpClass(cls):
def _has_unit_test_support(self):
default_compiler = MACHINE.get_default_compiler()
compiler = Compilers(MACHINE, compiler=default_compiler)
pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH")
attrs = {'MPILIB': 'mpi-serial', 'compile_threaded': 'false'}
pfunit_path = compiler.get_optional_compiler_node("PFUNIT_PATH",
attributes=attrs)
if pfunit_path is None:
return False
else:
Expand Down
8 changes: 8 additions & 0 deletions src/drivers/mct/unit_test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ add_definitions(
-DNUM_COMP_INST_ESP=1
)

# The following definitions are needed when building with the mpi-serial library
if (USE_MPI_SERIAL)
add_definitions(-DNO_MPI2 -DNO_MPIMOD)
endif()

# Add source directories from stubs. This should be done first, so that in the
# case of name collisions, the drv versions take precedence (when there are two
# files with the same name, the one added later wins).
Expand Down Expand Up @@ -46,6 +51,9 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
# tests need all of these libraries, but it's easiest just to set the same list
# for everyone.
set(DRV_UNIT_TEST_LIBS drv;csm_share;esmf_wrf_timemgr;mct;mpeu)
if (USE_MPI_SERIAL)
list(APPEND DRV_UNIT_TEST_LIBS mpi-serial)
endif()
list(APPEND DRV_UNIT_TEST_LIBS ${NETCDF_LIBRARIES})

# Add the test directories
Expand Down
16 changes: 16 additions & 0 deletions src/externals/CMake/CIME_initial_setup.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Module used for CIME testing.
#
# This module does some initial setup that must be done BEFORE the 'project'
# line in the main CMakeLists.txt file.

include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND)
if(NOT FOUND)
message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure")
endif()
if("$ENV{MPILIB}" STREQUAL "mpi-serial")
set(CMAKE_C_COMPILER ${SCC})
set(CMAKE_Fortran_COMPILER ${SFC})
else()
set(CMAKE_C_COMPILER ${MPICC})
set(CMAKE_Fortran_COMPILER ${MPIFC})
endif()
12 changes: 5 additions & 7 deletions src/externals/CMake/CIME_utils.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@
# This module contains statements that would otherwise be boilerplate in
# most CIME tests. It enables CTest testing, handles the USE_COLOR and
# ENABLE_GENF90 arguments, and includes several other modules.
#
# Some of the things done here must be done AFTER the 'project' line in the main
# CMakeLists.txt file. This assumes that CIME_initial_setup has already been
# included.

#==========================================================================
# Copyright (c) 2013-2014, University Corporation for Atmospheric Research
Expand Down Expand Up @@ -30,16 +34,10 @@ set(CMAKE_COLOR_MAKEFILE "${USE_COLOR}")
# Compiler info
#=================================================

include(${CMAKE_BINARY_DIR}/Macros.cmake RESULT_VARIABLE FOUND)
list(APPEND CMAKE_MODULE_PATH "../pio2/cmake")
set(CMAKE_C_COMPILER ${MPICC})
set(CMAKE_Fortran_COMPILER ${MPIF90})
set(CMAKE_C_FLAGS "${CPPDEFS} ${CFLAGS}")
set(CMAKE_Fortran_FLAGS "${CPPDEFS} ${FFLAGS}")
set (CMAKE_EXE_LINKER_FLAGS ${LDFLAGS})
if(NOT FOUND)
message(FATAL_ERROR "You must generate a Macros.cmake file using CIME's configure")
endif()
set(CMAKE_EXE_LINKER_FLAGS "${LDFLAGS} ${SLIBS}")

include(Compilers)

Expand Down
8 changes: 7 additions & 1 deletion src/externals/CMake/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ Sourcelist_utils - Use source file lists defined over multiple directories.

Modules that are CESM-specific and/or incomplete:

CIME_utils - Handles a few options, and includes several other modules.
CIME\_initial\_setup - Handles setup that must be done before the 'project'
line. This must be included before the 'project' line in the main CMakeLists.txt
file.

CIME_utils - Handles a few options, and includes several other modules. This
must be included after the 'project' line in the main CMakeLists.txt file, and
after the inclusion of CIME\_initial\_setup.

Compilers - Specify compiler-specific behavior, add build types for CESM.
4 changes: 3 additions & 1 deletion tools/unit_testing/Examples/circle_area/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
cmake_minimum_required(VERSION 2.8)
project(circle_area_tests Fortran C)

list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
include(CIME_initial_setup)

project(circle_area_tests Fortran C)

include(CIME_utils)

Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
cmake_minimum_required(VERSION 2.8)
project(interpolate_1d_tests Fortran C)

list(APPEND CMAKE_MODULE_PATH ${CIME_CMAKE_MODULE_DIRECTORY})
include(CIME_initial_setup)

project(interpolate_1d_tests Fortran C)

include(CIME_utils)

add_subdirectory(../src interpolate_1d)
Expand Down
Loading

0 comments on commit 962b20b

Please sign in to comment.