Skip to content

Commit

Permalink
Merge branch 'jgfouca/csm_share_no_makefile' into master (PR #6025)
Browse files Browse the repository at this point in the history
CSM_share: no cime/Tools/Makefile

The CSM_share build is very similar to how we build our main
components, except it was being driven by the
CIME/Tools/Makefile. This PR changes it to use our CMake build_model
function. A few minor tweaks were needed, the main problem was with
the Depends files. The CSM_share build cannot use our standard Depends
system due to the likelihood that the Depends file will modify flags
for files that CSM doesn't know about, which is currently an
error. Instead, the CSM depends files are moved to E3SM/share, which
is a bit unfortunate since it is not consistent and also makes them
non-customizable within a case. On the plus side, this allows us to
get rid of our last remaining Makefile-style Depends files.

[BFB]
  • Loading branch information
jgfouca committed Oct 27, 2023
2 parents 0b367fb + 6e70c30 commit b2ea1c3
Show file tree
Hide file tree
Showing 9 changed files with 204 additions and 79 deletions.
4 changes: 0 additions & 4 deletions cime_config/machines/Depends.ibm

This file was deleted.

27 changes: 0 additions & 27 deletions cime_config/machines/Depends.intel

This file was deleted.

71 changes: 43 additions & 28 deletions components/cmake/build_model.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ function(build_model COMP_CLASS COMP_NAME)

# We support component-specific configuration of flags, etc, so this setup
# need to be done here.
include(${CMAKE_SOURCE_DIR}/cmake/common_setup.cmake)
include(${SRCROOT}/components/cmake/common_setup.cmake)

set(MODELCONF_DIR "${BUILDCONF}/${COMP_NAME}conf")

Expand Down Expand Up @@ -34,8 +34,11 @@ function(build_model COMP_CLASS COMP_NAME)

# Source files don't live in components/cmake/$COMP_CLASS. This path won't work for
# generated files.
set(SOURCE_PATH "../..")
set(CIMESRC_PATH "../cime/src")
if (COMP_NAME STREQUAL "csm_share")
set(SOURCE_PATH ".")
else()
set(SOURCE_PATH "../..")
endif()

#-------------------------------------------------------------------------------
# Build & include dependency files
Expand Down Expand Up @@ -127,7 +130,7 @@ function(build_model COMP_CLASS COMP_NAME)
# Add samxx F90 files to the main E3SM build
set(SOURCES ${SOURCES} cmake/atm/../../eam/src/physics/crm/pam/params.F90
cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90
cmake/atm/../../eam/src/physics/crm/pam/pam_driver.F90)
cmake/atm/../../eam/src/physics/crm/pam/pam_driver.F90)
# Pam interface need to include modules from pam
include_directories(${PAM_BIN}/external/pam_core)
endif()
Expand Down Expand Up @@ -204,9 +207,16 @@ function(build_model COMP_CLASS COMP_NAME)
endforeach()

# Load machine/compiler specific settings
set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${COMPILER}.cmake)
set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.cmake)
set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.${COMPILER}.cmake)
if (COMP_NAME STREQUAL "csm_share")
# csm_share uses special Depends files, not customizable per case
set(DEPENDS_LOC "${SRCROOT}/share")
else()
set(DEPENDS_LOC "${CASEROOT}")
endif()

set(COMPILER_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${COMPILER}.cmake)
set(MACHINE_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${MACH}.cmake)
set(PLATFORM_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${MACH}.${COMPILER}.cmake)
set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS})
foreach(ITEM IN LISTS TRY_TO_LOAD)
if (EXISTS ${ITEM})
Expand Down Expand Up @@ -256,30 +266,35 @@ function(build_model COMP_CLASS COMP_NAME)
set(TARGET_NAME ${COMP_CLASS})
add_library(${TARGET_NAME})
target_sources(${TARGET_NAME} PRIVATE ${REAL_SOURCES})
target_link_libraries(${TARGET_NAME} PRIVATE csm_share)
if (COMP_NAME STREQUAL "eam")
if (USE_YAKL)
target_link_libraries(${TARGET_NAME} PRIVATE yakl)
endif()
if (USE_SAMXX)
target_link_libraries(${TARGET_NAME} PRIVATE samxx)
endif()
if (USE_PAM)
target_link_libraries(${TARGET_NAME} PRIVATE pam_driver)
endif()
if (USE_RRTMGPXX)
target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
if (COMP_NAME STREQUAL "csm_share")
find_package(NETCDF REQUIRED)
target_link_libraries(${TARGET_NAME} PRIVATE netcdf)
else()
target_link_libraries(${TARGET_NAME} PRIVATE csm_share)
if (COMP_NAME STREQUAL "eam")
if (USE_YAKL)
target_link_libraries(${TARGET_NAME} PRIVATE yakl)
endif()
if (USE_SAMXX)
target_link_libraries(${TARGET_NAME} PRIVATE samxx)
endif()
if (USE_PAM)
target_link_libraries(${TARGET_NAME} PRIVATE pam_driver)
endif()
if (USE_RRTMGPXX)
target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface)
endif()
endif()
endif()
if (COMP_NAME STREQUAL "elm")
if (USE_PETSC)
target_link_libraries(${TARGET_NAME} PRIVATE "${PETSC_LIBRARIES}")
target_include_directories(${TARGET_NAME} PRIVATE "${PETSC_INCLUDES}")
if (COMP_NAME STREQUAL "elm")
if (USE_PETSC)
target_link_libraries(${TARGET_NAME} PRIVATE "${PETSC_LIBRARIES}")
target_include_directories(${TARGET_NAME} PRIVATE "${PETSC_INCLUDES}")
endif()
endif()
if (USE_KOKKOS)
target_link_libraries (${TARGET_NAME} PRIVATE Kokkos::kokkos)
endif ()
endif()
if (USE_KOKKOS)
target_link_libraries (${TARGET_NAME} PRIVATE Kokkos::kokkos)
endif ()
endif()

# Subtle: In order for fortran dependency scanning to work, our CPPFPP/DEFS must be registered
Expand Down
3 changes: 0 additions & 3 deletions components/cmake/build_mpas_model.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,6 @@ function(build_mpas_models)
set(ALBANY True)
endif()

# set CIME source path relative to components
set(CIMESRC_PATH "../cime/src")

if (CORES)
add_subdirectory("mpas-framework/src")
endif()
Expand Down
2 changes: 1 addition & 1 deletion components/cmake/cmake_util.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ function(gather_sources FILEPATH_DIRS_ARG CIMEROOT_ARG)
set(GEN_F90_SOURCES_RESULT)

foreach(DIRSEARCH ${FILEPATH_DIRS_ARG})
file(GLOB MATCHES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/../.." "${DIRSEARCH}/*.[Ffc]" "${DIRSEARCH}/*.[Ff]90" "${DIRSEARCH}/*.cpp" "${DIRSEARCH}/*.F90.in")
file(GLOB MATCHES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_PATH}" "${DIRSEARCH}/*.[Ffc]" "${DIRSEARCH}/*.[Ff]90" "${DIRSEARCH}/*.cpp" "${DIRSEARCH}/*.F90.in")
if (MATCHES)
foreach (MATCH IN LISTS MATCHES)
get_filename_component(BASENAME ${MATCH} NAME)
Expand Down
107 changes: 107 additions & 0 deletions share/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
#===============================================================================
# The CMake system for csm_share. This is mostly copied from the CMakeLists.txt
# for the main E3SM.
#===============================================================================

cmake_minimum_required(VERSION 3.18)
cmake_policy(SET CMP0057 NEW)
cmake_policy(SET CMP0074 NEW)
cmake_policy(SET CMP0079 NEW) # Remove once scorpio in a better state
set(CMAKE_CXX_STANDARD 17)

# We need to set the compilers *before* calling `project`.
# The only way to get the compiler name, is to load Macros.cmake
# However, we do *not* want to pollute the environment with other
# vars coming from Macros.cmake, so we encapsulate its inclusion
# in a new scope.
# Additionally, we also set CMAKE_BUILD_TYPE=DEBUG if Macros.cmake
# contains DEBUG set to true
function(set_compilers_e3sm)
# Grab CXX compiler from CIME
include(${CASEROOT}/Macros.cmake)

if (MPILIB STREQUAL "mpi-serial")
set(CC ${SCC})
set(FC ${SFC})
set(CXX ${SCXX})
else()
set(CC ${MPICC})
set(FC ${MPIFC})
set(CXX ${MPICXX})
endif()

set(CMAKE_CXX_COMPILER ${CXX} CACHE STRING "The CXX compiler")
set(CMAKE_C_COMPILER ${CC} CACHE STRING "The C compiler")
set(CMAKE_Fortran_COMPILER ${FC} CACHE STRING "The Fortran compiler")

# USE_CUDA or USE_HIP is set through Macros.cmake
# For instance: cime_config/machines/cmake_macros/gnugpu_summit.cmake
# If it exists, then set parent's scope to true; otherwise to false
# At this point, we use either CUDA or HIP.
# Revisit as needed for future systems.
if (USE_CUDA)
set(USE_CUDA TRUE PARENT_SCOPE)
elseif (USE_HIP)
set(USE_HIP TRUE PARENT_SCOPE)
else()
set(USE_CUDA FALSE PARENT_SCOPE)
set(USE_HIP FALSE PARENT_SCOPE)
endif()
endfunction()

set_compilers_e3sm()

project(CSM_SHARE C CXX Fortran)

if(USE_CUDA)
enable_language(CUDA)
elseif(USE_HIP)
enable_language(HIP)
endif()

# Any changes to SourceMods will require us to reconfigure
file(GLOB COMPONENT_SOURCE_MOD_DIRS "${CASEROOT}/SourceMods/src.*")
foreach(COMPONENT_SOURCE_MOD_DIR IN LISTS COMPONENT_SOURCE_MOD_DIRS)
set_property(
DIRECTORY
APPEND
PROPERTY CMAKE_CONFIGURE_DEPENDS
${COMPONENT_SOURCE_MOD_DIR})
endforeach()

# Include function definitions
include(${SRCROOT}/components/cmake/cmake_util.cmake)
include(${SRCROOT}/components/cmake/build_mpas_model.cmake)
include(${SRCROOT}/components/cmake/build_eamxx.cmake)
include(${SRCROOT}/components/cmake/build_model.cmake)

# Set up CMAKE_MODULE_PATH so any component can use E3SM
# and CIME cmake modules if they want.
list(APPEND CMAKE_MODULE_PATH ${SRCROOT}/components/cmake/modules)
list(APPEND CMAKE_MODULE_PATH ${CIMEROOT}/CIME/non_py/src/CMake)

set(CMAKE_VERBOSE_MAKEFILE TRUE)

# We do want CMAKE_BUILD_TYPE to be set, but we do NOT want CMake to
# decide what optimization flags to append, based on build type,
# for components who rely on CIME for build flags, so make all the following empty.
# JGF: I think we can remove this once proper CMake names are being used in the
# macros.
set (CMAKE_C_FLAGS_RELEASE "")
set (CMAKE_CXX_FLAGS_RELEASE "")
set (CMAKE_Fortran_FLAGS_RELEASE "")

set (CMAKE_C_FLAGS_DEBUG "")
set (CMAKE_CXX_FLAGS_DEBUG "")
set (CMAKE_Fortran_FLAGS_DEBUG "")

set(BUILDCONF ${CASEROOT}/Buildconf)

# Set global targets
if (NOT TARGET genf90)
add_custom_target(genf90
DEPENDS ${CIMEROOT}/CIME/non_py/externals/genf90/genf90.pl)
endif()

# Build CSM_share
build_model("csm_share" "csm_share")
1 change: 1 addition & 0 deletions share/Depends.ibm.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e3sm_add_flags("util/shr_reprosum_mod.F90" "-qnosmp")
27 changes: 27 additions & 0 deletions share/Depends.intel.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
if (NOT DEBUG)
# Note: FFLAGS contains flags such as -fp-model consistent (and -fimf-use-svml for intel version 18)
# The -fp-model fast flags below will effectively override other -fp-model settings.

# shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits
# enormously from a lower precision in the vector functions.
e3sm_add_flags("util/shr_wv_sat_mod.F90" "-fimf-precision=low -fp-model fast")

set(SHR_RANDNUM_FORT_SRCS
"RandNum/src/kissvec/kissvec_mod.F90"
"RandNum/src/mt19937/mersennetwister_mod.F90"
"RandNum/src/dsfmt_f03/dSFMT_interface.F90"
"RandNum/src/shr_RandNum_mod.F90")

foreach(SHR_RANDNUM_FORT_SRC IN LISTS SHR_RANDNUM_FORT_SRCS)
e3sm_add_flags("${SHR_RANDNUM_FORT_SRC}" "-fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits")
endforeach()

set(SHR_RANDNUM_C_SRCS
"RandNum/src/dsfmt_f03/dSFMT.c"
"RandNum/src/dsfmt_f03/dSFMT_utils.c"
"RandNum/src/kissvec/kissvec.c")

foreach(SHR_RANDNUM_C_SRC IN LISTS SHR_RANDNUM_C_SRCS)
e3sm_add_flags("${SHR_RANDNUM_C_SRC}" "-fp-model fast")
endforeach()
endif()
41 changes: 25 additions & 16 deletions share/build/buildlib.csm_share
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from standard_script_setup import *
from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect, symlink_force
from CIME.case import Case
from CIME.build import get_standard_makefile_args
from CIME.build import get_standard_cmake_args
from CIME.XML.files import Files
import glob

Expand Down Expand Up @@ -46,17 +46,17 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter
###############################################################################
def buildlib(bldroot, installpath, case):
###############################################################################
gmake_args = get_standard_makefile_args(case, shared_lib=True)
comp_interface = case.get_value("COMP_INTERFACE")
srcroot = case.get_value("SRCROOT")
caseroot = case.get_value("CASEROOT")
libroot = case.get_value("LIBROOT")

csm_share_dir = os.environ.get("csm_share_ROOT")
if csm_share_dir is not None:
# We are using a pre-installed csm_share
expect(os.path.exists(csm_share_dir), f"Non-existent csm_share root {csm_share_dir}")
return
else:
csm_share_dir = os.path.join(srcroot, "share")

filepath = [os.path.join(caseroot,"SourceMods","src.share"),
os.path.join(srcroot,"share","streams"),
Expand Down Expand Up @@ -84,7 +84,11 @@ def buildlib(bldroot, installpath, case):
if not os.path.isdir(libdir):
os.makedirs(libdir)

filepathfile = os.path.join(libdir, "Filepath")
config_dir = os.path.join(caseroot, "Buildconf", "csm_shareconf")
if not os.path.isdir(config_dir):
os.makedirs(config_dir)

filepathfile = os.path.join(config_dir, "Filepath")
# if the filepathfile has a different number of lines than filepath, replace it
file_len = 0
if os.path.isfile(filepathfile):
Expand All @@ -110,6 +114,11 @@ def buildlib(bldroot, installpath, case):
if case.get_value("COMP_OCN") == "nemo":
multiinst_cppdefs += " -DNEMO_IN_CCSM "

cppdefs_file = os.path.join(config_dir, "CIME_cppdefs")
if not os.path.isfile(cppdefs_file):
with open(cppdefs_file, "w") as fd:
fd.write(multiinst_cppdefs + " -DTIMING\n")

installdir = os.path.join(installpath, comp_interface,
use_esmf, ninst_value)
for ndir in ("lib", "include"):
Expand All @@ -121,27 +130,27 @@ def buildlib(bldroot, installpath, case):
for _file in glob.iglob(os.path.join(srcroot,"share","RandNum","include","*")):
copyifnewer(_file, os.path.join(installdir, "include", os.path.basename(_file)))

# This runs the make command
gmake_opts = "-f {}/Makefile complib COMP_NAME=csm_share ".format(os.path.join(caseroot,"Tools"))
gmake_opts += "-j {} ".format(case.get_value("GMAKE_J"))
gmake_opts += " COMPLIB=libcsm_share.a"
gmake_opts += ' USER_CPPDEFS="{} -DTIMING" '.format(multiinst_cppdefs)
gmake_opts += "INCLUDE_DIR={} ".format(os.path.join(installdir, "include"))
gmake_opts += gmake_args
gmake_opts += " -C {}".format(libdir)
# This runs the cmake command
cmake_args = get_standard_cmake_args(case, installpath)
run_bld_cmd_ensure_logging(f"cmake {cmake_args} {csm_share_dir}", logger, from_dir=libdir)

gmake_cmd = case.get_value("GMAKE")
gmake_j = case.get_value("GMAKE_J")

cmd = "{} {}".format(gmake_cmd, gmake_opts)
run_bld_cmd_ensure_logging(cmd, logger)
cmd = "{} -j {}".format(gmake_cmd, gmake_j)
run_bld_cmd_ensure_logging(cmd, logger, from_dir=libdir)
#
# The pgi compiler sometimes has issues with long include paths
# on the command line, this is a workaround for that problem
#
case_inc_dir = os.path.join(libroot, "include")
case_inc_dir = os.path.join(libdir, "..", "include")
if not os.path.isdir(case_inc_dir):
os.mkdir(case_inc_dir)
for _file in glob.iglob(os.path.join(installdir,"include","*")):

for _file in glob.iglob(os.path.join(libdir,"*.mod")):
symlink_force(_file, os.path.join(case_inc_dir,os.path.basename(_file)))

for _file in glob.iglob(os.path.join(libdir,"*.h")):
symlink_force(_file, os.path.join(case_inc_dir,os.path.basename(_file)))

###############################################################################
Expand Down

0 comments on commit b2ea1c3

Please sign in to comment.