From dbfcc2baff07c235e7e1df9491bb69b3f943f586 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 20 Oct 2023 15:00:11 -0600 Subject: [PATCH 1/6] csm_share: Build via cmake --- components/cmake/build_model.cmake | 56 +++++++------ components/cmake/build_mpas_model.cmake | 3 - components/cmake/cmake_util.cmake | 14 ++-- share/CMakeLists.txt | 107 ++++++++++++++++++++++++ share/build/buildlib.csm_share | 33 +++++--- 5 files changed, 166 insertions(+), 47 deletions(-) create mode 100644 share/CMakeLists.txt diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 6afc1e5ab0b8..0e2c07024f7d 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -2,7 +2,7 @@ function(build_model COMP_CLASS COMP_NAME) # We support component-specific configuration of flags, etc, so this setup # need to be done here. - include(${CMAKE_SOURCE_DIR}/cmake/common_setup.cmake) + include(${SRCROOT}/components/cmake/common_setup.cmake) set(MODELCONF_DIR "${BUILDCONF}/${COMP_NAME}conf") @@ -34,8 +34,11 @@ function(build_model COMP_CLASS COMP_NAME) # Source files don't live in components/cmake/$COMP_CLASS. This path won't work for # generated files. - set(SOURCE_PATH "../..") - set(CIMESRC_PATH "../cime/src") + if (COMP_NAME STREQUAL "csm_share") + set(SOURCE_PATH ".") + else() + set(SOURCE_PATH "../..") + endif() #------------------------------------------------------------------------------- # Build & include dependency files @@ -256,30 +259,35 @@ function(build_model COMP_CLASS COMP_NAME) set(TARGET_NAME ${COMP_CLASS}) add_library(${TARGET_NAME}) target_sources(${TARGET_NAME} PRIVATE ${REAL_SOURCES}) - target_link_libraries(${TARGET_NAME} PRIVATE csm_share) - if (COMP_NAME STREQUAL "eam") - if (USE_YAKL) - target_link_libraries(${TARGET_NAME} PRIVATE yakl) - endif() - if (USE_SAMXX) - target_link_libraries(${TARGET_NAME} PRIVATE samxx) - endif() - if (USE_PAM) - target_link_libraries(${TARGET_NAME} PRIVATE pam_driver) - endif() - if (USE_RRTMGPXX) - target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface) + if (COMP_NAME STREQUAL "csm_share") + find_package(NETCDF REQUIRED) + target_link_libraries(${TARGET_NAME} PRIVATE netcdf) + else() + target_link_libraries(${TARGET_NAME} PRIVATE csm_share) + if (COMP_NAME STREQUAL "eam") + if (USE_YAKL) + target_link_libraries(${TARGET_NAME} PRIVATE yakl) + endif() + if (USE_SAMXX) + target_link_libraries(${TARGET_NAME} PRIVATE samxx) + endif() + if (USE_PAM) + target_link_libraries(${TARGET_NAME} PRIVATE pam_driver) + endif() + if (USE_RRTMGPXX) + target_link_libraries(${TARGET_NAME} PRIVATE rrtmgp rrtmgp_interface) + endif() endif() - endif() - if (COMP_NAME STREQUAL "elm") - if (USE_PETSC) - target_link_libraries(${TARGET_NAME} PRIVATE "${PETSC_LIBRARIES}") - target_include_directories(${TARGET_NAME} PRIVATE "${PETSC_INCLUDES}") + if (COMP_NAME STREQUAL "elm") + if (USE_PETSC) + target_link_libraries(${TARGET_NAME} PRIVATE "${PETSC_LIBRARIES}") + target_include_directories(${TARGET_NAME} PRIVATE "${PETSC_INCLUDES}") + endif() endif() + if (USE_KOKKOS) + target_link_libraries (${TARGET_NAME} PRIVATE Kokkos::kokkos) + endif () endif() - if (USE_KOKKOS) - target_link_libraries (${TARGET_NAME} PRIVATE Kokkos::kokkos) - endif () endif() # Subtle: In order for fortran dependency scanning to work, our CPPFPP/DEFS must be registered diff --git a/components/cmake/build_mpas_model.cmake b/components/cmake/build_mpas_model.cmake index 9201a07d7b74..f59007c0fcc6 100644 --- a/components/cmake/build_mpas_model.cmake +++ b/components/cmake/build_mpas_model.cmake @@ -29,9 +29,6 @@ function(build_mpas_models) set(ALBANY True) endif() - # set CIME source path relative to components - set(CIMESRC_PATH "../cime/src") - if (CORES) add_subdirectory("mpas-framework/src") endif() diff --git a/components/cmake/cmake_util.cmake b/components/cmake/cmake_util.cmake index 4dfffe7f2e6c..88fb93305460 100644 --- a/components/cmake/cmake_util.cmake +++ b/components/cmake/cmake_util.cmake @@ -12,7 +12,7 @@ function(gather_sources FILEPATH_DIRS_ARG CIMEROOT_ARG) set(GEN_F90_SOURCES_RESULT) foreach(DIRSEARCH ${FILEPATH_DIRS_ARG}) - file(GLOB MATCHES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/../.." "${DIRSEARCH}/*.[Ffc]" "${DIRSEARCH}/*.[Ff]90" "${DIRSEARCH}/*.cpp" "${DIRSEARCH}/*.F90.in") + file(GLOB MATCHES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE_PATH}" "${DIRSEARCH}/*.[Ffc]" "${DIRSEARCH}/*.[Ff]90" "${DIRSEARCH}/*.cpp" "${DIRSEARCH}/*.F90.in") if (MATCHES) foreach (MATCH IN LISTS MATCHES) get_filename_component(BASENAME ${MATCH} NAME) @@ -49,9 +49,9 @@ function(e3sm_add_flags FILE_ARG FLAGS_ARG) if (FILE_ARG MATCHES "${CMAKE_BINARY_DIR}/.*") # is generated set(REAL_FILE ${FILE_ARG}) else() - if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) - message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}, looked for ${PROJECT_SOURCE_DIR}/${FILE_ARG}") - endif() + # if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) + # message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}, looked for ${PROJECT_SOURCE_DIR}/${FILE_ARG}") + # endif() set(REAL_FILE "${SOURCE_PATH}/${FILE_ARG}") endif() set_property(SOURCE ${REAL_FILE} APPEND_STRING PROPERTY COMPILE_FLAGS " ${FLAGS_ARG} ") @@ -63,9 +63,9 @@ function(e3sm_remove_flags FILE_ARG FLAGS_ARG) if (FILE_ARG MATCHES "${CMAKE_BINARY_DIR}/.*") # is generated set(REAL_FILE ${FILE_ARG}) else() - if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) - message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}") - endif() + # if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) + # message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}") + # endif() set(REAL_FILE "${SOURCE_PATH}/${FILE_ARG}") endif() diff --git a/share/CMakeLists.txt b/share/CMakeLists.txt new file mode 100644 index 000000000000..e62b58afa3b5 --- /dev/null +++ b/share/CMakeLists.txt @@ -0,0 +1,107 @@ +#=============================================================================== +# The CMake system for csm_share. This is mostly copied from the CMakeLists.txt +# for the main E3SM. +#=============================================================================== + +cmake_minimum_required(VERSION 3.18) +cmake_policy(SET CMP0057 NEW) +cmake_policy(SET CMP0074 NEW) +cmake_policy(SET CMP0079 NEW) # Remove once scorpio in a better state +set(CMAKE_CXX_STANDARD 17) + +# We need to set the compilers *before* calling `project`. +# The only way to get the compiler name, is to load Macros.cmake +# However, we do *not* want to pollute the environment with other +# vars coming from Macros.cmake, so we encapsulate its inclusion +# in a new scope. +# Additionally, we also set CMAKE_BUILD_TYPE=DEBUG if Macros.cmake +# contains DEBUG set to true +function(set_compilers_e3sm) + # Grab CXX compiler from CIME + include(${CASEROOT}/Macros.cmake) + + if (MPILIB STREQUAL "mpi-serial") + set(CC ${SCC}) + set(FC ${SFC}) + set(CXX ${SCXX}) + else() + set(CC ${MPICC}) + set(FC ${MPIFC}) + set(CXX ${MPICXX}) + endif() + + set(CMAKE_CXX_COMPILER ${CXX} CACHE STRING "The CXX compiler") + set(CMAKE_C_COMPILER ${CC} CACHE STRING "The C compiler") + set(CMAKE_Fortran_COMPILER ${FC} CACHE STRING "The Fortran compiler") + + # USE_CUDA or USE_HIP is set through Macros.cmake + # For instance: cime_config/machines/cmake_macros/gnugpu_summit.cmake + # If it exists, then set parent's scope to true; otherwise to false + # At this point, we use either CUDA or HIP. + # Revisit as needed for future systems. + if (USE_CUDA) + set(USE_CUDA TRUE PARENT_SCOPE) + elseif (USE_HIP) + set(USE_HIP TRUE PARENT_SCOPE) + else() + set(USE_CUDA FALSE PARENT_SCOPE) + set(USE_HIP FALSE PARENT_SCOPE) + endif() +endfunction() + +set_compilers_e3sm() + +project(CSM_SHARE C CXX Fortran) + +if(USE_CUDA) + enable_language(CUDA) +elseif(USE_HIP) + enable_language(HIP) +endif() + +# Any changes to SourceMods will require us to reconfigure +file(GLOB COMPONENT_SOURCE_MOD_DIRS "${CASEROOT}/SourceMods/src.*") +foreach(COMPONENT_SOURCE_MOD_DIR IN LISTS COMPONENT_SOURCE_MOD_DIRS) + set_property( + DIRECTORY + APPEND + PROPERTY CMAKE_CONFIGURE_DEPENDS + ${COMPONENT_SOURCE_MOD_DIR}) +endforeach() + +# Include function definitions +include(${SRCROOT}/components/cmake/cmake_util.cmake) +include(${SRCROOT}/components/cmake/build_mpas_model.cmake) +include(${SRCROOT}/components/cmake/build_eamxx.cmake) +include(${SRCROOT}/components/cmake/build_model.cmake) + +# Set up CMAKE_MODULE_PATH so any component can use E3SM +# and CIME cmake modules if they want. +list(APPEND CMAKE_MODULE_PATH ${SRCROOT}/components/cmake/modules) +list(APPEND CMAKE_MODULE_PATH ${CIMEROOT}/CIME/non_py/src/CMake) + +set(CMAKE_VERBOSE_MAKEFILE TRUE) + +# We do want CMAKE_BUILD_TYPE to be set, but we do NOT want CMake to +# decide what optimization flags to append, based on build type, +# for components who rely on CIME for build flags, so make all the following empty. +# JGF: I think we can remove this once proper CMake names are being used in the +# macros. +set (CMAKE_C_FLAGS_RELEASE "") +set (CMAKE_CXX_FLAGS_RELEASE "") +set (CMAKE_Fortran_FLAGS_RELEASE "") + +set (CMAKE_C_FLAGS_DEBUG "") +set (CMAKE_CXX_FLAGS_DEBUG "") +set (CMAKE_Fortran_FLAGS_DEBUG "") + +set(BUILDCONF ${CASEROOT}/Buildconf) + +# Set global targets +if (NOT TARGET genf90) + add_custom_target(genf90 + DEPENDS ${CIMEROOT}/CIME/non_py/externals/genf90/genf90.pl) +endif() + +# Build CSM_share +build_model("csm_share" "csm_share") diff --git a/share/build/buildlib.csm_share b/share/build/buildlib.csm_share index 0c4373213c87..17e32a4a1d6e 100755 --- a/share/build/buildlib.csm_share +++ b/share/build/buildlib.csm_share @@ -2,7 +2,7 @@ from standard_script_setup import * from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect, symlink_force from CIME.case import Case -from CIME.build import get_standard_makefile_args +from CIME.build import get_standard_cmake_args from CIME.XML.files import Files import glob @@ -46,7 +46,6 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ############################################################################### def buildlib(bldroot, installpath, case): ############################################################################### - gmake_args = get_standard_makefile_args(case, shared_lib=True) comp_interface = case.get_value("COMP_INTERFACE") srcroot = case.get_value("SRCROOT") caseroot = case.get_value("CASEROOT") @@ -57,6 +56,8 @@ def buildlib(bldroot, installpath, case): # We are using a pre-installed csm_share expect(os.path.exists(csm_share_dir), f"Non-existent csm_share root {csm_share_dir}") return + else: + csm_share_dir = os.path.join(srcroot, "share") filepath = [os.path.join(caseroot,"SourceMods","src.share"), os.path.join(srcroot,"share","streams"), @@ -84,7 +85,11 @@ def buildlib(bldroot, installpath, case): if not os.path.isdir(libdir): os.makedirs(libdir) - filepathfile = os.path.join(libdir, "Filepath") + config_dir = os.path.join(caseroot, "Buildconf", "csm_shareconf") + if not os.path.isdir(config_dir): + os.makedirs(config_dir) + + filepathfile = os.path.join(config_dir, "Filepath") # if the filepathfile has a different number of lines than filepath, replace it file_len = 0 if os.path.isfile(filepathfile): @@ -110,6 +115,11 @@ def buildlib(bldroot, installpath, case): if case.get_value("COMP_OCN") == "nemo": multiinst_cppdefs += " -DNEMO_IN_CCSM " + cppdefs_file = os.path.join(config_dir, "CIME_cppdefs") + if not os.path.isfile(cppdefs_file): + with open(cppdefs_file, "w") as fd: + fd.write(multiinst_cppdefs + " -DTIMING\n") + installdir = os.path.join(installpath, comp_interface, use_esmf, ninst_value) for ndir in ("lib", "include"): @@ -121,19 +131,16 @@ def buildlib(bldroot, installpath, case): for _file in glob.iglob(os.path.join(srcroot,"share","RandNum","include","*")): copyifnewer(_file, os.path.join(installdir, "include", os.path.basename(_file))) - # This runs the make command - gmake_opts = "-f {}/Makefile complib COMP_NAME=csm_share ".format(os.path.join(caseroot,"Tools")) - gmake_opts += "-j {} ".format(case.get_value("GMAKE_J")) - gmake_opts += " COMPLIB=libcsm_share.a" - gmake_opts += ' USER_CPPDEFS="{} -DTIMING" '.format(multiinst_cppdefs) - gmake_opts += "INCLUDE_DIR={} ".format(os.path.join(installdir, "include")) - gmake_opts += gmake_args - gmake_opts += " -C {}".format(libdir) + # This runs the cmake command + logger.info(f"JGF libdir={libdir} installpath={installpath}") + cmake_args = get_standard_cmake_args(case, installpath) + run_bld_cmd_ensure_logging(f"cmake {cmake_args} {csm_share_dir}", logger, from_dir=libdir) gmake_cmd = case.get_value("GMAKE") + gmake_j = case.get_value("GMAKE_J") - cmd = "{} {}".format(gmake_cmd, gmake_opts) - run_bld_cmd_ensure_logging(cmd, logger) + cmd = "{} -j {}".format(gmake_cmd, gmake_j) + run_bld_cmd_ensure_logging(cmd, logger, from_dir=libdir) # # The pgi compiler sometimes has issues with long include paths # on the command line, this is a workaround for that problem From 7f1b1f94f4c68d34c054106a03ceee659783656d Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 20 Oct 2023 15:26:29 -0600 Subject: [PATCH 2/6] Fixes. LIBROOT is not really used anymore --- share/build/buildlib.csm_share | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/share/build/buildlib.csm_share b/share/build/buildlib.csm_share index 17e32a4a1d6e..80d74defb196 100755 --- a/share/build/buildlib.csm_share +++ b/share/build/buildlib.csm_share @@ -49,7 +49,6 @@ def buildlib(bldroot, installpath, case): comp_interface = case.get_value("COMP_INTERFACE") srcroot = case.get_value("SRCROOT") caseroot = case.get_value("CASEROOT") - libroot = case.get_value("LIBROOT") csm_share_dir = os.environ.get("csm_share_ROOT") if csm_share_dir is not None: @@ -132,7 +131,6 @@ def buildlib(bldroot, installpath, case): copyifnewer(_file, os.path.join(installdir, "include", os.path.basename(_file))) # This runs the cmake command - logger.info(f"JGF libdir={libdir} installpath={installpath}") cmake_args = get_standard_cmake_args(case, installpath) run_bld_cmd_ensure_logging(f"cmake {cmake_args} {csm_share_dir}", logger, from_dir=libdir) @@ -145,10 +143,14 @@ def buildlib(bldroot, installpath, case): # The pgi compiler sometimes has issues with long include paths # on the command line, this is a workaround for that problem # - case_inc_dir = os.path.join(libroot, "include") + case_inc_dir = os.path.join(libdir, "..", "include") if not os.path.isdir(case_inc_dir): os.mkdir(case_inc_dir) - for _file in glob.iglob(os.path.join(installdir,"include","*")): + + for _file in glob.iglob(os.path.join(libdir,"*.mod")): + symlink_force(_file, os.path.join(case_inc_dir,os.path.basename(_file))) + + for _file in glob.iglob(os.path.join(libdir,"*.h")): symlink_force(_file, os.path.join(case_inc_dir,os.path.basename(_file))) ############################################################################### From 9b027e26f3fee8d9426b65eae0599f584029122e Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 20 Oct 2023 15:56:20 -0600 Subject: [PATCH 3/6] Get rid of last Makefile Depends --- cime_config/machines/Depends.ibm | 4 --- cime_config/machines/Depends.intel | 27 --------------- components/cmake/build_model.cmake | 54 +++++++++++++++++++++++++----- components/cmake/cmake_util.cmake | 12 +++---- 4 files changed, 51 insertions(+), 46 deletions(-) delete mode 100644 cime_config/machines/Depends.ibm delete mode 100644 cime_config/machines/Depends.intel diff --git a/cime_config/machines/Depends.ibm b/cime_config/machines/Depends.ibm deleted file mode 100644 index b1c9026a046f..000000000000 --- a/cime_config/machines/Depends.ibm +++ /dev/null @@ -1,4 +0,0 @@ - -# shr_reprosum_mod.F90 is built in csm_share with gmake -shr_reprosum_mod.o: shr_reprosum_mod.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -qnosmp $< diff --git a/cime_config/machines/Depends.intel b/cime_config/machines/Depends.intel deleted file mode 100644 index 3de8f68238ad..000000000000 --- a/cime_config/machines/Depends.intel +++ /dev/null @@ -1,27 +0,0 @@ -# shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits -# enormously from a lower precision in the vector functions. -REDUCED_PRECISION_OBJS=\ -shr_wv_sat_mod.o - -SHR_RANDNUM_FORT_OBJS=\ -kissvec_mod.o \ -mersennetwister_mod.o \ -dSFMT_interface.o \ -shr_RandNum_mod.o - -SHR_RANDNUM_C_OBJS=\ -dSFMT.o \ -dSFMT_utils.o \ -kissvec.o - -# Note: FFLAGS contains flags such as -fp-model consistent (and -fimf-use-svml for intel version 18) -# The -fp-model fast flags below will effectively override other -fp-model settings. - -ifeq ($(DEBUG),FALSE) - $(REDUCED_PRECISION_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $< - $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits $< - $(SHR_RANDNUM_C_OBJS): %.o: %.c - $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $< -endif diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 0e2c07024f7d..5b910de749b4 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -130,7 +130,7 @@ function(build_model COMP_CLASS COMP_NAME) # Add samxx F90 files to the main E3SM build set(SOURCES ${SOURCES} cmake/atm/../../eam/src/physics/crm/pam/params.F90 cmake/atm/../../eam/src/physics/crm/crm_ecpp_output_module.F90 - cmake/atm/../../eam/src/physics/crm/pam/pam_driver.F90) + cmake/atm/../../eam/src/physics/crm/pam/pam_driver.F90) # Pam interface need to include modules from pam include_directories(${PAM_BIN}/external/pam_core) endif() @@ -207,15 +207,51 @@ function(build_model COMP_CLASS COMP_NAME) endforeach() # Load machine/compiler specific settings - set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${COMPILER}.cmake) - set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.cmake) - set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.${COMPILER}.cmake) - set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS}) - foreach(ITEM IN LISTS TRY_TO_LOAD) - if (EXISTS ${ITEM}) - include(${ITEM}) + if (COMP_NAME STREQUAL "csm_share") + # csm_share customizes things in a special way (no Depends) + if (COMPILER STREQUAL "ibm") + e3sm_add_flags("util/shr_reprosum_mod.F90" "-qnosmp") + + elseif(COMPILER STREQUAL "intel") + if (NOT DEBUG) + # Note: FFLAGS contains flags such as -fp-model consistent (and -fimf-use-svml for intel version 18) + # The -fp-model fast flags below will effectively override other -fp-model settings. + + # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits + # enormously from a lower precision in the vector functions. + e3sm_add_flags("util/shr_wv_sat_mod.F90" "-fimf-precision=low -fp-model fast") + + set(SHR_RANDNUM_FORT_SRCS + "RandNum/src/kissvec/kissvec_mod.F90" + "RandNum/src/mt19937/mersennetwister_mod.F90" + "RandNum/src/dsfmt_f03/dSFMT_interface.F90" + "RandNum/src/shr_RandNum_mod.F90") + + foreach(SHR_RANDNUM_FORT_SRC IN LISTS SHR_RANDNUM_FORT_SRCS) + e3sm_add_flags("${SHR_RANDNUM_FORT_SRC}" "-fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits") + endforeach() + + set(SHR_RANDNUM_C_SRCS + "RandNum/src/dsfmt_f03/dSFMT.c" + "RandNum/src/dsfmt_f03/dSFMT_utils.c" + "RandNum/src/kissvec/kissvec.c") + + foreach(SHR_RANDNUM_C_SRC IN LISTS SHR_RANDNUM_C_SRCS) + e3sm_add_flags("${SHR_RANDNUM_C_SRC}" "-fp-model fast") + endforeach() + endif() endif() - endforeach() + else() + set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${COMPILER}.cmake) + set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.cmake) + set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.${COMPILER}.cmake) + set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS}) + foreach(ITEM IN LISTS TRY_TO_LOAD) + if (EXISTS ${ITEM}) + include(${ITEM}) + endif() + endforeach() + endif() # Disable optimizations on some files that would take too long to compile, expect these to all be fortran files foreach (SOURCE_FILE IN LISTS NOOPT_FILES) diff --git a/components/cmake/cmake_util.cmake b/components/cmake/cmake_util.cmake index 88fb93305460..e83c63b460ab 100644 --- a/components/cmake/cmake_util.cmake +++ b/components/cmake/cmake_util.cmake @@ -49,9 +49,9 @@ function(e3sm_add_flags FILE_ARG FLAGS_ARG) if (FILE_ARG MATCHES "${CMAKE_BINARY_DIR}/.*") # is generated set(REAL_FILE ${FILE_ARG}) else() - # if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) - # message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}, looked for ${PROJECT_SOURCE_DIR}/${FILE_ARG}") - # endif() + if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) + message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}, looked for ${PROJECT_SOURCE_DIR}/${FILE_ARG}") + endif() set(REAL_FILE "${SOURCE_PATH}/${FILE_ARG}") endif() set_property(SOURCE ${REAL_FILE} APPEND_STRING PROPERTY COMPILE_FLAGS " ${FLAGS_ARG} ") @@ -63,9 +63,9 @@ function(e3sm_remove_flags FILE_ARG FLAGS_ARG) if (FILE_ARG MATCHES "${CMAKE_BINARY_DIR}/.*") # is generated set(REAL_FILE ${FILE_ARG}) else() - # if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) - # message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}") - # endif() + if (NOT EXISTS ${PROJECT_SOURCE_DIR}/${FILE_ARG}) + message(FATAL_ERROR "Trying to set flags on non-existent source: ${FILE_ARG}") + endif() set(REAL_FILE "${SOURCE_PATH}/${FILE_ARG}") endif() From c98976b7244abf12ffec202d0d502d832224a6f6 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 26 Oct 2023 12:32:08 -0600 Subject: [PATCH 4/6] Clean up csm_share depends files a bit --- cime | 2 +- .../machines/Depends.csm_share.ibm.cmake | 1 + .../machines/Depends.csm_share.intel.cmake | 27 +++++++++ components/cmake/build_model.cmake | 55 ++++--------------- 4 files changed, 41 insertions(+), 44 deletions(-) create mode 100644 cime_config/machines/Depends.csm_share.ibm.cmake create mode 100644 cime_config/machines/Depends.csm_share.intel.cmake diff --git a/cime b/cime index 7b579110a1c8..00f957cce1d3 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 7b579110a1c8c8e1843bb61dbd7ad9dad531087a +Subproject commit 00f957cce1d3c4e29cfadb9e99ffb9e76a2e175a diff --git a/cime_config/machines/Depends.csm_share.ibm.cmake b/cime_config/machines/Depends.csm_share.ibm.cmake new file mode 100644 index 000000000000..ee6c81539f56 --- /dev/null +++ b/cime_config/machines/Depends.csm_share.ibm.cmake @@ -0,0 +1 @@ +e3sm_add_flags("util/shr_reprosum_mod.F90" "-qnosmp") diff --git a/cime_config/machines/Depends.csm_share.intel.cmake b/cime_config/machines/Depends.csm_share.intel.cmake new file mode 100644 index 000000000000..2a940de2b610 --- /dev/null +++ b/cime_config/machines/Depends.csm_share.intel.cmake @@ -0,0 +1,27 @@ +if (NOT DEBUG) + # Note: FFLAGS contains flags such as -fp-model consistent (and -fimf-use-svml for intel version 18) + # The -fp-model fast flags below will effectively override other -fp-model settings. + + # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits + # enormously from a lower precision in the vector functions. + e3sm_add_flags("util/shr_wv_sat_mod.F90" "-fimf-precision=low -fp-model fast") + + set(SHR_RANDNUM_FORT_SRCS + "RandNum/src/kissvec/kissvec_mod.F90" + "RandNum/src/mt19937/mersennetwister_mod.F90" + "RandNum/src/dsfmt_f03/dSFMT_interface.F90" + "RandNum/src/shr_RandNum_mod.F90") + + foreach(SHR_RANDNUM_FORT_SRC IN LISTS SHR_RANDNUM_FORT_SRCS) + e3sm_add_flags("${SHR_RANDNUM_FORT_SRC}" "-fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits") + endforeach() + + set(SHR_RANDNUM_C_SRCS + "RandNum/src/dsfmt_f03/dSFMT.c" + "RandNum/src/dsfmt_f03/dSFMT_utils.c" + "RandNum/src/kissvec/kissvec.c") + + foreach(SHR_RANDNUM_C_SRC IN LISTS SHR_RANDNUM_C_SRCS) + e3sm_add_flags("${SHR_RANDNUM_C_SRC}" "-fp-model fast") + endforeach() +endif() diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 5b910de749b4..9dff033c88d8 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -208,51 +208,20 @@ function(build_model COMP_CLASS COMP_NAME) # Load machine/compiler specific settings if (COMP_NAME STREQUAL "csm_share") - # csm_share customizes things in a special way (no Depends) - if (COMPILER STREQUAL "ibm") - e3sm_add_flags("util/shr_reprosum_mod.F90" "-qnosmp") - - elseif(COMPILER STREQUAL "intel") - if (NOT DEBUG) - # Note: FFLAGS contains flags such as -fp-model consistent (and -fimf-use-svml for intel version 18) - # The -fp-model fast flags below will effectively override other -fp-model settings. - - # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits - # enormously from a lower precision in the vector functions. - e3sm_add_flags("util/shr_wv_sat_mod.F90" "-fimf-precision=low -fp-model fast") - - set(SHR_RANDNUM_FORT_SRCS - "RandNum/src/kissvec/kissvec_mod.F90" - "RandNum/src/mt19937/mersennetwister_mod.F90" - "RandNum/src/dsfmt_f03/dSFMT_interface.F90" - "RandNum/src/shr_RandNum_mod.F90") - - foreach(SHR_RANDNUM_FORT_SRC IN LISTS SHR_RANDNUM_FORT_SRCS) - e3sm_add_flags("${SHR_RANDNUM_FORT_SRC}" "-fp-model fast -no-prec-div -no-prec-sqrt -qoverride-limits") - endforeach() - - set(SHR_RANDNUM_C_SRCS - "RandNum/src/dsfmt_f03/dSFMT.c" - "RandNum/src/dsfmt_f03/dSFMT_utils.c" - "RandNum/src/kissvec/kissvec.c") - - foreach(SHR_RANDNUM_C_SRC IN LISTS SHR_RANDNUM_C_SRCS) - e3sm_add_flags("${SHR_RANDNUM_C_SRC}" "-fp-model fast") - endforeach() - endif() - endif() - else() - set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${COMPILER}.cmake) - set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.cmake) - set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${MACH}.${COMPILER}.cmake) - set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS}) - foreach(ITEM IN LISTS TRY_TO_LOAD) - if (EXISTS ${ITEM}) - include(${ITEM}) - endif() - endforeach() + # csm_share uses special Depends files + set(SHARE_DEPENDS_EXT "csm_share.") endif() + set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${COMPILER}.cmake) + set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${MACH}.cmake) + set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${MACH}.${COMPILER}.cmake) + set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS}) + foreach(ITEM IN LISTS TRY_TO_LOAD) + if (EXISTS ${ITEM}) + include(${ITEM}) + endif() + endforeach() + # Disable optimizations on some files that would take too long to compile, expect these to all be fortran files foreach (SOURCE_FILE IN LISTS NOOPT_FILES) e3sm_deoptimize_file("${SOURCE_FILE}" "${FFLAGS_NOOPT}") From b64b5d6016bae479af3fb0f57b5b957931474c18 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 26 Oct 2023 13:20:18 -0600 Subject: [PATCH 5/6] Reorg csm_share Depends --- components/cmake/build_model.cmake | 12 +++++++----- .../Depends.ibm.cmake | 0 .../Depends.intel.cmake | 0 3 files changed, 7 insertions(+), 5 deletions(-) rename cime_config/machines/Depends.csm_share.ibm.cmake => share/Depends.ibm.cmake (100%) rename cime_config/machines/Depends.csm_share.intel.cmake => share/Depends.intel.cmake (100%) diff --git a/components/cmake/build_model.cmake b/components/cmake/build_model.cmake index 9dff033c88d8..fce7e1242662 100644 --- a/components/cmake/build_model.cmake +++ b/components/cmake/build_model.cmake @@ -208,13 +208,15 @@ function(build_model COMP_CLASS COMP_NAME) # Load machine/compiler specific settings if (COMP_NAME STREQUAL "csm_share") - # csm_share uses special Depends files - set(SHARE_DEPENDS_EXT "csm_share.") + # csm_share uses special Depends files, not customizable per case + set(DEPENDS_LOC "${SRCROOT}/share") + else() + set(DEPENDS_LOC "${CASEROOT}") endif() - set(COMPILER_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${COMPILER}.cmake) - set(MACHINE_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${MACH}.cmake) - set(PLATFORM_SPECIFIC_DEPENDS ${CASEROOT}/Depends.${SHARE_DEPENDS_EXT}${MACH}.${COMPILER}.cmake) + set(COMPILER_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${COMPILER}.cmake) + set(MACHINE_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${MACH}.cmake) + set(PLATFORM_SPECIFIC_DEPENDS ${DEPENDS_LOC}/Depends.${MACH}.${COMPILER}.cmake) set(TRY_TO_LOAD ${COMPILER_SPECIFIC_DEPENDS} ${MACHINE_SPECIFIC_DEPENDS} ${PLATFORM_SPECIFIC_DEPENDS}) foreach(ITEM IN LISTS TRY_TO_LOAD) if (EXISTS ${ITEM}) diff --git a/cime_config/machines/Depends.csm_share.ibm.cmake b/share/Depends.ibm.cmake similarity index 100% rename from cime_config/machines/Depends.csm_share.ibm.cmake rename to share/Depends.ibm.cmake diff --git a/cime_config/machines/Depends.csm_share.intel.cmake b/share/Depends.intel.cmake similarity index 100% rename from cime_config/machines/Depends.csm_share.intel.cmake rename to share/Depends.intel.cmake From 6e70c30f56847c655fdec10b19438b6039cfe0d7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 26 Oct 2023 15:42:19 -0600 Subject: [PATCH 6/6] Don't change submodule --- cime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cime b/cime index 00f957cce1d3..7b579110a1c8 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 00f957cce1d3c4e29cfadb9e99ffb9e76a2e175a +Subproject commit 7b579110a1c8c8e1843bb61dbd7ad9dad531087a