improved cmake build to better support various compilers and libraries.

* added new cmake variables of OMEGA_INCLUDE_DIRECTORIES OMEGA_LINK_DIRECTORIES OMEGA_LINK_OPTIONS * better supports compilers that are not wrapped in a compiler wrapper(hipcc) * uses cmake variables from E3SM for GPU models including CUDA_FLAGS, HIP, SYCL and CXXFLAGS * supports older Python versions
E3SM-Project · Sep 8, 2023 · 4938590 · 4938590
1 parent c404f4b
commit 4938590
Show file tree

Hide file tree

Showing 7 changed files with 162 additions and 59 deletions.
diff --git a/components/omega/CMakeLists.txt b/components/omega/CMakeLists.txt
@@ -11,6 +11,9 @@ cmake_policy(SET CMP0011 NEW)
 # list command no longer ignores empty elements
 cmake_policy(SET CMP0007 NEW)
 
+# Only interpret if() arguments as variables or keywords when unquoted
+cmake_policy(SET CMP0054 NEW)
+
 ###########################################################
 # STEP 1: Setup                                           #
 #                                                         #

diff --git a/components/omega/OmegaBuild.cmake b/components/omega/OmegaBuild.cmake
@@ -23,11 +23,22 @@ macro(setup_common_variables)
 
   option(OMEGA_DEBUG "Turn on error message throwing (default OFF)." OFF)
 
-  if(NOT DEFINED OMEGA_CXX_FLAGS )
-    # initialize cxx flags as an empty list
+  if(NOT DEFINED OMEGA_CXX_FLAGS)
     set(OMEGA_CXX_FLAGS "")
   endif()
 
+  if(NOT DEFINED OMEGA_INCLUDE_DIRECTORIES)
+    set(OMEGA_INCLUDE_DIRECTORIES "")
+  endif()
+
+  if(NOT DEFINED OMEGA_LINK_DIRECTORIES)
+    set(OMEGA_LINK_DIRECTORIES "")
+  endif()
+
+  if(NOT DEFINED OMEGA_LINK_OPTIONS)
+    set(OMEGA_LINK_OPTIONS "")
+  endif()
+
 endmacro()
 
 ###########################
@@ -51,18 +62,24 @@ macro(preset)
 
   execute_process(COMMAND ${Python_EXECUTABLE} gen_machine_info.py
     -p ${E3SM_CIME_ROOT} -o ${_TMP_CMAKE_FILE} ${_TMP_COMPILER}
+    OUTPUT_QUIET ERROR_QUIET
     WORKING_DIRECTORY ${OMEGA_SOURCE_DIR}
     OUTPUT_VARIABLE _MACHINE_INFO)
 
   include(${_TMP_CMAKE_FILE})
   file(REMOVE ${_TMP_CMAKE_FILE})
 
-  if(DEFINED OMEGA_CXX_COMPILER)
-    set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
-  else()
-    set(CMAKE_CXX_COMPILER ${MPICXX})
+  if(NOT OMEGA_CXX_COMPILER)
+    if (MPILIB STREQUAL "mpi-serial")
+      set(OMEGA_CXX_COMPILER ${SCXX})
+    else()
+      set(OMEGA_CXX_COMPILER ${MPICXX})
+    endif()
   endif()
 
+  set(CMAKE_CXX_COMPILER ${OMEGA_CXX_COMPILER})
+  message(STATUS "OMEGA_CXX_COMPILER = ${OMEGA_CXX_COMPILER}")
+
 endmacro()
 
 macro(setup_standalone_build)
@@ -115,17 +132,73 @@ macro(update_variables)
   # Set the build type
   set(CMAKE_BUILD_TYPE ${OMEGA_BUILD_TYPE})
 
-  if(DEFINED OMEGA_CXX_FLAGS)
-    set(CMAKE_CXX_FLAGS ${OMEGA_CXX_FLAGS})
+  if (NETCDF_PATH)
+    list(APPEND OMEGA_INCLUDE_DIRECTORIES "${NETCDF_PATH}/include")
+    list(APPEND OMEGA_LINK_DIRECTORIES "${NETCDF_PATH}/lib")
+    list(APPEND OMEGA_LINK_OPTIONS "-lnetcdf")
+  endif()
+
+  if (PNETCDF_PATH)
+    list(APPEND OMEGA_INCLUDE_DIRECTORIES "${PNETCDF_PATH}/include")
+    list(APPEND OMEGA_LINK_DIRECTORIES "${PNETCDF_PATH}/lib")
+    list(APPEND OMEGA_LINK_OPTIONS "-lpnetcdf")
+  endif()
+
+  string(CONCAT _TestMPISource
+    "#include \"mpi.h\"\n"
+    "int main(int argc, char* argv[])\n"
+    "{MPI_Init(&argc, &argv)\; return 0\;}\n")
+  set(_TestMPISrcFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.cpp)
+  set(_TestMPIObjFile ${CMAKE_CURRENT_BINARY_DIR}/_testMPI.o)
+  file(WRITE ${_TestMPISrcFile}  ${_TestMPISource})
+
+  execute_process(
+	COMMAND ${OMEGA_CXX_COMPILER} -c ${_TestMPISrcFile} -o ${_TestMPIObjFile}
+    OUTPUT_QUIET ERROR_QUIET
+    RESULT_VARIABLE _MPI_TEST_RESULT)
+
+  if(OMEGA_BUILD_TYPE EQUAL Release)
+	file(REMOVE ${_TestMPISrcFile})
+	file(REMOVE ${_TestMPIObjFile})
   endif()
 
-  if(DEFINED OMEGA_LINK_FLAGS)
-    set(LINK_FLAGS ${OMEGA_LINK_FLAGS})
+  if (NOT _MPI_TEST_RESULT EQUAL 0)
+    if (_MPI_TEST_RESULT MATCHES "^[-]?[0-9]+$")
+      find_package(MPI)
+      if(MPI_CXX_FOUND)
+        list(APPEND OMEGA_INCLUDE_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}")
+        list(APPEND OMEGA_LINK_DIRECTORIES "${MPI_CXX_INCLUDE_DIRS}/../lib")
+        list(APPEND OMEGA_LINK_OPTIONS "-lmpi")
+      else()
+        message(FATAL_ERROR "MPI is not found" )
+      endif()
+    else()
+      message(FATAL_ERROR "MPI test failure: ${_MPI_TEST_RESULT}" )
+    endif()
+  endif()
+
+  if(USE_CUDA)
+    list(APPEND OMEGA_CXX_FLAGS ${CUDA_FLAGS})
+
+  elseif(USE_HIP)
+    list(APPEND OMEGA_CXX_FLAGS ${HIP_FLAGS})
+
+  elseif(USE_SYCL)
+    list(APPEND OMEGA_CXX_FLAGS ${SYCL_FLAGS})
+
+  else()
+    list(APPEND OMEGA_CXX_FLAGS ${CXXFLAGS})
+
+    if(CXX_LIBS)
+      list(APPEND OMEGA_LINK_OPTIONS  ${CXX_LIBS})
+    endif()
+
   endif()
 
-  message(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}")
-  message(STATUS "CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}")
-  message(STATUS "LINK_FLAGS = ${LINK_FLAGS}")
+  message(STATUS "OMEGA_CXX_FLAGS           = ${CMAKE_CXX_FLAGS}")
+  message(STATUS "OMEGA_INCLUDE_DIRECTORIES = ${OMEGA_INCLUDE_DIRECTORIES}")
+  message(STATUS "OMEGA_LINK_DIRECTORIES    = ${OMEGA_LINK_DIRECTORIES}")
+  message(STATUS "OMEGA_LINK_OPTIONS        = ${OMEGA_LINK_OPTIONS}")
 
   if(OMEGA_INSTALL_PREFIX)
     set(CMAKE_INSTALL_PREFIX ${OMEGA_INSTALL_PREFIX})

diff --git a/components/omega/doc/devGuide/CMakeBuild.md b/components/omega/doc/devGuide/CMakeBuild.md
@@ -14,6 +14,8 @@ steps: Setup, Update, Build, and Output.
 The build step consists of adding three subdirectories that drive builds
 for external libraries, the Omega model, and optional tests.
 
+Python is required to use this build system.
+
 ## Step 1: Setup
 
 During this step, the build-controlling variables are configured.
@@ -34,7 +36,7 @@ with an underscore ("_").
 The following is a list of Omega-specific variables available in
 this version:
 
-```vbnet
+```
 OMEGA_PROJECT_NAME: Name of the project ("OmegaOceanModel")
 OMEGA_EXE_NAME: Name of the executable ("omega.exe")
 OMEGA_LIB_NAME: Name of the library ("OmegaLib")
@@ -47,15 +49,17 @@ OMEGA_ARCH: User-defined programming framework (e.g., "CUDA", "HIP", "OPENMP", "
 OMEGA_${YAKL_ARCH}_FLAGS: Framework-specific compiler flags
 OMEGA_CXX_COMPILER: C++ compiler
 OMEGA_CIME_COMPILER: E3SM compiler name defined in config_machines.xml
-OMEGA_CXX_FLAGS: C++ compiler flags
-OMEGA_LINK_FLAGS: linker flags
+OMEGA_CXX_FLAGS: a list for C++ compiler flags
+OMEGA_LINK_OPTIONS: a list for linker flags
+OMEGA_INCLUDE_DIRECTORIES: Directory where header files are located
+OMEGA_LINK_DIRECTORIES: Directory where library files are located
 OMEGA_BUILD_EXECUTABLE: Enable building the Omega executable
 OMEGA_BUILD_TEST: Enable building Omega tests
 ```
 
 E3SM-specific variables
 
-```vbnet
+```
 E3SM_SOURCE_DIR: E3SM component directory (${E3SM_ROOT}/components)
 E3SM_CIME_ROOT: CIME root directory
 E3SM_CIMECONFIG_ROOT: E3SM CIME config directory
@@ -65,12 +69,11 @@ E3SM_DEFAULT_BUILD_TYPE: E3SM build type (Release or Debug)
 
 CMake variables
 
-```vbnet
+```
 CMAKE_CURRENT_SOURCE_DIR
 CMAKE_CXX_STANDARD
 CMAKE_CXX_COMPILER
 CMAKE_CXX_FLAGS
-LINK_FLAGS
 CMAKE_CURRENT_LIST_DIR
 CMAKE_BUILD_TYPE
 CMAKE_INSTALL_PREFIX
@@ -81,8 +84,9 @@ CMAKE_VERSION
 ## Step 2: Update
 
 In this step, CMake is configured, and external library variables,
-such as YAKL, are set based on the settings defined in the Setup step.
-The integrity of the build setup is verified at the end of this step.
+such as YAKL, MPI, NetCDF, and PNetCDF, are set based on the settings
+defined in the Setup step. The integrity of the build setup is verified
+at the end of this step.
 
 ## Step 3: Build
 

diff --git a/components/omega/doc/userGuide/OmegaBuild.md b/components/omega/doc/userGuide/OmegaBuild.md
@@ -70,6 +70,8 @@ test
 
 To build the Omega, execute the `make` command in the build directory.
 
+Typical output will look something like:
+
 ```sh
 >> make
 Scanning dependencies of target yakl

diff --git a/components/omega/gen_machine_info.py b/components/omega/gen_machine_info.py
@@ -1,5 +1,7 @@
 import os, sys, shutil, argparse, re, subprocess
 
+# TODO: print basic info on screen and verbose info too
+
 pat_envvar = re.compile(r'^([_\d\w]+)=(.*)$', flags=re.MULTILINE)
 
 def parse_cmdline():
@@ -11,18 +13,19 @@ def parse_cmdline():
                     description='generate machine specific infomation',
                     epilog='Contact: <T.B.D.>')
 
-    parser.add_argument('-p', '--cimepath', default=os.path.join(
-                            here, "..", "..", "cime"))              # CIME root path
+    parser.add_argument('-p', '--cimepath', default=os.path.realpath(os.path.join(
+                            here, "..", "..", "cime")))             # CIME root path
     parser.add_argument('-o', '--outpath', default="_Omega.cmake")  # outfile path
     parser.add_argument('-c', '--compiler') # compiler
+    parser.add_argument('-v', '--verbose') # verbose output
 
     args = parser.parse_args()
 
-    return (args.cimepath, args.outpath, args.compiler)
+    return (args.cimepath, args.outpath, args.compiler, args.verbose)
 
 def main():
 
-    cimepath, outpath, compiler = parse_cmdline()
+    cimepath, outpath, compiler, verbose = parse_cmdline()
 
     sys.path.insert(0, cimepath)
 
@@ -50,13 +53,6 @@ def __init__(self, cimepath, outpath, compiler):
 
         def get_modules(self, outvar):
 
-            #### NOTE
-            # machtocheck = self.get(node, "MACH")  # to get attribute
-            # self.get_optional_child("NODENAME_REGEX", root=node) get optioal child
-            # self.text(regex_str_node) # get text value
-            # regex = re.compile(regex_str) # regex
-            # if regex.match(nametomatch):
-
             module_system_node = self.get_child("module_system") 
             module_system_type = self.get(module_system_node, "type")
             check  = (True if self.get(module_system_node,
@@ -66,18 +62,17 @@ def get_modules(self, outvar):
                 print("ERROR: '%s' module system is not supported." % module_system_type)
                 exit(-1)
 
-            out1 = subprocess.run("env", capture_output=True, shell=True, check=check)
-            env1 = str(out1.stdout, 'UTF-8')
+            out1 = subprocess.check_output("env", shell=True)
+            env1 = str(out1, 'UTF-8')
 
             module_nodes = self.get_children(
                 "modules", root=module_system_node
             )
 
-            SEP = "########################################"
             modcmd = "module"
 
             shcmds = []
-            #for node in module_system.get_children():
+
             for module_node in module_nodes:
                 compiler = self.get(module_node, "compiler")
                 if compiler is None or re.match(compiler, self.compiler):
@@ -90,11 +85,10 @@ def get_modules(self, outvar):
                         else:
                             shcmds.append("%s %s %s" % (modcmd, name, module))
 
-            shcmds.append("export _OMEGA_CXX_PATH=$(which CC)")
             shcmds.append("env")
 
-            out2 = subprocess.run(";".join(shcmds), capture_output=True, shell=True, check=check)
-            env2 = str(out2.stdout, 'UTF-8')
+            out2 = subprocess.check_output(";".join(shcmds), shell=True)
+            env2 = str(out2, 'UTF-8')
 
             parsed1 = {}
 
@@ -110,10 +104,6 @@ def get_modules(self, outvar):
 
         def get_envs(self, outvar):
 
-#            env_nodes = self.get_children(
-#                "environment_variables", 
-#            )
-
             envvar_nodes = self.get_children(root=self.get_child("environment_variables"))
 
             for envvar_node in envvar_nodes:
@@ -155,9 +145,8 @@ def write_output(self, outvar):
                 f.write("set(MACH %s)\n" % self.machname)
                 f.write("set(OS %s)\n" % self.machos)
                 f.write("set(COMPILER %s)\n" % self.compiler)
-                f.write("set(CASEROOT %s)\n" % self.machpath)
-                f.write("include(%s)\n" % self.macrospath)
-                f.write("set(CMAKE_CXX_COMPILER %s)\n" % outvar["_OMEGA_CXX_PATH"])
+                f.write("set(CASEROOT %s)\n" % os.path.realpath(self.machpath))
+                f.write("include(%s)\n" % os.path.realpath(self.macrospath))
 
                 f.write("message(STATUS \"End of reading E3SM machine info\")\n")
 

diff --git a/components/omega/src/CMakeLists.txt b/components/omega/src/CMakeLists.txt
@@ -5,14 +5,27 @@ file(GLOB _LIBSRC_FILES base/*.cpp infra/*.cpp ocn/*.cpp)
 
 # Create the library target
 add_library(${OMEGA_LIB_NAME} ${_LIBSRC_FILES})
-target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LDFLAGS}")
+target_link_libraries(${OMEGA_LIB_NAME} spdlog "${OMEGA_LINK_FLAGS}")
 
 # add include directories
 target_include_directories(
-		${OMEGA_LIB_NAME}
-		PRIVATE 
-		"${OMEGA_SOURCE_DIR}/src/base"
-		"${OMEGA_SOURCE_DIR}/src/infra"
+    ${OMEGA_LIB_NAME}
+    PRIVATE 
+    "${OMEGA_SOURCE_DIR}/src/base"
+    "${OMEGA_SOURCE_DIR}/src/infra"
+     ${OMEGA_INCLUDE_DIRECTORIES}
+)
+
+target_link_directories(
+    ${OMEGA_LIB_NAME}
+    PRIVATE 
+    ${OMEGA_LINK_DIRECTORIES}
+)
+
+target_link_options(
+    ${OMEGA_LIB_NAME}
+    PRIVATE 
+    ${OMEGA_LINK_OPTIONS}
 )
 
 # include yakl cmake utility