Merge branch 'jgfouca/mct_no_makefile' into next (PR #6009)

MCT/Mpi-serial: Remove dependence on CIME/Tools/Makefile

This is a continuation of the effort to remove all use of the
CIME/Tools/Makefile from the E3SM build system so we can be free to
change how we express our build settings in our cmake macros.

Changes:
1) Move the python build wrapper scripts for MCT and
mpi-serial from CIME to E3SM.
2) Refactor and clean up these scripts
to use a similar approach as was used for the scorpio build wrapper:
remove the reliance on CIME/Tools/Makefile and instead pull the needed
items from the Macros directly and call the native build system
(autoconf in this case) directly.
3) Clean up the macro extraction a bit
4) Fix pnetcdf setting for spio

Testing:
* Mappy_gnu A case (Mpilib=serial, openmpi)
* anlgce_gnu A case (Mpilib=serial, mpich)
* pm-cpu_gnu A case ((Mpilib=mpich, mpi-serial does not work on pm-cpu on master)
* summit_ibm A case (Mpilib=spectrum-mpi, mpi-serial does not work on summit_ibm on master)
* summit_gnu A case (Mpilib=spectrum-mpi, mpi-serial)

[BFB]

cime_config/config_files.xml

@@ -540,9 +540,9 @@
       <value lib="gptl">$SRCROOT/share/build/buildlib.gptl</value>
       <value lib="pio">$CIMEROOT/CIME/build_scripts/buildlib.pio</value>
       <value lib="spio">$SRCROOT/share/build/buildlib.spio</value>
-      <value lib="mct">$CIMEROOT/CIME/build_scripts/buildlib.mct</value>
+      <value lib="mct">$SRCROOT/share/build/buildlib.mct</value>
       <value lib="csm_share">$SRCROOT/share/build/buildlib.csm_share</value>
-      <value lib="mpi-serial">$CIMEROOT/CIME/build_scripts/buildlib.mpi-serial</value>
+      <value lib="mpi-serial">$SRCROOT/share/build/buildlib.mpi-serial</value>
       <value lib="cprnc">$CIMEROOT/CIME/build_scripts/buildlib.cprnc</value>
     </values>
     <group>case_last</group>

share/build/buildlib.mct

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+import sys, os, logging, argparse
+
+_CIMEROOT = os.getenv("CIMEROOT")
+sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools"))
+
+from standard_script_setup import *
+from CIME.config import Config
+from CIME import utils
+from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging, expect
+from CIME.case import Case
+import glob
+
+sys.path.append(os.path.dirname(__file__))
+from buildlib_util import extract_from_macros
+
+logger = logging.getLogger(__name__)
+
+###############################################################################
+def parse_command_line(args, description):
+###############################################################################
+    parser = argparse.ArgumentParser(
+        usage="""\n{0} [--debug]
+OR
+{0} --verbose
+OR
+{0} --help
+
+\033[1mEXAMPLES:\033[0m
+    \033[1;32m# Run \033[0m
+    > {0}
+""".format(
+            os.path.basename(args[0])
+        ),
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    utils.setup_standard_logging_options(parser)
+
+    parser.add_argument("buildroot", help="build path root")
+
+    parser.add_argument("installpath", help="install path ")
+
+    parser.add_argument(
+        "caseroot", nargs="?", default=os.getcwd(), help="Case directory to build"
+    )
+
+    args = utils.parse_args_and_handle_standard_logging_options(args, parser)
+
+    return args.buildroot, args.installpath, args.caseroot
+
+###############################################################################
+def buildlib(bldroot, installpath, case):
+###############################################################################
+    caseroot  = case.get_value("CASEROOT")
+    cimeroot  = case.get_value("CIMEROOT")
+    srcroot   = case.get_value("SRCROOT")
+    gmake_cmd = case.get_value("GMAKE")
+    gmake_j   = case.get_value("GMAKE_J")
+    mpilib    = case.get_value("MPILIB")
+
+    expect(
+        os.path.abspath(os.path.realpath(cimeroot)) == os.path.abspath(os.path.realpath(_CIMEROOT)),
+        "CIMEROOT mismatch {} vs {}".format(_CIMEROOT, cimeroot),
+    )
+
+    customize_path = os.path.join(srcroot, "cime_config", "customize")
+
+    config = Config.load(customize_path)
+
+    mct_path = config.mct_path.format(srcroot=srcroot)
+
+    for _dir in ("mct", "mpeu"):
+        if not os.path.isdir(os.path.join(bldroot, _dir)):
+            os.makedirs(os.path.join(bldroot, _dir))
+
+        copyifnewer(
+            os.path.join(mct_path, _dir, "Makefile"),
+            os.path.join(bldroot, _dir, "Makefile"),
+        )
+
+    #
+    # Get autoconf arguments
+    #
+
+    fc, cc, _, mfc, mcc, _, fflags, cflags, _, cppdefs, ldflags, ffree, config_args = \
+        extract_from_macros(case, "mct", extra_vars=("FREEFLAGS", "CONFIG_ARGS"))
+
+    fflags += f" {ffree} -I{installpath}/include"
+    cflags += f" -I{installpath}/include"
+    if mpilib == "mpi-serial":
+        mfc = fc
+        mcc = cc
+    else:
+        fc = mfc
+        cc = mcc
+
+    # Only need the netcdf_c library
+    if "NETCDF_PATH" in os.environ:
+        netcdf_args = f"NETCDF_PATH={os.environ['NETCDF_PATH']} "
+    elif "NETCDF_C_PATH" in os.environ:
+        netcdf_args = f"NETCDF_PATH={os.environ['NETCDF_C_PATH']} "
+
+    config_cmd = f"{mct_path}/configure CC={cc} FC={fc} MPICC={mcc} MPIFC={mfc} FCFLAGS='{fflags}' CPPDEFS='{cppdefs}' CFLAGS='{cflags}' LDFLAGS='{ldflags}' {config_args} {netcdf_args} --srcdir {mct_path}"
+
+    # run configure
+    run_bld_cmd_ensure_logging(config_cmd, logger, from_dir=bldroot)
+
+    # Now we run the mct make command
+    gmake_opts = "-f {} ".format(os.path.join(mct_path, "Makefile"))
+    gmake_opts += " -C {} ".format(bldroot)
+    gmake_opts += " -j {} ".format(gmake_j)
+    gmake_opts += " SRCDIR={} ".format(os.path.join(mct_path))
+
+    cmd = "{} {}".format(gmake_cmd, gmake_opts)
+    run_bld_cmd_ensure_logging(cmd, logger, from_dir=bldroot)
+
+    for _dir in ("mct", "mpeu"):
+        for _file in glob.iglob(os.path.join(bldroot, _dir, "*.a")):
+            logger.info("Installing {} to {}".format(_file, installpath))
+            copyifnewer(
+                _file, os.path.join(installpath, "lib", os.path.basename(_file))
+            )
+        for _file in glob.iglob(os.path.join(bldroot, _dir, "*.mod")):
+            logger.info("Installing {} to {}".format(_file, installpath))
+            copyifnewer(
+                _file, os.path.join(installpath, "include", os.path.basename(_file))
+            )
+
+###############################################################################
+def _main(argv, documentation):
+###############################################################################
+    bldroot, installpath, caseroot = parse_command_line(argv, documentation)
+    with Case(caseroot) as case:
+        buildlib(bldroot, installpath, case)
+
+###############################################################################
+if __name__ == "__main__":
+    _main(sys.argv, __doc__)

share/build/buildlib.mpi-serial

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+import os, sys, logging, argparse
+
+from standard_script_setup import *
+from CIME import utils
+from CIME.config import Config
+from CIME.utils import copyifnewer, run_bld_cmd_ensure_logging
+from CIME.case import Case
+from CIME.build import get_standard_makefile_args
+import glob
+
+sys.path.append(os.path.dirname(__file__))
+from buildlib_util import extract_from_macros
+
+logger = logging.getLogger(__name__)
+
+###############################################################################
+def parse_command_line(args, description):
+###############################################################################
+    parser = argparse.ArgumentParser(
+        usage="""\n{0} [--debug]
+OR
+{0} --verbose
+OR
+{0} --help
+
+\033[1mEXAMPLES:\033[0m
+    \033[1;32m# Run \033[0m
+    > {0}
+""".format(
+            os.path.basename(args[0])
+        ),
+        description=description,
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+
+    utils.setup_standard_logging_options(parser)
+
+    parser.add_argument("buildroot", help="build path root")
+
+    parser.add_argument("installpath", help="install path ")
+
+    parser.add_argument(
+        "caseroot", nargs="?", default=os.getcwd(), help="Case directory to build"
+    )
+
+    args = utils.parse_args_and_handle_standard_logging_options(args, parser)
+
+    return args.buildroot, args.installpath, args.caseroot
+
+###############################################################################
+def buildlib(bldroot, installpath, case):
+###############################################################################
+    caseroot  = case.get_value("CASEROOT")
+    srcroot   = case.get_value("SRCROOT")
+    gmake_cmd = case.get_value("GMAKE")
+    gmake_j   = case.get_value("GMAKE_J")
+    mpilib    = case.get_value("MPILIB")
+
+    customize_path = os.path.join(srcroot, "cime_config", "customize")
+
+    config = Config.load(customize_path)
+
+    mct_path = config.mct_path.format(srcroot=srcroot)
+    mpi_serial_path = os.path.join(mct_path, "mpi-serial")
+
+    for _file in glob.iglob(os.path.join(mpi_serial_path, "*.h")):
+        copyifnewer(_file, os.path.join(bldroot, os.path.basename(_file)))
+
+    # Hacky way of getting stuff from Macros.
+    fc, cc, _, _, _, _, fflags, cflags, _, cppdefs, ldflags, ffree, config_args = \
+        extract_from_macros(case, "mpi-serial", extra_vars=("FREEFLAGS", "CONFIG_ARGS"))
+
+    fflags += f" {ffree}"
+
+    # Only need the netcdf_c library
+    if "NETCDF_PATH" in os.environ:
+        netcdf_args = f"NETCDF_PATH={os.environ['NETCDF_PATH']} "
+    elif "NETCDF_C_PATH" in os.environ:
+        netcdf_args = f"NETCDF_PATH={os.environ['NETCDF_C_PATH']} "
+
+    config_cmd = f"{mpi_serial_path}/configure CC={cc} FC={fc} FCFLAGS='{fflags}' CPPDEFS='{cppdefs}' CFLAGS='{cflags}' LDFLAGS='{ldflags}' {config_args} {netcdf_args} --srcdir {mpi_serial_path}"
+
+    # run configure
+    run_bld_cmd_ensure_logging(config_cmd, logger, from_dir=bldroot)
+
+    # Now we run the mpi-serial make command
+    gmake_opts = "-f {} ".format(os.path.join(mpi_serial_path, "Makefile"))
+    gmake_opts += " -C {} ".format(bldroot)
+    gmake_opts += " -j {} ".format(case.get_value("GMAKE_J"))
+    gmake_opts += " SRCDIR={} ".format(os.path.join(mct_path))
+
+    cmd = "{} {}".format(gmake_cmd, gmake_opts)
+    run_bld_cmd_ensure_logging(cmd, logger)
+
+    copyifnewer(
+        os.path.join(bldroot, "libmpi-serial.a"),
+        os.path.join(installpath, "lib", "libmpi-serial.a"),
+    )
+    for _file in ("mpi.h", "mpif.h", "mpi.mod", "MPI.mod"):
+        if os.path.isfile(os.path.join(bldroot, _file)):
+            copyifnewer(
+                os.path.join(bldroot, _file),
+                os.path.join(installpath, "include", _file),
+            )
+
+###############################################################################
+def _main(argv, documentation):
+###############################################################################
+    bldroot, installpath, caseroot = parse_command_line(argv, documentation)
+    with Case(caseroot) as case:
+        buildlib(bldroot, installpath, case)
+
+###############################################################################
+if __name__ == "__main__":
+    _main(sys.argv, __doc__)

share/build/buildlib.spio

@@ -7,21 +7,14 @@ sys.path.append(os.path.join(cimeroot, "CIME", "Tools"))
 import glob, re, shutil
 from standard_script_setup import *
 from CIME import utils
-from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy, run_cmd
+from CIME.utils import expect, run_bld_cmd_ensure_logging, safe_copy
 from CIME.build import get_standard_makefile_args
 from CIME.case import Case
 
-logger = logging.getLogger(__name__)
+sys.path.append(os.path.dirname(__file__))
+from buildlib_util import extract_from_macros
 
-###############################################################################
-def extract_from_macros(macro_dump, varname):
-###############################################################################
-    look_for = f"{varname} :="
-    for line in macro_dump.splitlines():
-        if line.startswith(look_for):
-            return line.split(":=")[-1].strip()
-
-    return ""
+logger = logging.getLogger(__name__)
 
 ###############################################################################
 def parse_command_line(args, description):
@@ -120,6 +113,11 @@ def buildlib(bldroot, installpath, case):
     if "NETCDF_FORTRAN_PATH" in os.environ:
         cmake_opts += f"-DNetCDF_Fortran_PATH:PATH={os.environ['NETCDF_FORTRAN_PATH']} "
 
+    if "PNETCDF_PATH" in os.environ and mpilib != "mpi-serial":
+        cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} "
+    else:
+        cmake_opts += "-DWITH_PNETCDF:LOGICAL=FALSE -DPIO_USE_MPIIO:LOGICAL=FALSE "
+
     # scorpio needs a little help finding hdf5. The FindHDF5 in scopio uses the
     # env var "HDF5" instead of "HDF5_ROOT". If HDF5_ROOT is not set, but we have
     # h5dump in our path, we can use that info to set up HDF5. For now, we will
@@ -139,40 +137,43 @@ def buildlib(bldroot, installpath, case):
     if "SZIP_ROOT" in os.environ:
         os.environ["SZIP"] = os.environ["SZIP_ROOT"]
 
-    if "PNETCDF_PATH" in os.environ:
-        cmake_opts += f"-DPnetCDF_PATH:PATH={os.environ['PNETCDF_PATH']} "
-    else:
-        cmake_opts += "-DWITH_PNETCDF:LOGICAL=FALSE -DPIO_USE_MPIIO:LOGICAL=FALSE "
     if mpilib == "mpi-serial":
         cmake_opts += f"-DPIO_USE_MPISERIAL=TRUE -DMPISERIAL_PATH={installpath} "
+
     cmake_opts += "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON "
     cmake_opts += f"-DGPTL_PATH:STRING={installpath} "
     cmake_opts += "-DPIO_ENABLE_TESTS:BOOL=OFF -DPIO_USE_MALLOC:BOOL=ON "
     cmake_opts += f"-DUSER_CMAKE_MODULE_PATH:LIST={cimeroot}/CIME/non_py/src/CMake "
 
-    # Hacky way of getting stuff from Macros. Should scorpio just load the E3SM
-    # macros once they are using cmake names for things?
-    make_args = get_standard_makefile_args(case, shared_lib=True)
-    macro_dump = run_cmd(f"make -f Macros.make COMP_NAME=spio {make_args} -p")[1]
-    fflags = extract_from_macros(macro_dump, "FFLAGS")
+    # Scorpio should just load the E3SM macros once  are using cmake names for things
+    sfc, scc, scxx, mfc, mcc, mcxx, fflags, cflags, cxxflags, cppdefs, ldflags, cxx_libs, spio_cmake_opts, pio_hints = \
+        extract_from_macros(case, "spio", extra_vars=("CXX_LIBS", "CMAKE_OPTS", "PIO_FILESYSTEM_HINTS"))
+
+    cppdefs += " -DTIMING"
+
     # Setting GPTL_PATH is currently insufficient scorpio to find gptl headers?
-    cflags = extract_from_macros(macro_dump, "CFLAGS") + f" -I{installpath}/include"
-    cxxflags = extract_from_macros(macro_dump, "CXXFLAGS") + f" -I{installpath}/include"
-    cppdefs  = extract_from_macros(macro_dump, "CPPDEFS") + f" -I{installpath}/include"
-    ldflags  = extract_from_macros(macro_dump, "LDFLAGS")
+    incl_flag = f" -I{installpath}/include"
+
     if mpilib == "mpi-serial":
-        fc = extract_from_macros(macro_dump, "SFC")
-        cc = extract_from_macros(macro_dump, "SCC")
-        cxx = extract_from_macros(macro_dump, "SCXX")
+        fc  = sfc
+        cc  = scc
+        cxx = scxx
     else:
-        fc = extract_from_macros(macro_dump, "MPIFC")
-        cc = extract_from_macros(macro_dump, "MPICC")
-        cxx = extract_from_macros(macro_dump, "MPICXX")
+        fc  = mfc
+        cc  = mcc
+        cxx = mcxx
 
-    cppdefs += " -DTIMING"
-    cmake_opts += f"-DCMAKE_Fortran_FLAGS:STRING='{fflags} {cppdefs}' "
-    cmake_opts += f"-DCMAKE_C_FLAGS:STRING='{cflags} {cppdefs}' "
-    cmake_opts += f"-DCMAKE_CXX_FLAGS:STRING='{cxxflags} {cppdefs}' "
+    cmake_opts += f"-DCMAKE_Fortran_FLAGS:STRING='{fflags} {incl_flag} {cppdefs}' "
+    cmake_opts += f"-DCMAKE_C_FLAGS:STRING='{cflags} {incl_flag} {cppdefs}' "
+    cmake_opts += f"-DCMAKE_CXX_FLAGS:STRING='{cxxflags} {incl_flag} {cppdefs}' "
+    if spio_cmake_opts:
+        cmake_opts += spio_cmake_opts + " "
+
+    if cxx_libs:
+        ldflags += " " + cxx_libs
+
+    if pio_hints:
+        cmake_opts += f"-DPIO_FILESYSTEM_HINTS:STRING='$(PIO_FILESYSTEM_HINTS)' "
 
     # This runs the pio cmake command from the cime case Makefile
     logger.info("Configuring SCORPIO")