Change E3SM to use cmake macro file system

CMakeLists.txt

@@ -93,32 +93,31 @@ endif()
 
 # csm_share (we don't build it here because it seems to be built differently
 # by different tests?)
-set(SHARE_ROOT "${CIME_ROOT}/src/share")
+
 if (EXISTS ${SRC_ROOT}/share/src)
   add_subdirectory(${SRC_ROOT}/share/src share_src)
   add_subdirectory(${SRC_ROOT}/components/cpl7/mct_shr mct_src)
   add_subdirectory(${SRC_ROOT}/share/unit_test_stubs/util csm_share_stubs)
   include_directories(${SRC_ROOT}/share/include)
 else()
-  set(SHARE_ROOT "${CIME_ROOT}/src/share")
-  add_subdirectory(${SHARE_ROOT}/util csm_share)
-  add_subdirectory(${SHARE_ROOT}/unit_test_stubs/util csm_share_stubs)
-  include_directories(${SHARE_ROOT}/include)
+  add_subdirectory(${SRC_ROOT}/share/util csm_share)
+  add_subdirectory(${SRC_ROOT}/share/unit_test_stubs/util csm_share_stubs)
+  include_directories(${SRC_ROOT}/share/include)
 endif()
 
 # esmf_wrf_timemgr not built here because it depends on csm_share.
 if (EXISTS ${SRC_ROOT}/share/src/esmf_wrf_timemgr)
   add_subdirectory(${SRC_ROOT}/share/src/esmf_wrf_timemgr esmf_wrf_timemgr)
   include_directories(${SRC_ROOT}/share/src/esmf_wrf_timemgr)
 else()
-  add_subdirectory(${SHARE_ROOT}/esmf_wrf_timemgr esmf_wrf_timemgr)
-  include_directories(${SHARE_ROOT}/esmf_wrf_timemgr)
+  add_subdirectory(${SRC_ROOT}/share/esmf_wrf_timemgr esmf_wrf_timemgr)
+  include_directories(${SRC_ROOT}/share/esmf_wrf_timemgr)
 endif()
 
 # Now the actual test directories.
 if (EXISTS ${SRC_ROOT}/components/cpl7/driver/unit_test)
   add_subdirectory(${SRC_ROOT}/components/cpl7/driver/unit_test unit_test)
 else()
-  add_subdirectory(${CIME_ROOT}/src/drivers/mct/unit_test)
+  add_subdirectory(${SRC_ROOT}/driver-mct/unit_test unit_test)
 endif()
 add_subdirectory(${SRC_ROOT}/share/test/unit ${CMAKE_BINARY_DIR}/unittests)

scripts/lib/CIME/BuildTools/configure.py

@@ -48,11 +48,11 @@ def configure(machobj, output_dir, macros_format, compiler, mpilib, debug,
         out_file_name = os.path.join(output_dir,"Macros."+suffixes[form])
         macro_maker.write_macros_file(macros_file=out_file_name, output_format=suffixes[form])
 
-    _copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler)
-    _generate_env_mach_specific(output_dir, machobj, compiler, mpilib,
-                                debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv)
+    copy_depends_files(machobj.get_machine_name(), machobj.machines_dir, output_dir, compiler)
+    generate_env_mach_specific(output_dir, machobj, compiler, mpilib,
+                               debug, comp_interface, sysos, unit_testing, threaded, noenv=noenv)
 
-def _copy_depends_files(machine_name, machines_dir, output_dir, compiler):
+def copy_depends_files(machine_name, machines_dir, output_dir, compiler):
     """
     Copy any system or compiler Depends files if they do not exist in the output directory
     If there is a match for Depends.machine_name.compiler copy that and ignore the others
@@ -87,8 +87,8 @@ def get_value(self, attrib):
         expect(attrib in self._vals, "FakeCase does not support getting value of '%s'" % attrib)
         return self._vals[attrib]
 
-def _generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
-                                comp_interface, sysos, unit_testing, threaded, noenv=False):
+def generate_env_mach_specific(output_dir, machobj, compiler, mpilib, debug,
+                               comp_interface, sysos, unit_testing, threaded, noenv=False):
     """
     env_mach_specific generation.
     """

scripts/lib/CIME/XML/machines.py

@@ -29,6 +29,8 @@ def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=Non
         self.machine = None
         self.machines_dir = None
         self.custom_settings = {}
+        self.extra_machines_dir = extra_machines_dir
+
         schema = None
         supported_models = []
         if files is None:
@@ -90,6 +92,9 @@ def get_machines_dir(self):
         """
         return self.machines_dir
 
+    def get_extra_machines_dir(self):
+        return self.extra_machines_dir
+
     def get_machine_name(self):
         """
         Return the name of the machine

scripts/lib/CIME/build.py

@@ -21,6 +21,38 @@
      "CAM_CONFIG_OPTS", "COMP_LND", "COMPARE_TO_NUOPC", "HOMME_TARGET",
      "OCN_SUBMODEL", "CISM_USE_TRILINOS", "USE_TRILINOS", "USE_ALBANY", "USE_PETSC")
 
+def generate_makefile_macro(case, caseroot, comp_name):
+    """
+    Only used by E3SM. Generates a flat Makefile macro file based on the CMake cache system.
+    This macro is only used by certain sharedlibs since components use CMake.
+    Since indirection based on comp_name is allowed for sharedlibs, each sharedlib must generate
+    their own macro.
+    """
+    new_cmake_dir = os.path.join(caseroot, "cmake_macros")
+    if not os.path.isdir(new_cmake_dir):
+        return
+
+    cmake_macro = os.path.join(caseroot, "Macros.cmake")
+    expect(os.path.exists(cmake_macro), "Cannot generate Makefile macro without {}".format(cmake_macro))
+
+    cmake_args = get_standard_cmake_args(case, "DO_NOT_USE", shared_lib=True)
+    output = run_cmd_no_fail("cmake -DCONVERT_TO_MAKE=ON -DCOMP_NAME={} {} -C Macros.cmake |& grep E3SM_SET_MAKEFILE_VAR | grep -v BUILD_INTERNAL_IGNORE".format(comp_name, cmake_args), from_dir=caseroot)
+
+    # The Tools/Makefile may have already adding things to CPPDEFS and SLIBS
+    real_output = output.replace("E3SM_SET_MAKEFILE_VAR ", "").\
+                  replace("CPPDEFS := ", "CPPDEFS := $(CPPDEFS) ").\
+                  replace("SLIBS := ", "SLIBS := $(SLIBS) ")
+
+    with open(os.path.join(caseroot, "Macros.make"), "w") as fd:
+        fd.write(
+"""
+# This file is auto-generated, do not edit. If you want to change
+# sharedlib flags, you can edit the cmake_macros in this case. You
+# can change flags for specific sharedlibs only by checking COMP_NAME.
+
+""")
+        fd.write(real_output)
+
 def get_standard_makefile_args(case, shared_lib=False):
     make_args = "CIME_MODEL={} ".format(case.get_value("MODEL"))
     make_args += " SMP={} ".format(stringify_bool(case.get_build_threaded()))
@@ -419,7 +451,6 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
     for lib in libs:
         build_script[lib] = files.get_value("BUILD_LIB_FILE",{"lib":lib})
 
-
     sharedlibroot = os.path.abspath(case.get_value("SHAREDLIBROOT"))
     # Check if we need to build our own cprnc
     if case.get_value("TEST"):
@@ -459,6 +490,10 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
         expect(os.path.exists(my_file),"Build script {} for component {} not found.".format(my_file, lib))
         logger.info("Building {} with output to file {}".format(lib,file_build))
 
+        # generate Makefile macro if e3sm
+        if get_model() == "e3sm":
+            generate_makefile_macro(case, caseroot, lib)
+
         run_sub_or_cmd(my_file, [full_lib_path, os.path.join(exeroot, sharedpath), caseroot], 'buildlib',
                        [full_lib_path, os.path.join(exeroot, sharedpath), case], logfile=file_build)
 
@@ -583,6 +618,7 @@ def _clean_impl(case, cleanlist, clean_all, clean_depends):
         casetools = case.get_value("CASETOOLS")
         classic_cmd = "{} -f {} {}".format(gmake, os.path.join(casetools, "Makefile"),
                                            get_standard_makefile_args(case, shared_lib=True))
+        caseroot = case.get_value("CASEROOT")
 
         for clean_item in things_to_clean:
             logging.info("Cleaning {}".format(clean_item))
@@ -592,6 +628,9 @@ def _clean_impl(case, cleanlist, clean_all, clean_depends):
                 clean_cmd = "cd {} && {} clean".format(cmake_path, gmake)
             else:
                 # Item was created by classic build system
+                if get_model() == "e3sm":
+                    generate_makefile_macro(case, caseroot, clean_item)
+
                 clean_cmd = "{} {}{}".format(classic_cmd, "clean" if clean_item in cleanlist else "clean_depends", clean_item)
 
             logger.info("calling {}".format(clean_cmd))

scripts/lib/CIME/case/case_setup.py

@@ -3,19 +3,19 @@
 case_setup is a member of class Case from file case.py
 """
 
-import errno
-
 from CIME.XML.standard_module_setup import *
 
 from CIME.XML.machines      import Machines
-from CIME.BuildTools.configure import configure
+from CIME.BuildTools.configure import configure, generate_env_mach_specific, copy_depends_files
 from CIME.utils             import run_and_log_case_status, get_model, \
     get_batch_script_for_job, safe_copy, file_contains_python_function, import_from_file
 from CIME.utils             import batch_jobid
 from CIME.utils             import transform_vars
 from CIME.test_status       import *
 from CIME.locked_files      import unlock_file, lock_file
 
+import errno, shutil
+
 logger = logging.getLogger(__name__)
 
 ###############################################################################
@@ -112,6 +112,48 @@ def _get_user_nl_list(case, default_nlfile, model_dir):
     else:
         return [default_nlfile]
 
+###############################################################################
+def _create_macros_e3sm(caseroot, srcroot, mach_obj, compiler):
+###############################################################################
+    if not os.path.isfile("Macros.cmake"):
+        safe_copy(os.path.join(srcroot, "cime_config/machines/cmake_macros/Macros.cmake"), caseroot)
+    if not os.path.exists("cmake_macros"):
+        shutil.copytree(os.path.join(srcroot, "cime_config/machines/cmake_macros"), os.path.join(caseroot, "cmake_macros"))
+
+    copy_depends_files(mach_obj.get_machine_name(), mach_obj.machines_dir, caseroot, compiler)
+
+###############################################################################
+def _create_macros(case, mach_obj, caseroot, compiler, mpilib, debug, comp_interface, sysos):
+###############################################################################
+    """
+    creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
+    and env_mach_specific.xml if they don't already exist.
+    """
+    reread = not os.path.isfile("env_mach_specific.xml")
+    srcroot = case.get_value("SRCROOT")
+    new_cmake_macro = os.path.join(srcroot, "cime_config/machines/cmake_macros/Macros.cmake")
+    if reread:
+        case.flush()
+        generate_env_mach_specific(
+            caseroot, mach_obj, compiler, mpilib, debug, comp_interface,
+            sysos, False, threaded=case.get_build_threaded(), noenv=True,)
+        case.read_xml()
+
+    if get_model() == "e3sm" and os.path.exists(new_cmake_macro): # change "e3sm" to "xxxx" to disable new macro code
+        _create_macros_e3sm(caseroot, srcroot, mach_obj, compiler)
+    else:
+        if not os.path.isfile("Macros.make"):
+            configure(mach_obj,
+                      caseroot, ["Makefile"], compiler, mpilib, debug, comp_interface, sysos, noenv=True,
+                      extra_machines_dir=mach_obj.get_extra_machines_dir())
+
+        # Also write out Cmake macro file
+        if not os.path.isfile("Macros.cmake"):
+            configure(mach_obj,
+                      caseroot, ["CMake"], compiler, mpilib, debug, comp_interface, sysos, noenv=True,
+                      extra_machines_dir=mach_obj.get_extra_machines_dir())
+
+
 ###############################################################################
 def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False, keep=None):
 ###############################################################################
@@ -127,8 +169,11 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False,
     sysos = case.get_value("OS")
     comp_interface = case.get_value("COMP_INTERFACE")
     extra_machines_dir = case.get_value("EXTRA_MACHDIR")
+
     expect(mach is not None, "xml variable MACH is not set")
 
+    mach_obj = Machines(machine=mach, extra_machines_dir=extra_machines_dir)
+
     # Check that $DIN_LOC_ROOT exists or can be created:
     if not non_local:
         din_loc_root = case.get_value("DIN_LOC_ROOT")
@@ -148,10 +193,13 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False,
     if reset or clean:
         # clean setup-generated files
         batch_script = get_batch_script_for_job(case.get_primary_job())
-        files_to_clean = [batch_script, "env_mach_specific.xml", "Macros.make", "Macros.cmake"]
+        files_to_clean = [batch_script, "env_mach_specific.xml", "Macros.make", "Macros.cmake", "cmake_macros"]
         for file_to_clean in files_to_clean:
             if os.path.exists(file_to_clean) and not (keep and file_to_clean in keep):
-                os.remove(file_to_clean)
+                if os.path.isdir(file_to_clean):
+                    shutil.rmtree(file_to_clean)
+                else:
+                    os.remove(file_to_clean)
                 logger.info("Successfully cleaned {}".format(file_to_clean))
 
         if not test_mode:
@@ -161,32 +209,16 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False,
         # Cannot leave case in bad state (missing env_mach_specific.xml)
         if clean and not os.path.isfile("env_mach_specific.xml"):
             case.flush()
-            configure(Machines(machine=mach, extra_machines_dir=extra_machines_dir),
-                      caseroot, ["Makefile"], compiler, mpilib, debug, comp_interface, sysos, noenv=True,
-                      threaded=case.get_build_threaded(),extra_machines_dir=extra_machines_dir)
+            generate_env_mach_specific(
+                caseroot, mach_obj, compiler, mpilib, debug, comp_interface,
+                sysos, False, threaded=case.get_build_threaded(), noenv=True,)
             case.read_xml()
 
     if not clean:
         if not non_local:
             case.load_env()
 
-        # creates the Macros.make, Depends.compiler, Depends.machine, Depends.machine.compiler
-        # and env_mach_specific.xml if they don't already exist.
-        if not os.path.isfile("Macros.make") or not os.path.isfile("env_mach_specific.xml"):
-            reread = not os.path.isfile("env_mach_specific.xml")
-            if reread:
-                case.flush()
-            configure(Machines(machine=mach, extra_machines_dir=extra_machines_dir),
-                      caseroot, ["Makefile"], compiler, mpilib, debug, comp_interface, sysos, noenv=True,
-                      extra_machines_dir=extra_machines_dir)
-            if reread:
-                case.read_xml()
-
-        # Also write out Cmake macro file
-        if not os.path.isfile("Macros.cmake"):
-            configure(Machines(machine=mach, extra_machines_dir=extra_machines_dir),
-                      caseroot, ["CMake"], compiler, mpilib, debug, comp_interface, sysos, noenv=True,
-                      extra_machines_dir=extra_machines_dir)
+        _create_macros(case, mach_obj, caseroot, compiler, mpilib, debug, comp_interface, sysos)
 
         # Set tasks to 1 if mpi-serial library
         if mpilib == "mpi-serial":

scripts/tests/scripts_regression_tests.py

@@ -365,7 +365,8 @@ def test_a_createnewcase(self):
         if os.path.exists(testdir):
             shutil.rmtree(testdir)
         args = " --case %s --compset X --output-root %s --handle-preexisting-dirs=r" % (testdir, cls._testroot)
-        args += " --run-unsupported"
+        if get_model() == "cesm":
+            args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args = args +  " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -455,7 +456,8 @@ def test_b_user_mods(self):
 
         user_mods_dir = os.path.join(CIME.utils.get_python_libs_root(), "..", "tests", "user_mods_test1")
         args = " --case %s --compset X --user-mods-dir %s --output-root %s --handle-preexisting-dirs=r"% (testdir, user_mods_dir, cls._testroot)
-        args += " --run-unsupported"
+        if get_model() == "cesm":
+            args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args = args + " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -649,7 +651,8 @@ def test_h_primary_component(self):
 
         cls._testdirs.append(testdir)
         args = " --case CreateNewcaseTest --script-root %s --compset X --output-root %s --handle-preexisting-dirs u" % (testdir, cls._testroot)
-        args += " --run-unsupported"
+        if get_model() == "cesm":
+            args += " --run-unsupported"
         if TEST_COMPILER is not None:
             args += " --compiler %s"%TEST_COMPILER
         if TEST_MPILIB is not None:
@@ -837,10 +840,12 @@ def test_ka_createnewcase_extra_machines_dir(self):
         # config_machines_template.xml
         args = (" --case {testdir} --compset X --mach mymachine"
                 " --output-root {testroot} --non-local"
-                " --extra-machines-dir {extra_machines_dir}"
-                " --run-unsupported".format(
+                " --extra-machines-dir {extra_machines_dir}".format(
                     testdir=testdir, testroot=cls._testroot,
                     extra_machines_dir=extra_machines_dir))
+        if get_model() == "cesm":
+            args += " --run-unsupported"
+
         if CIME.utils.get_cime_default_driver() == "nuopc":
             args += " --res f19_g17 "
         else:
@@ -854,12 +859,13 @@ def test_ka_createnewcase_extra_machines_dir(self):
                               from_dir=testdir)
 
         # Make sure Macros file contains expected text
-        macros_file_name = os.path.join(testdir, "Macros.make")
-        self.assertTrue(os.path.isfile(macros_file_name))
-        with open(macros_file_name) as macros_file:
-            macros_contents = macros_file.read()
-        expected_re = re.compile("NETCDF_PATH.*/my/netcdf/path")
-        self.assertTrue(expected_re.search(macros_contents))
+        if get_model() != "e3sm":
+            macros_file_name = os.path.join(testdir, "Macros.make")
+            self.assertTrue(os.path.isfile(macros_file_name))
+            with open(macros_file_name) as macros_file:
+                macros_contents = macros_file.read()
+            expected_re = re.compile("NETCDF_PATH.*/my/netcdf/path")
+            self.assertTrue(expected_re.search(macros_contents))
 
     def test_m_createnewcase_alternate_drivers(self):
         # Test that case.setup runs for nuopc and moab drivers