Build scripts in source mods

doc/source/users_guide/create-a-case.rst

@@ -94,7 +94,8 @@ Running **create_newcase** creates various scripts, files and directories ``$CAS
 
    =====================  ===============================================================================================================================
    SourceMods             Top-level directory containing subdirectories for each compset component where
-                          you can place modified source code for that component.
+                          you can place modified source code for that component.  You may also place modified
+			  buildnml and buildlib scripts here.
    =====================  ===============================================================================================================================
 
 - ``Provenance``
@@ -137,5 +138,3 @@ These other files can be "unlocked" as follows:
 - To unlock **env_mach_pes.xml**, run ``case.setup --clean``.
 
 - To unlock **env_build.xml**, run ``case.build --clean``.
-
-

scripts/lib/CIME/build.py

@@ -57,7 +57,7 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
         # thread_bad_results captures error output from thread (expected to be empty)
         # logs is a list of log files to be compressed and added to the case logs/bld directory
         t = threading.Thread(target=_build_model_thread,
-            args=(config_dir, model, caseroot, libroot, bldroot, incroot, file_build,
+            args=(config_dir, model, comp, caseroot, libroot, bldroot, incroot, file_build,
                   thread_bad_results, smp, compiler))
         t.start()
 
@@ -67,18 +67,6 @@ def _build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
     while(threading.active_count() > 1):
         time.sleep(1)
 
-    # aquap has a dependency on atm so we build it after the threaded loop
-
-    for model, comp, nthrds, _, config_dir in complist:
-        smp = nthrds > 1 or build_threaded
-        if comp == "aquap":
-            logger.debug("Now build aquap ocn component")
-            # thread_bad_results captures error output from thread (expected to be empty)
-            # logs is a list of log files to be compressed and added to the case logs/bld directory
-            _build_model_thread(config_dir, comp, caseroot, libroot, bldroot, incroot, file_build,
-                                thread_bad_results, smp, compiler)
-            logs.append(file_build)
-
     expect(not thread_bad_results, "\n".join(thread_bad_results))
 
     #
@@ -282,33 +270,40 @@ def _build_libraries(case, exeroot, sharedpath, caseroot, cimeroot, libroot, lid
             # thread_bad_results captures error output from thread (expected to be empty)
             # logs is a list of log files to be compressed and added to the case logs/bld directory
             thread_bad_results = []
-            _build_model_thread(config_lnd_dir, "lnd", caseroot, libroot, bldroot, incroot,
+            _build_model_thread(config_lnd_dir, "lnd", comp_lnd, caseroot, libroot, bldroot, incroot,
                                 file_build, thread_bad_results, smp, compiler)
             logs.append(file_build)
             expect(not thread_bad_results, "\n".join(thread_bad_results))
 
     return logs
 
 ###############################################################################
-def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incroot, file_build,
+def _build_model_thread(config_dir, compclass, compname, caseroot, libroot, bldroot, incroot, file_build,
                         thread_bad_results, smp, compiler):
 ###############################################################################
     logger.info("Building {} with output to {}".format(compclass, file_build))
     t1 = time.time()
+    cmd = os.path.join(caseroot, "SourceMods", "src." + compname, "buildlib")
+    if os.path.isfile(cmd):
+        logger.warn("WARNING: using local buildlib script for {}".format(compname))
+    else:
+        cmd = os.path.join(config_dir, "buildlib")
+        expect(os.path.isfile(cmd), "Could not find buildlib for {}".format(compname))
+
     with open(file_build, "w") as fd:
-        stat = run_cmd("MODEL={} SMP={} {}/buildlib {} {} {} "
-                       .format(compclass, stringify_bool(smp), config_dir, caseroot, libroot, bldroot),
+        stat = run_cmd("MODEL={} SMP={} {} {} {} {} "
+                       .format(compclass, stringify_bool(smp), cmd, caseroot, libroot, bldroot),
                        from_dir=bldroot,  arg_stdout=fd,
                        arg_stderr=subprocess.STDOUT)[0]
     analyze_build_log(compclass, file_build, compiler)
     if (stat != 0):
-        thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compclass, file_build))
+        thread_bad_results.append("BUILD FAIL: {}.buildlib failed, cat {}".format(compname, file_build))
 
     for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
         shutil.copy(mod_file, incroot)
 
     t2 = time.time()
-    logger.info("{} built in {:f} seconds".format(compclass, (t2 - t1)))
+    logger.info("{} built in {:f} seconds".format(compname, (t2 - t1)))
 
 ###############################################################################
 def _clean_impl(case, cleanlist, clean_all):

scripts/lib/CIME/buildlib.py

@@ -4,7 +4,6 @@
 
 from CIME.XML.standard_module_setup import *
 from CIME.utils import parse_args_and_handle_standard_logging_options, setup_standard_logging_options
-from CIME.case  import Case
 import sys, os, argparse, doctest
 
 logger = logging.getLogger(__name__)
@@ -34,67 +33,27 @@ def parse_input(argv):
 
     return args.caseroot, args.libroot, args.bldroot
 
-###############################################################################
-def build_data_lib(argv, compclass):
-###############################################################################
-
-    caseroot, libroot, _ = parse_input(argv)
+def build_cime_component_lib(case, compname, libroot, bldroot):
+    cimeroot  = case.get_value("CIMEROOT")
+    compclass = compname[1:]
 
-    with Case(caseroot) as case:
-
-        cimeroot  = case.get_value("CIMEROOT")
-
-        # Write directory list (Filepath)
-        compname = "d" + compclass
-        with open('Filepath', 'w') as out:
-            out.write(os.path.join(caseroot, "SourceMods", "src.{}\n".format(compname)) + "\n")
+    with open(os.path.join(bldroot,'Filepath'), 'w') as out:
+        out.write(os.path.join(case.get_value('CASEROOT'), "SourceMods",
+                               "src.{}\n".format(compname)) + "\n")
+        if compname.startswith('d'):
             out.write(os.path.join(cimeroot, "src", "components", "data_comps", compname) + "\n")
-
-        # Build the component
-        run_gmake(case, compclass, libroot)
-
-###############################################################################
-def build_xcpl_lib(argv, compclass):
-###############################################################################
-
-    caseroot, libroot, _ = parse_input(argv)
-
-    with Case(caseroot) as case:
-
-        cimeroot  = case.get_value("CIMEROOT")
-
-        # Write directory list
-        compname = "x" + compclass
-        with open('Filepath', 'w') as out:
-            out.write(os.path.join(caseroot, "SourceMods", "src.{}", compname) + "\n")
+        elif compname.startswith('x'):
             out.write(os.path.join(cimeroot, "src", "components", "xcpl_comps", "xshare") + "\n")
             out.write(os.path.join(cimeroot, "src", "components", "xcpl_comps",compname, "cpl") + "\n")
-
-        # Build the component
-        run_gmake(case, compclass, libroot)
-
-###############################################################################
-def build_stub_lib(argv, compclass):
-###############################################################################
-
-    caseroot, libroot, _ = parse_input(argv)
-
-    with Case(caseroot) as case:
-
-        cimeroot = case.get_value("CIMEROOT")
-
-        # Write directory list
-        compname = "s" + compclass
-        with open('Filepath', 'w') as out:
-            out.write(os.path.join(caseroot, "SourceMods", "src.{}", compname) + "\n")
+        elif compname.startswith('s'):
             out.write(os.path.join(cimeroot, "src", "components", "stub_comps", "xshare") + "\n")
             out.write(os.path.join(cimeroot, "src", "components", "stub_comps",compname, "cpl") + "\n")
 
-        # Build the component
-        run_gmake(case, compclass, libroot)
+    # Build the component
+    run_gmake(case, compclass, libroot, bldroot)
 
 ###############################################################################
-def run_gmake(case, compclass, libroot, libname="", user_cppdefs=""):
+def run_gmake(case, compclass, libroot, bldroot, libname="", user_cppdefs=""):
 ###############################################################################
 
     caseroot  = case.get_value("CASEROOT")
@@ -112,14 +71,12 @@ def run_gmake(case, compclass, libroot, libname="", user_cppdefs=""):
     makefile = os.path.join(casetools, "Makefile")
     macfile  = os.path.join(caseroot, "Macros.{}".format(mach))
 
+    cmd = "{} complib -j {:d} MODEL={} COMPLIB={} -f {} -C {} MACFILE={} " \
+        .format(gmake, gmake_j, compclass, complib, makefile, bldroot, macfile )
     if user_cppdefs:
-        cmd = "{} complib -j {:d} MODEL={} COMPLIB={} -f {} MACFILE={} USER_CPPDEFS='{}'" \
-            .format(gmake, gmake_j, compclass, complib, makefile, macfile, user_cppdefs )
-    else:
-        cmd = "{} complib -j {:d} MODEL={} COMPLIB={} -f {} MACFILE={} " \
-            .format(gmake, gmake_j, compclass, complib, makefile, macfile )
+        cmd = cmd + "USER_CPPDEFS='{}'".format(user_cppdefs )
 
     rc, out, err = run_cmd(cmd)
     expect(rc == 0, "Command {} failed rc={:d}\nout={}\nerr={}".format(cmd, rc, out, err))
 
-    logger.info("Command {} completed with output {}\nerr {}".format(cmd, out, err))
+    print "Command {} completed with output {}\nerr {}".format(cmd, out, err)

scripts/lib/CIME/preview_namelists.py

@@ -3,8 +3,8 @@
 """
 
 from CIME.XML.standard_module_setup import *
-
-import glob, shutil, imp
+from CIME.utils import run_sub_or_cmd
+import glob, shutil
 logger = logging.getLogger(__name__)
 
 def create_dirs(case):
@@ -74,36 +74,15 @@ def create_namelists(case, component=None):
             compname = case.get_value("COMP_{}".format(model_str.upper()))
 
         if component is None or component == model_str:
-            cmd = os.path.join(config_dir, "buildnml")
-            do_run_cmd = False
-            # This code will try to import and run each buildnml as a subroutine
-            # if that fails it will run it as a program in a seperate shell
-            try:
-                with open(cmd, 'r') as f:
-                    first_line = f.readline()
-                if "python" in first_line:
-                    mod = imp.load_source("buildnml", cmd)
-                    logger.info("   Calling {} buildnml".format(compname))
-                    mod.buildnml(case, caseroot, compname)
-                else:
-                    raise SyntaxError
-            except SyntaxError as detail:
-                if 'python' in first_line:
-                    expect(False, detail)
-                else:
-                    do_run_cmd = True
-            except AttributeError:
-                do_run_cmd = True
-            except:
-                raise
-
-            if do_run_cmd:
-                logger.info("   Running {} buildnml".format(compname))
-                case.flush()
-                output = run_cmd_no_fail("{} {}".format(cmd, caseroot), verbose=False)
-                logger.info(output)
-                # refresh case xml object from file
-                case.read_xml()
+            # first look in the case SourceMods directory
+            cmd = os.path.join(caseroot, "SourceMods", "src."+compname, "buildnml")
+            if os.path.isfile(cmd):
+                logger.warn("\nWARNING: Using local buildnml file {}\n".format(cmd))
+            else:
+                # otherwise look in the component config_dir
+                cmd = os.path.join(config_dir, "buildnml")
+            expect(os.path.isfile(cmd), "Could not find buildnml file for component {}".format(compname))
+            run_sub_or_cmd(cmd, (caseroot), "buildnml", (case, caseroot, compname), case=case)
 
     logger.info("Finished creating component namelists")
 

scripts/lib/CIME/utils.py

@@ -2,13 +2,33 @@
 Common functions used by cime python scripts
 Warning: you cannot use CIME Classes in this module as it causes circular dependencies
 """
-import logging, gzip, sys, os, time, re, shutil, glob, string, random
+import logging, gzip, sys, os, time, re, shutil, glob, string, random, imp
 import stat as statlib
 import warnings
+from contextlib import contextmanager
+
 # Return this error code if the scripts worked but tests failed
 TESTS_FAILED_ERR_CODE = 100
 logger = logging.getLogger(__name__)
 
+@contextmanager
+def redirect_stdout(new_target):
+    old_target, sys.stdout = sys.stdout, new_target # replace sys.stdout
+    try:
+        yield new_target # run some code with the replaced stdout
+    finally:
+        sys.stdout = old_target # restore to the previous value
+
+
+@contextmanager
+def redirect_stderr(new_target):
+    old_target, sys.stderr = sys.stderr, new_target # replace sys.stdout
+    try:
+        yield new_target # run some code with the replaced stdout
+    finally:
+        sys.stderr = old_target # restore to the previous value
+
+
 def expect(condition, error_msg, exc_type=SystemExit, error_prefix="ERROR:"):
     """
     Similar to assert except doesn't generate an ugly stacktrace. Useful for
@@ -177,6 +197,46 @@ def _convert_to_fd(filearg, from_dir):
     return open(filearg, "a")
 
 _hack=object()
+
+def run_sub_or_cmd(cmd, cmdargs, subname, subargs, logfile=None, case=None,
+                   input_str=None, from_dir=None, verbose=None,
+                   arg_stdout=_hack, arg_stderr=_hack, env=None, combine_output=False):
+
+    # This code will try to import and run each buildnml as a subroutine
+    # if that fails it will run it as a program in a seperate shell
+    do_run_cmd = False
+    stat = 0
+    output = ""
+    errput = ""
+    try:
+        mod = imp.load_source(subname, cmd)
+        logger.info("   Calling {}".format(cmd))
+        if logfile:
+            with redirect_stdout(open(logfile,"w")):
+                getattr(mod, subname)(*subargs)
+        else:
+            getattr(mod, subname)(*subargs)
+    except SyntaxError:
+        do_run_cmd = True
+    except AttributeError:
+        do_run_cmd = True
+    except:
+        raise
+
+    if do_run_cmd:
+        logger.info("   Running {} ".format(cmd))
+        if case is not None:
+            case.flush()
+        stat, output, errput = run_cmd("{} {}".format(cmd, cmdargs), input_str=input_str, from_dir=from_dir,
+                                 verbose=verbose, arg_stdout=arg_stdout, arg_stderr=arg_stderr, env=env,
+                                 combine_output=combine_output)
+
+        logger.info(output)
+        # refresh case xml object from file
+        if case is not None:
+            case.read_xml()
+    return stat, output, errput
+
 def run_cmd(cmd, input_str=None, from_dir=None, verbose=None,
             arg_stdout=_hack, arg_stderr=_hack, env=None, combine_output=False):
     """

src/build_scripts/buildlib.internal_components

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+"""
+build cime component model library.   This buildlib script is used by all cime internal
+components.
+"""
+
+import sys, os
+
+_CIMEROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..","..","..","..", "..")
+sys.path.append(os.path.join(_CIMEROOT, "scripts", "Tools"))
+
+from standard_script_setup import *
+from CIME.buildlib import build_cime_component_lib, parse_input
+from CIME.case import Case
+
+def buildlib(caseroot, libroot, bldroot, compname=None):
+    with Case(caseroot) as case:
+        if compname is None:
+            thisdir = os.path.dirname(os.path.abspath(__file__))
+            path, dir1 = os.path.split(thisdir)
+            _, dir2 = os.path.split(path)
+            if dir1 == "cime_config":
+                compname = dir2
+            else:
+                print "HERE %s"%dir1
+                compname = dir1.split('.')[1]
+        build_cime_component_lib(case, compname, libroot, bldroot)
+
+def _main_func(args):
+    caseroot, libroot, bldroot = parse_input(args)
+    buildlib(caseroot, libroot, bldroot)
+
+
+if __name__ == "__main__":
+    _main_func(sys.argv)

src/components/data_comps/datm/cime_config/buildlib

@@ -0,0 +1 @@
+../../../../build_scripts/buildlib.internal_components