Merge pull request #1140 from jedwards4b/cleanbld_refactor

Make case.build clean options clean more thoroughly.

Test suite: scripts_regression_tests.py
Test baseline:
Test namelist changes:
Test status: bit for bit

Fixes #1138

User interface changes?:

Code review: Goldy

README.unit_testing

@@ -1,3 +1,10 @@
 # To run all CIME unit tests on caldera, run the following command:
 # (Note that this must be done from an interactive caldera session, not from yellowstone)
+# Note also that this requires module load all-python-libs
+#
+# We would encourage you to port these tests to other platforms.
+# The test requires an install of pFunit available from
+# https://sourceforge.net/projects/pfunit/
+#
+
 tools/unit_testing/run_tests.py --test-spec-dir=. --compiler=intel --mpilib=mpich2 --use-openmp --mpirun-command=mpirun.lsf

cime_config/cesm/machines/Makefile

@@ -799,38 +799,38 @@ endif
 
 cleanatm:
 	$(RM) -f $(LIBROOT)/libatm.a
-	cd $(EXEROOT)/atm/obj;  $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/atm/obj
 
 cleancpl:
-	cd $(EXEROOT)/cpl/obj;  $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/cpl/obj
 
 cleanocn:
 	$(RM) -f $(LIBROOT)/libocn.a
-	cd $(EXEROOT)/ocn/obj ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/ocn/obj
 
 cleanwav:
 	$(RM) -f $(LIBROOT)/libwav.a
-	cd $(EXEROOT)/wav/obj ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/wav/obj
 
 cleanglc:
 	$(RM) -f $(LIBROOT)/libglc.a
 	$(RM) -fr $(EXEROOT)/glc
 
 cleanesp:
 	$(RM) -f $(LIBROOT)/libesp.a
-	cd $(EXEROOT)/esp/obj ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/esp/obj
 
 cleanice:
 	$(RM) -f $(LIBROOT)/libice.a
-	cd $(EXEROOT)/ice/obj ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/ice/obj
 
 cleanrof:
 	$(RM) -f $(LIBROOT)/librof.a
-	cd $(EXEROOT)/rof/obj ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(EXEROOT)/rof/obj
 
 cleanlnd:
 	$(RM) -f $(LNDLIBDIR)/$(LNDLIB)
-	cd $(LNDOBJDIR) ; $(RM) -f *.o *.$(MOD_SUFFIX)
+	$(RM) -fr $(LNDOBJDIR)
 
 cleancsmshare:
 	$(RM) -f $(CSMSHARELIB)

cime_config/cesm/machines/config_machines.xml

@@ -1699,6 +1699,7 @@
       <modules>
 	<command name="load">ncarcompilers/1.0</command>
 	<command name="load">cmake/3.0.2</command>
+	<command name="load">all-python-libs</command>
       </modules>
     </module_system>
     <environment_variables>

externals/pio1/pio/CMakeLists.txt

@@ -81,7 +81,7 @@ IF(NetCDF_Fortran_FOUND)
 ELSE()
   SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_NONETCDF)
 ENDIF()
-IF(PnetCDF_Fortran_FOUND)
+IF(PnetCDF_C_FOUND)
   SET(pio_include_dirs_ ${pio_include_dirs_} ${PNetCDF_INCLUDE_DIR})
   SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_PNETCDF)
 ELSE()

scripts/Tools/case.build

@@ -17,7 +17,7 @@ from CIME.test_status     import *
 def parse_command_line(args, description):
 ###############################################################################
     parser = argparse.ArgumentParser(
-        usage="""\n%s [--verbose]
+        usage="""\n%s [--verbose] [--clean [atm lnd pio ...]] [--clean-all]
 OR
 %s --help
 OR
@@ -51,20 +51,18 @@ OR
 
     parser.add_argument("--clean", nargs="*", choices=allobjs,
                         help="objects to clean"
-                        "if no arguments then clean all objects other than csmshare, mct, pio, and gptl")
+                        "if no arguments then clean all objects other than sharedlib objects")
 
     parser.add_argument("--clean-all", action="store_true",
-                        help="clean all objects ")
+                        help="clean all objects including sharedlibobjects that may be used by other builds")
 
     args = parser.parse_args(args[1:])
 
-    cleanlist = allobjs if args.clean_all else []
-    if args.clean is not None:
-        cleanlist = args.clean if args.clean else comps
-
     CIME.utils.handle_standard_logging_options(args)
 
-    return args.caseroot, args.sharedlib_only, args.model_only, cleanlist
+    cleanlist = args.clean if args.clean is None or len(args.clean) else comps
+
+    return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all
 
 ###############################################################################
 def _main_func(description):
@@ -73,15 +71,14 @@ def _main_func(description):
         test_results = doctest.testmod(verbose=True)
         sys.exit(1 if test_results.failed > 0 else 0)
 
-    caseroot, sharedlib_only, model_only, cleanlist = parse_command_line(sys.argv, description)
-    logging.info("calling build.case_build with caseroot=%s" %caseroot)
+    caseroot, sharedlib_only, model_only, cleanlist, clean_all = parse_command_line(sys.argv, description)
 
     success = True
     with Case(caseroot, read_only=False) as case:
         testname = case.get_value('TESTCASE')
 
-        if cleanlist:
-            build.clean(case, cleanlist)
+        if cleanlist is not None or clean_all:
+            build.clean(case, cleanlist, clean_all)
         elif(testname is not None):
             logging.warn("Building test for %s in directory %s" %
                          (testname, caseroot))

utils/python/CIME/SystemTests/system_tests_common.py

@@ -118,6 +118,8 @@ def build_indv(self, sharedlib_only=False, model_only=False):
                          sharedlib_only=sharedlib_only, model_only=model_only)
 
     def clean_build(self, comps=None):
+        if comps is None:
+            comps = [x.lower() for x in self._case.get_values("COMP_CLASSES")]
         build.clean(self._case, cleanlist=comps)
 
     def run(self):

utils/python/CIME/build.py

@@ -93,6 +93,8 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist,
     config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config")
     f = open(file_build, "w")
     bldroot = os.path.join(exeroot, "cpl", "obj")
+    if not os.path.isdir(bldroot):
+        os.makedirs(bldroot)
     stat = run_cmd("%s/buildexe %s %s %s" %
                    (config_dir, caseroot, libroot, bldroot),
                    from_dir=bldroot, verbose=True, arg_stdout=f,
@@ -503,43 +505,47 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro
     for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")):
         shutil.copy(mod_file, incroot)
 
+
 ###############################################################################
-def clean(case, cleanlist=None):
+def clean(case, cleanlist=None, clean_all=False):
 ###############################################################################
+    caseroot = case.get_value("CASEROOT")
+    if clean_all:
+        # If cleanlist is empty just remove the bld directory
+        exeroot = case.get_value("EXEROOT")
+        expect(exeroot is not None,"No EXEROOT defined in case")
+        if os.path.isdir(exeroot):
+            logging.info("cleaning directory %s" %exeroot)
+            shutil.rmtree(exeroot)
+        # if clean_all is True also remove the sharedlibpath
+        sharedlibroot = case.get_value("SHAREDLIBROOT")
+        expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case")
+        if sharedlibroot != exeroot and os.path.isdir(sharedlibroot):
+            logging.warn("cleaning directory %s" %sharedlibroot)
+            shutil.rmtree(sharedlibroot)
+    else:
+        expect(cleanlist is not None and len(cleanlist) > 0,"Empty cleanlist not expected")
+        debug           = case.get_value("DEBUG")
+        use_esmf_lib    = case.get_value("USE_ESMF_LIB")
+        build_threaded  = case.get_value("BUILD_THREADED")
+        gmake           = case.get_value("GMAKE")
+        caseroot        = case.get_value("CASEROOT")
+        casetools       = case.get_value("CASETOOLS")
+        clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
 
-    clm_config_opts = case.get_value("CLM_CONFIG_OPTS")
-    comp_lnd = case.get_value("COMP_LND")
-    if cleanlist is None:
-        cleanlist = case.get_values("COMP_CLASSES")
-        cleanlist = [x.lower().replace('drv','cpl') for x in cleanlist]
-        testcase        = case.get_value("TESTCASE")
-        # we only want to clean clm here if it is clm4_0 otherwise remove
-        # it from the cleanlist
-        if testcase is not None and comp_lnd == "clm" and \
-                clm_config_opts is not None and "lnd" in cleanlist and \
-                "clm4_0" not in clm_config_opts:
-            cleanlist.remove('lnd')
-
-    debug           = case.get_value("DEBUG")
-    use_esmf_lib    = case.get_value("USE_ESMF_LIB")
-    build_threaded  = case.get_value("BUILD_THREADED")
-    gmake           = case.get_value("GMAKE")
-    caseroot        = case.get_value("CASEROOT")
-    casetools       = case.get_value("CASETOOLS")
-
-    os.environ["DEBUG"]           = stringify_bool(debug)
-    os.environ["USE_ESMF_LIB"]    = stringify_bool(use_esmf_lib)
-    os.environ["BUILD_THREADED"]  = stringify_bool(build_threaded)
-    os.environ["CASEROOT"]        = case.get_value("CASEROOT")
-    os.environ["COMP_INTERFACE"]  = case.get_value("COMP_INTERFACE")
-    os.environ["PIO_VERSION"]     = str(case.get_value("PIO_VERSION"))
-    os.environ["CLM_CONFIG_OPTS"] = clm_config_opts  if clm_config_opts is not None else ""
-
-    cmd = gmake + " -f " + casetools + "/Makefile"
-    for item in cleanlist:
-        cmd = cmd + " clean" + item
-    logger.info("calling %s "%(cmd))
-    run_cmd_no_fail(cmd)
+        os.environ["DEBUG"]           = stringify_bool(debug)
+        os.environ["USE_ESMF_LIB"]    = stringify_bool(use_esmf_lib)
+        os.environ["BUILD_THREADED"]  = stringify_bool(build_threaded)
+        os.environ["CASEROOT"]        = caseroot
+        os.environ["COMP_INTERFACE"]  = case.get_value("COMP_INTERFACE")
+        os.environ["PIO_VERSION"]     = str(case.get_value("PIO_VERSION"))
+        os.environ["CLM_CONFIG_OPTS"] = clm_config_opts  if clm_config_opts is not None else ""
+
+        cmd = gmake + " -f " + os.path.join(casetools, "Makefile")
+        for item in cleanlist:
+            tcmd = cmd + " clean" + item
+            logger.info("calling %s "%(tcmd))
+            run_cmd_no_fail(tcmd)
 
     # unlink Locked files directory
     unlock_file("env_build.xml")
@@ -552,5 +558,9 @@ def clean(case, cleanlist=None):
     case.flush()
 
     # append call of to CaseStatus
-    msg = "cleanbuild %s "%" ".join(cleanlist)
+    if cleanlist:
+        msg = "clean %s "%" ".join(cleanlist)
+    elif clean_all:
+        msg = "clean_all"
+
     append_status(msg, caseroot=caseroot, sfile="CaseStatus")