From b61033818135e9bb00de3fde13f79fce607dacd5 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 12:03:29 -0700 Subject: [PATCH 1/6] refactor clean bld --- scripts/Tools/case.build | 19 ++++------ utils/python/CIME/build.py | 76 ++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 47 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 47601814527..394e8d3c656 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -17,7 +17,7 @@ from CIME.test_status import * def parse_command_line(args, description): ############################################################################### parser = argparse.ArgumentParser( - usage="""\n%s [--verbose] + usage="""\n%s [--verbose] [--clean [atm lnd pio ...]] [--clean-all] OR %s --help OR @@ -51,20 +51,16 @@ OR parser.add_argument("--clean", nargs="*", choices=allobjs, help="objects to clean" - "if no arguments then clean all objects other than csmshare, mct, pio, and gptl") + "if no arguments then clean all objects other than sharedlib objects") parser.add_argument("--clean-all", action="store_true", - help="clean all objects ") + help="clean all objects including sharedlibobjects that may be used by other builds") args = parser.parse_args(args[1:]) - cleanlist = allobjs if args.clean_all else [] - if args.clean is not None: - cleanlist = args.clean if args.clean else comps - CIME.utils.handle_standard_logging_options(args) - return args.caseroot, args.sharedlib_only, args.model_only, cleanlist + return args.caseroot, args.sharedlib_only, args.model_only, args.clean, args.clean_all ############################################################################### def _main_func(description): @@ -73,15 +69,14 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - caseroot, sharedlib_only, model_only, cleanlist = parse_command_line(sys.argv, description) - logging.info("calling build.case_build with caseroot=%s" %caseroot) + caseroot, sharedlib_only, model_only, cleanlist, clean_all = parse_command_line(sys.argv, description) success = True with Case(caseroot, read_only=False) as case: testname = case.get_value('TESTCASE') - if cleanlist: - build.clean(case, cleanlist) + if cleanlist is not None or clean_all: + build.clean(case, cleanlist, clean_all) elif(testname is not None): logging.warn("Building test for %s in directory %s" % (testname, caseroot)) diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index 7bb1038bae2..0e808ecfea8 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -504,42 +504,42 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro shutil.copy(mod_file, incroot) ############################################################################### -def clean(case, cleanlist=None): +def clean(case, cleanlist=None, clean_all=False): ############################################################################### + caseroot = case.get_value("CASEROOT") + if clean_all or cleanlist is not None and len(cleanlist) == 0: + # If cleanlist is empty just remove the bld directory + exeroot = case.get_value("EXEROOT") + if os.path.isdir(exeroot): + logging.info("removing directory %s" %exeroot) + shutil.rmtree(case.get_value("EXEROOT")) + # if clean_all is True also remove the sharedlibpath + sharedlibroot = case.get_value("SHAREDLIBROOT") + if clean_all and os.path.isdir(sharedlibroot): + logging.info("removing directory %s" %sharedlibroot) + shutil.rmtree(sharedlibroot) + else: + debug = case.get_value("DEBUG") + use_esmf_lib = case.get_value("USE_ESMF_LIB") + build_threaded = case.get_value("BUILD_THREADED") + gmake = case.get_value("GMAKE") + caseroot = case.get_value("CASEROOT") + casetools = case.get_value("CASETOOLS") + clm_config_opts = case.get_value("CLM_CONFIG_OPTS") + + os.environ["DEBUG"] = stringify_bool(debug) + os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) + os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) + os.environ["CASEROOT"] = caseroot + os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE") + os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) + os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" - clm_config_opts = case.get_value("CLM_CONFIG_OPTS") - comp_lnd = case.get_value("COMP_LND") - if cleanlist is None: - cleanlist = case.get_values("COMP_CLASSES") - cleanlist = [x.lower().replace('drv','cpl') for x in cleanlist] - testcase = case.get_value("TESTCASE") - # we only want to clean clm here if it is clm4_0 otherwise remove - # it from the cleanlist - if testcase is not None and comp_lnd == "clm" and \ - clm_config_opts is not None and "lnd" in cleanlist and \ - "clm4_0" not in clm_config_opts: - cleanlist.remove('lnd') - - debug = case.get_value("DEBUG") - use_esmf_lib = case.get_value("USE_ESMF_LIB") - build_threaded = case.get_value("BUILD_THREADED") - gmake = case.get_value("GMAKE") - caseroot = case.get_value("CASEROOT") - casetools = case.get_value("CASETOOLS") - - os.environ["DEBUG"] = stringify_bool(debug) - os.environ["USE_ESMF_LIB"] = stringify_bool(use_esmf_lib) - os.environ["BUILD_THREADED"] = stringify_bool(build_threaded) - os.environ["CASEROOT"] = case.get_value("CASEROOT") - os.environ["COMP_INTERFACE"] = case.get_value("COMP_INTERFACE") - os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) - os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" - - cmd = gmake + " -f " + casetools + "/Makefile" - for item in cleanlist: - cmd = cmd + " clean" + item - logger.info("calling %s "%(cmd)) - run_cmd_no_fail(cmd) + cmd = gmake + " -f " + casetools + "/Makefile" + for item in cleanlist: + cmd = cmd + " clean" + item + logger.info("calling %s "%(cmd)) + run_cmd_no_fail(cmd) # unlink Locked files directory unlock_file("env_build.xml") @@ -552,5 +552,11 @@ def clean(case, cleanlist=None): case.flush() # append call of to CaseStatus - msg = "cleanbuild %s "%" ".join(cleanlist) + if cleanlist: + msg = "clean %s "%" ".join(cleanlist) + elif clean_all: + msg = "clean_all" + else: + msg = "clean bld" + append_status(msg, caseroot=caseroot, sfile="CaseStatus") From 4a5076c627ae557acc7132b4a314d8d72c227eed Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 13:55:05 -0700 Subject: [PATCH 2/6] get clean working for tests --- scripts/Tools/case.build | 4 +++- .../CIME/SystemTests/system_tests_common.py | 2 +- utils/python/CIME/build.py | 23 ++++++++++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/scripts/Tools/case.build b/scripts/Tools/case.build index 394e8d3c656..3e94c4db0e6 100755 --- a/scripts/Tools/case.build +++ b/scripts/Tools/case.build @@ -60,7 +60,9 @@ OR CIME.utils.handle_standard_logging_options(args) - return args.caseroot, args.sharedlib_only, args.model_only, args.clean, args.clean_all + cleanlist = args.clean if args.clean is None or len(args.clean) else comps + + return args.caseroot, args.sharedlib_only, args.model_only, cleanlist, args.clean_all ############################################################################### def _main_func(description): diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index bdcec2261c9..c8a09c1c683 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -118,7 +118,7 @@ def build_indv(self, sharedlib_only=False, model_only=False): sharedlib_only=sharedlib_only, model_only=model_only) def clean_build(self, comps=None): - build.clean(self._case, cleanlist=comps) + build.clean(self._case, cleanlist=comps if comps is not None else []) def run(self): """ diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index 0e808ecfea8..23c0f60497e 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -503,21 +503,32 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): shutil.copy(mod_file, incroot) +def cleantree(base, ignoredirs=None): + """ + clean all files below base level, leave directories and top level files + """ + ignoredirs.append(base) + for root, _, files in os.walk(base, topdown=False): + if root not in ignoredirs: + for name in files: + os.remove(os.path.join(root, name)) + + ############################################################################### def clean(case, cleanlist=None, clean_all=False): ############################################################################### caseroot = case.get_value("CASEROOT") - if clean_all or cleanlist is not None and len(cleanlist) == 0: + if clean_all: # If cleanlist is empty just remove the bld directory exeroot = case.get_value("EXEROOT") if os.path.isdir(exeroot): - logging.info("removing directory %s" %exeroot) - shutil.rmtree(case.get_value("EXEROOT")) + logging.info("cleaning directory %s" %exeroot) + cleantree(exeroot, ignoredirs=[os.path.join(exeroot,"lib")]) # if clean_all is True also remove the sharedlibpath sharedlibroot = case.get_value("SHAREDLIBROOT") - if clean_all and os.path.isdir(sharedlibroot): - logging.info("removing directory %s" %sharedlibroot) - shutil.rmtree(sharedlibroot) + if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): + logging.warn("cleaning directory %s" %sharedlibroot) + cleantree(sharedlibroot) else: debug = case.get_value("DEBUG") use_esmf_lib = case.get_value("USE_ESMF_LIB") From 3782eb0cdad2e4b034874f16413ce489df5c23bd Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 18:24:33 -0700 Subject: [PATCH 3/6] response to review --- cime_config/cesm/machines/Makefile | 16 ++++++++-------- utils/python/CIME/build.py | 27 +++++++++------------------ 2 files changed, 17 insertions(+), 26 deletions(-) diff --git a/cime_config/cesm/machines/Makefile b/cime_config/cesm/machines/Makefile index 8760ba21e25..8b6216dafec 100644 --- a/cime_config/cesm/machines/Makefile +++ b/cime_config/cesm/machines/Makefile @@ -799,18 +799,18 @@ endif cleanatm: $(RM) -f $(LIBROOT)/libatm.a - cd $(EXEROOT)/atm/obj; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/atm/obj cleancpl: - cd $(EXEROOT)/cpl/obj; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/cpl/obj cleanocn: $(RM) -f $(LIBROOT)/libocn.a - cd $(EXEROOT)/ocn/obj ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/ocn/obj cleanwav: $(RM) -f $(LIBROOT)/libwav.a - cd $(EXEROOT)/wav/obj ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/wav/obj cleanglc: $(RM) -f $(LIBROOT)/libglc.a @@ -818,19 +818,19 @@ cleanglc: cleanesp: $(RM) -f $(LIBROOT)/libesp.a - cd $(EXEROOT)/esp/obj ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/esp/obj cleanice: $(RM) -f $(LIBROOT)/libice.a - cd $(EXEROOT)/ice/obj ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/ice/obj cleanrof: $(RM) -f $(LIBROOT)/librof.a - cd $(EXEROOT)/rof/obj ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(EXEROOT)/rof/obj cleanlnd: $(RM) -f $(LNDLIBDIR)/$(LNDLIB) - cd $(LNDOBJDIR) ; $(RM) -f *.o *.$(MOD_SUFFIX) + $(RM) -fr $(LNDOBJDIR) cleancsmshare: $(RM) -f $(CSMSHARELIB) diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index 23c0f60497e..586c79ffde1 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -503,16 +503,6 @@ def _build_model_thread(config_dir, compclass, caseroot, libroot, bldroot, incro for mod_file in glob.glob(os.path.join(bldroot, "*_[Cc][Oo][Mm][Pp]_*.mod")): shutil.copy(mod_file, incroot) -def cleantree(base, ignoredirs=None): - """ - clean all files below base level, leave directories and top level files - """ - ignoredirs.append(base) - for root, _, files in os.walk(base, topdown=False): - if root not in ignoredirs: - for name in files: - os.remove(os.path.join(root, name)) - ############################################################################### def clean(case, cleanlist=None, clean_all=False): @@ -521,15 +511,18 @@ def clean(case, cleanlist=None, clean_all=False): if clean_all: # If cleanlist is empty just remove the bld directory exeroot = case.get_value("EXEROOT") + expect(exeroot is not None,"No EXEROOT defined in case") if os.path.isdir(exeroot): logging.info("cleaning directory %s" %exeroot) - cleantree(exeroot, ignoredirs=[os.path.join(exeroot,"lib")]) + os.rmtree(exeroot) # if clean_all is True also remove the sharedlibpath sharedlibroot = case.get_value("SHAREDLIBROOT") + expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case") if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): logging.warn("cleaning directory %s" %sharedlibroot) - cleantree(sharedlibroot) + os.rmtree(sharedlibroot) else: + expect(cleanlist is not None and len(cleanlist) > 0,"Empty cleanlist not expected") debug = case.get_value("DEBUG") use_esmf_lib = case.get_value("USE_ESMF_LIB") build_threaded = case.get_value("BUILD_THREADED") @@ -546,11 +539,11 @@ def clean(case, cleanlist=None, clean_all=False): os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) os.environ["CLM_CONFIG_OPTS"] = clm_config_opts if clm_config_opts is not None else "" - cmd = gmake + " -f " + casetools + "/Makefile" + cmd = gmake + " -f " + os.path.join(casetools, "Makefile") for item in cleanlist: - cmd = cmd + " clean" + item - logger.info("calling %s "%(cmd)) - run_cmd_no_fail(cmd) + tcmd = cmd + " clean" + item + logger.info("calling %s "%(tcmd)) + run_cmd_no_fail(tcmd) # unlink Locked files directory unlock_file("env_build.xml") @@ -567,7 +560,5 @@ def clean(case, cleanlist=None, clean_all=False): msg = "clean %s "%" ".join(cleanlist) elif clean_all: msg = "clean_all" - else: - msg = "clean bld" append_status(msg, caseroot=caseroot, sfile="CaseStatus") From 00b7038addf5d04b9f4426cd3c6256780dd2410b Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 19:51:17 -0700 Subject: [PATCH 4/6] revert to rmtree for clean-all option --- cime_config/cesm/machines/config_machines.xml | 1 + utils/python/CIME/build.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cime_config/cesm/machines/config_machines.xml b/cime_config/cesm/machines/config_machines.xml index 2c71d349b6a..2a4a7987ce4 100644 --- a/cime_config/cesm/machines/config_machines.xml +++ b/cime_config/cesm/machines/config_machines.xml @@ -1699,6 +1699,7 @@ ncarcompilers/1.0 cmake/3.0.2 + all-python-libs diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index 586c79ffde1..2c341ebc879 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -514,13 +514,13 @@ def clean(case, cleanlist=None, clean_all=False): expect(exeroot is not None,"No EXEROOT defined in case") if os.path.isdir(exeroot): logging.info("cleaning directory %s" %exeroot) - os.rmtree(exeroot) + shutil.rmtree(exeroot) # if clean_all is True also remove the sharedlibpath sharedlibroot = case.get_value("SHAREDLIBROOT") expect(sharedlibroot is not None,"No SHAREDLIBROOT defined in case") if sharedlibroot != exeroot and os.path.isdir(sharedlibroot): logging.warn("cleaning directory %s" %sharedlibroot) - os.rmtree(sharedlibroot) + shutil.rmtree(sharedlibroot) else: expect(cleanlist is not None and len(cleanlist) > 0,"Empty cleanlist not expected") debug = case.get_value("DEBUG") From fa5ed4c804d871981af678a40c25cb68791679c4 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 19:58:49 -0700 Subject: [PATCH 5/6] update README.unit_testing; --- README.unit_testing | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.unit_testing b/README.unit_testing index af6e05f4949..ad3fa0be1f8 100644 --- a/README.unit_testing +++ b/README.unit_testing @@ -1,3 +1,10 @@ # To run all CIME unit tests on caldera, run the following command: # (Note that this must be done from an interactive caldera session, not from yellowstone) +# Note also that this requires module load all-python-libs +# +# We would encourage you to port these tests to other platforms. +# The test requires an install of pFunit available from +# https://sourceforge.net/projects/pfunit/ +# + tools/unit_testing/run_tests.py --test-spec-dir=. --compiler=intel --mpilib=mpich2 --use-openmp --mpirun-command=mpirun.lsf From 1497e8090c6eb35871ff46973d2155ff4632aa16 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Tue, 14 Feb 2017 21:15:19 -0700 Subject: [PATCH 6/6] fix issues with tests --- externals/pio1/pio/CMakeLists.txt | 2 +- utils/python/CIME/SystemTests/system_tests_common.py | 4 +++- utils/python/CIME/build.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/externals/pio1/pio/CMakeLists.txt b/externals/pio1/pio/CMakeLists.txt index e2f064a004e..c9869a2cc9d 100644 --- a/externals/pio1/pio/CMakeLists.txt +++ b/externals/pio1/pio/CMakeLists.txt @@ -81,7 +81,7 @@ IF(NetCDF_Fortran_FOUND) ELSE() SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_NONETCDF) ENDIF() -IF(PnetCDF_Fortran_FOUND) +IF(PnetCDF_C_FOUND) SET(pio_include_dirs_ ${pio_include_dirs_} ${PNetCDF_INCLUDE_DIR}) SET(bld_PIO_DEFINITIONS ${bld_PIO_DEFINITIONS} -D_PNETCDF) ELSE() diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index c8a09c1c683..4d808872789 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -118,7 +118,9 @@ def build_indv(self, sharedlib_only=False, model_only=False): sharedlib_only=sharedlib_only, model_only=model_only) def clean_build(self, comps=None): - build.clean(self._case, cleanlist=comps if comps is not None else []) + if comps is None: + comps = [x.lower() for x in self._case.get_values("COMP_CLASSES")] + build.clean(self._case, cleanlist=comps) def run(self): """ diff --git a/utils/python/CIME/build.py b/utils/python/CIME/build.py index 2c341ebc879..56151bcc2f6 100644 --- a/utils/python/CIME/build.py +++ b/utils/python/CIME/build.py @@ -93,6 +93,8 @@ def build_model(build_threaded, exeroot, clm_config_opts, incroot, complist, config_dir = os.path.join(cimeroot, "driver_cpl", "cime_config") f = open(file_build, "w") bldroot = os.path.join(exeroot, "cpl", "obj") + if not os.path.isdir(bldroot): + os.makedirs(bldroot) stat = run_cmd("%s/buildexe %s %s %s" % (config_dir, caseroot, libroot, bldroot), from_dir=bldroot, verbose=True, arg_stdout=f,