From dd877199bac4d5ca36bcd9231189589ad82934d1 Mon Sep 17 00:00:00 2001 From: kdraeder Date: Mon, 18 Sep 2017 14:22:53 -0600 Subject: [PATCH 1/5] Committing all changes, may want to undo some. Just for testing. --- scripts/lib/CIME/case_st_archive.py | 47 +++++++++++++++++------------ 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index c997cbeb107..93b4860aed3 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -121,27 +121,36 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, temp_rpointer_content = rpointer_content # put in a temporary setting for ninst_strings if they are empty # in order to have just one loop over ninst_strings below - if rpointer_content is not 'unset': + # KDR 'is not' queries whether they are the same object. + # We want to compare the strings/contents of the objects + # if rpointer_content is not 'unset': + if rpointer_content != 'unset': + # KDR debugging why rpointer.unset are being created. + logger.info("rpointer_content not unset? {}".format(rpointer_content)) if not ninst_strings: ninst_strings = ["empty"] - for ninst_string in ninst_strings: - rpointer_file = temp_rpointer_file - rpointer_content = temp_rpointer_content - if ninst_string == 'empty': - ninst_string = "" - for key, value in [('$CASE', casename), - ('$DATENAME', datename), - ('$NINST_STRING', ninst_string)]: - rpointer_file = rpointer_file.replace(key, value) - rpointer_content = rpointer_content.replace(key, value) - - # write out the respect files with the correct contents - rpointer_file = os.path.join(archive_restdir, rpointer_file) - logger.info("writing rpointer_file {}".format(rpointer_file)) - f = open(rpointer_file, 'w') - for output in rpointer_content.split(','): - f.write("{} \n".format(output)) - f.close() + # KDR indenting all the rest 4 spaces to put under control of 'if ... unset' + for ninst_string in ninst_strings: + rpointer_file = temp_rpointer_file + rpointer_content = temp_rpointer_content + if ninst_string == 'empty': + ninst_string = "" + for key, value in [('$CASE', casename), + ('$DATENAME', datename), + ('$NINST_STRING', ninst_string)]: + rpointer_file = rpointer_file.replace(key, value) + rpointer_content = rpointer_content.replace(key, value) + + # write out the respect files with the correct contents + rpointer_file = os.path.join(archive_restdir, rpointer_file) + logger.info("writing rpointer_file {}".format(rpointer_file)) + f = open(rpointer_file, 'w') + for output in rpointer_content.split(','): + f.write("{} \n".format(output)) + f.close() + else: + # KDR debugging why rpointer.unset are being created. + logger.info("rpointer_content unset {}".format(rpointer_content)) ############################################################################### From 1c2446f1ac78f63fbfb2367973c0aa4bbdb3af25 Mon Sep 17 00:00:00 2001 From: kdraeder Date: Fri, 22 Sep 2017 15:51:19 -0600 Subject: [PATCH 2/5] Updating scripts/lib/CIME/case_st_archive.py > _get_ninst_info; comment out else: ninst_strings.append('') Appends 2 single quotes, instead of nothing as intended. > _get_component_archive_entries: add log info about components that are skipped. > _archive_rpointer_files: debugging statements in can eventually be removed. > _archive_history_files, _archive_restart_date_comps: changed handling of 'dart' compname to 'esp' and use fact that DART output, which st_archive should archive, now all uses standard CESM naming conventions. Updated logger information. --- scripts/lib/CIME/case_st_archive.py | 38 ++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 93b4860aed3..8ff2c7b2bf7 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -67,8 +67,10 @@ def _get_ninst_info(case, compclass): for i in range(1,ninst+1): if ninst > 1: ninst_strings.append('_' + '{:04d}'.format(i)) - else: - ninst_strings.append('') +# KDR This doesn't do what's intended. The result is ninst_strings is not empty. +# It has "''" in it. +## else: +## ninst_strings.append('') logger.debug("ninst and ninst_strings are: {} and {} for {}".format(ninst, ninst_strings, compclass)) return ninst, ninst_strings @@ -77,7 +79,7 @@ def _get_ninst_info(case, compclass): def _get_component_archive_entries(case, archive): ############################################################################### """ - Each time this is generator function is called, it yields a tuple + Each time this generator function is called, it yields a tuple (archive_entry, compname, compclass) for one component in this case's compset components. """ @@ -90,6 +92,10 @@ def _get_component_archive_entries(case, archive): if archive_entry is not None: yield(archive_entry, compname, archive_entry.get("compclass")) + if compname not in compset_comps: + # KDR Say if something is being skipped + logger.info('Skipping compname %s' %(compname)) + ############################################################################### def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, datename, datename_is_last): @@ -129,7 +135,7 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, logger.info("rpointer_content not unset? {}".format(rpointer_content)) if not ninst_strings: ninst_strings = ["empty"] - # KDR indenting all the rest 4 spaces to put under control of 'if ... unset' + # KDR indented 4 spaces, up to 'else:' to put under control of 'if ... unset' for ninst_string in ninst_strings: rpointer_file = temp_rpointer_file rpointer_content = temp_rpointer_content @@ -201,10 +207,13 @@ def _archive_history_files(case, archive, archive_entry, rundir = case.get_value("RUNDIR") for suffix in archive.get_hist_file_extensions(archive_entry): for i in range(ninst): - if compname == 'dart': - newsuffix = casename + suffix - elif compname.find('mpas') == 0: - newsuffix = compname + '.*' + suffix + if compname.find('mpas') == 0: + newsuffix = compname + '.*' + suffix +# KDR I've chosen to use compname 'cam' for DART output from CAM assims, +# inside compclass 'esp', to distinguish it from other components in multi-component assims. +# All CAM+DART output conforms to CESM naming conventions (2017-9-1). + elif ninst_string: + newsuffix = casename + '.' + compname + ".*" + ninst_string[i] + suffix else: if ninst_string: newsuffix = casename + '.' + compname + ".*" + ninst_string[i] + suffix @@ -336,6 +345,11 @@ def _archive_restarts_date_comp(case, archive, archive_entry, pattern = compname + suffix + '_'.join(datename.rsplit('-', 1)) pfile = re.compile(pattern) restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] +# KDR CAM+DART doesn't need special treatment as of 2017-9-1, so this is commented out. +# if compclass == 'esp' and 'cam' in pattern: +# pattern = suffix + datename +# pfile = re.compile(pattern) +# restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] else: pattern = r"{}\.{}\d*.*".format(casename, compname) if "dart" not in pattern: @@ -373,21 +387,21 @@ def _archive_restarts_date_comp(case, archive, archive_entry, srcfile = os.path.join(rundir, restfile) destfile = os.path.join(archive_restdir, restfile) last_restart_file_fn(srcfile, destfile) - logger.info("{} \n{} to \n{}".format( - last_restart_file_fn_msg, srcfile, destfile)) + logger.info("{} {} \n{} to \n{}".format( + "datename_is_last", last_restart_file_fn_msg, srcfile, destfile)) for histfile in histfiles_for_restart: srcfile = os.path.join(rundir, histfile) destfile = os.path.join(archive_restdir, histfile) expect(os.path.isfile(srcfile), "restart file {} does not exist ".format(srcfile)) shutil.copy(srcfile, destfile) - logger.info("copying \n{} to \n{}".format(srcfile, destfile)) + logger.info("datename_is_last + histfiles_for_restart copying \n{} to \n{}".format(srcfile, destfile)) else: # Only archive intermediate restarts if requested - otherwise remove them if case.get_value('DOUT_S_SAVE_INTERIM_RESTART_FILES'): srcfile = os.path.join(rundir, restfile) destfile = os.path.join(archive_restdir, restfile) - logger.info("moving \n{} to \n{}".format(srcfile, destfile)) +# KDR redundant? logger.info("moving \n{} to \n{}".format(srcfile, destfile)) expect(os.path.isfile(srcfile), "restart file {} does not exist ".format(srcfile)) archive_file_fn(srcfile, destfile) From 5eef51036a1f62c46b5f9deb74a2b327fd7ebc12 Mon Sep 17 00:00:00 2001 From: kdraeder Date: Tue, 26 Sep 2017 14:02:30 -0600 Subject: [PATCH 3/5] resolved merge conflicts. Also responded to comments --- scripts/lib/CIME/case_setup.py | 1 - scripts/lib/CIME/case_st_archive.py | 25 ++++++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/lib/CIME/case_setup.py b/scripts/lib/CIME/case_setup.py index 085395009c9..7365e984d5b 100644 --- a/scripts/lib/CIME/case_setup.py +++ b/scripts/lib/CIME/case_setup.py @@ -133,7 +133,6 @@ def _case_setup_impl(case, caseroot, clean=False, test_mode=False, reset=False): if comp == "CPL": continue ninst = case.get_value("NINST_{}".format(comp)) - # But the NINST_LAYOUT may only be concurrent in multi_driver mode if multi_driver: expect(case.get_value("NINST_LAYOUT_{}".format(comp)) == "concurrent", "If multi_driver is TRUE, NINST_LAYOUT_{} must be concurrent".format(comp)) diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 8ff2c7b2bf7..dd3c549302e 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -93,8 +93,7 @@ def _get_component_archive_entries(case, archive): yield(archive_entry, compname, archive_entry.get("compclass")) if compname not in compset_comps: - # KDR Say if something is being skipped - logger.info('Skipping compname %s' %(compname)) + logger.debug('Skipping compname %s; it is not in compset_comps' %(compname)) ############################################################################### def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, @@ -207,18 +206,18 @@ def _archive_history_files(case, archive, archive_entry, rundir = case.get_value("RUNDIR") for suffix in archive.get_hist_file_extensions(archive_entry): for i in range(ninst): - if compname.find('mpas') == 0: - newsuffix = compname + '.*' + suffix -# KDR I've chosen to use compname 'cam' for DART output from CAM assims, -# inside compclass 'esp', to distinguish it from other components in multi-component assims. -# All CAM+DART output conforms to CESM naming conventions (2017-9-1). - elif ninst_string: - newsuffix = casename + '.' + compname + ".*" + ninst_string[i] + suffix + if ninst_string: + if compname.find('mpas') == 0: + # Not correct, but MPAS' multi-instance name format is unknown. + newsuffix = compname + '.*' + suffix + else: + newsuffix = casename + '.' + compname + ".*" + ninst_string[i] + suffix else: - if ninst_string: - newsuffix = casename + '.' + compname + ".*" + ninst_string[i] + suffix - else: - newsuffix = casename + '.' + compname + ".*" + suffix + if compname.find('mpas') == 0: + newsuffix = compname + '.*' + suffix + else + newsuffix = casename + '.' + compname + ".*" + suffix + logger.debug("short term archiving suffix is {} ".format(newsuffix)) pfile = re.compile(newsuffix) histfiles = [f for f in os.listdir(rundir) if pfile.search(f)] From 7ac36e34db0c440427a12030817f3b8453011473 Mon Sep 17 00:00:00 2001 From: kdraeder Date: Wed, 27 Sep 2017 14:44:43 -0600 Subject: [PATCH 4/5] Re-introducing changes based on reviews of first push. There were 2 failures in scripts_regression_tests.py, but probably not related to case_st_archive.py changes. --- scripts/lib/CIME/case_st_archive.py | 47 +++++++---------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index dd3c549302e..62839fb8e88 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -67,10 +67,6 @@ def _get_ninst_info(case, compclass): for i in range(1,ninst+1): if ninst > 1: ninst_strings.append('_' + '{:04d}'.format(i)) -# KDR This doesn't do what's intended. The result is ninst_strings is not empty. -# It has "''" in it. -## else: -## ninst_strings.append('') logger.debug("ninst and ninst_strings are: {} and {} for {}".format(ninst, ninst_strings, compclass)) return ninst, ninst_strings @@ -126,15 +122,9 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, temp_rpointer_content = rpointer_content # put in a temporary setting for ninst_strings if they are empty # in order to have just one loop over ninst_strings below - # KDR 'is not' queries whether they are the same object. - # We want to compare the strings/contents of the objects - # if rpointer_content is not 'unset': if rpointer_content != 'unset': - # KDR debugging why rpointer.unset are being created. - logger.info("rpointer_content not unset? {}".format(rpointer_content)) if not ninst_strings: ninst_strings = ["empty"] - # KDR indented 4 spaces, up to 'else:' to put under control of 'if ... unset' for ninst_string in ninst_strings: rpointer_file = temp_rpointer_file rpointer_content = temp_rpointer_content @@ -145,18 +135,14 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, ('$NINST_STRING', ninst_string)]: rpointer_file = rpointer_file.replace(key, value) rpointer_content = rpointer_content.replace(key, value) - - # write out the respect files with the correct contents + + # write out the respective files with the correct contents rpointer_file = os.path.join(archive_restdir, rpointer_file) - logger.info("writing rpointer_file {}".format(rpointer_file)) + logger.debug("writing rpointer_file {}".format(rpointer_file)) f = open(rpointer_file, 'w') for output in rpointer_content.split(','): f.write("{} \n".format(output)) f.close() - else: - # KDR debugging why rpointer.unset are being created. - logger.info("rpointer_content unset {}".format(rpointer_content)) - ############################################################################### def _archive_log_files(case, archive_incomplete, archive_file_fn): @@ -215,7 +201,7 @@ def _archive_history_files(case, archive, archive_entry, else: if compname.find('mpas') == 0: newsuffix = compname + '.*' + suffix - else + else: newsuffix = casename + '.' + compname + ".*" + suffix logger.debug("short term archiving suffix is {} ".format(newsuffix)) @@ -344,28 +330,18 @@ def _archive_restarts_date_comp(case, archive, archive_entry, pattern = compname + suffix + '_'.join(datename.rsplit('-', 1)) pfile = re.compile(pattern) restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] -# KDR CAM+DART doesn't need special treatment as of 2017-9-1, so this is commented out. -# if compclass == 'esp' and 'cam' in pattern: -# pattern = suffix + datename -# pfile = re.compile(pattern) -# restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] else: pattern = r"{}\.{}\d*.*".format(casename, compname) - if "dart" not in pattern: + pfile = re.compile(pattern) + files = [f for f in os.listdir(rundir) if pfile.search(f)] + if ninst_strings: + pattern = ninst_strings[i] + suffix + datename pfile = re.compile(pattern) - files = [f for f in os.listdir(rundir) if pfile.search(f)] - if ninst_strings: - pattern = ninst_strings[i] + suffix + datename - pfile = re.compile(pattern) - restfiles = [f for f in files if pfile.search(f)] - else: - pattern = suffix + datename - pfile = re.compile(pattern) - restfiles = [f for f in files if pfile.search(f)] + restfiles = [f for f in files if pfile.search(f)] else: - pattern = suffix + pattern = suffix + datename pfile = re.compile(pattern) - restfiles = [f for f in os.listdir(rundir) if pfile.search(f)] + restfiles = [f for f in files if pfile.search(f)] for restfile in restfiles: restfile = os.path.basename(restfile) @@ -400,7 +376,6 @@ def _archive_restarts_date_comp(case, archive, archive_entry, if case.get_value('DOUT_S_SAVE_INTERIM_RESTART_FILES'): srcfile = os.path.join(rundir, restfile) destfile = os.path.join(archive_restdir, restfile) -# KDR redundant? logger.info("moving \n{} to \n{}".format(srcfile, destfile)) expect(os.path.isfile(srcfile), "restart file {} does not exist ".format(srcfile)) archive_file_fn(srcfile, destfile) From 8ea228a02fdf8e31bb3097950201ceb700a483c2 Mon Sep 17 00:00:00 2001 From: kdraeder Date: Thu, 28 Sep 2017 11:03:02 -0600 Subject: [PATCH 5/5] Removed outdated debugging statement and 2 lines of redundant code. --- scripts/lib/CIME/case_st_archive.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 62839fb8e88..487fadd3f6e 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -88,9 +88,6 @@ def _get_component_archive_entries(case, archive): if archive_entry is not None: yield(archive_entry, compname, archive_entry.get("compclass")) - if compname not in compset_comps: - logger.debug('Skipping compname %s; it is not in compset_comps' %(compname)) - ############################################################################### def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, datename, datename_is_last): @@ -336,12 +333,10 @@ def _archive_restarts_date_comp(case, archive, archive_entry, files = [f for f in os.listdir(rundir) if pfile.search(f)] if ninst_strings: pattern = ninst_strings[i] + suffix + datename - pfile = re.compile(pattern) - restfiles = [f for f in files if pfile.search(f)] else: pattern = suffix + datename - pfile = re.compile(pattern) - restfiles = [f for f in files if pfile.search(f)] + pfile = re.compile(pattern) + restfiles = [f for f in files if pfile.search(f)] for restfile in restfiles: restfile = os.path.basename(restfile)