From cb80ee9aca7b62481df242f5df9ac79dad4f33ec Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 8 Sep 2022 11:22:29 -0600 Subject: [PATCH] Feature dtcenter/METplus-Internal#32 add timestamp to final conf and file_lists (#1763) --- .github/parm/test_settings.conf | 2 ++ internal_tests/pytests/minimum_pytest.conf | 5 ++-- metplus/util/config_metplus.py | 33 ++++------------------ metplus/util/diff_util.py | 8 +++--- metplus/wrappers/command_builder.py | 5 ++-- parm/metplus_config/defaults.conf | 5 ++-- 6 files changed, 19 insertions(+), 39 deletions(-) diff --git a/.github/parm/test_settings.conf b/.github/parm/test_settings.conf index 01cbc7f58d..956c7e48aa 100644 --- a/.github/parm/test_settings.conf +++ b/.github/parm/test_settings.conf @@ -8,6 +8,8 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} LOG_METPLUS = {LOG_DIR}/metplus.log LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists # also set path to GempakToCF.jar for GEMPAK use cases GEMPAKTOCF_JAR = /data/input/GempakToCF.jar diff --git a/internal_tests/pytests/minimum_pytest.conf b/internal_tests/pytests/minimum_pytest.conf index d7bf430143..5a68934956 100644 --- a/internal_tests/pytests/minimum_pytest.conf +++ b/internal_tests/pytests/minimum_pytest.conf @@ -1,10 +1,9 @@ -[dir] +[config] INPUT_BASE = {ENV[METPLUS_TEST_INPUT_BASE]} OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]} MET_INSTALL_DIR = {ENV[METPLUS_TEST_MET_INSTALL_DIR]} TMP_DIR = {ENV[METPLUS_TEST_TMP_DIR]} -[config] LOG_LEVEL = DEBUG LOG_MET_OUTPUT_TO_METPLUS = no LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s @@ -14,3 +13,5 @@ LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} LOG_METPLUS = {LOG_DIR}/metplus.log LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists \ No newline at end of file diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 5ada71c0f5..316a8c5349 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -293,6 +293,10 @@ def _set_logvars(config, logger=None): log_filenametimestamp = date_t.strftime(log_timestamp_template) + # Adding LOG_TIMESTAMP to the final configuration file. + logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) + config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) + log_dir = config.getdir('LOG_DIR') # NOTE: LOG_METPLUS or metpluslog is meant to include the absolute path @@ -302,6 +306,7 @@ def _set_logvars(config, logger=None): # if LOG_METPLUS = unset in the conf file, means NO logging. # Also, assUmes the user has included the intended path in LOG_METPLUS. user_defined_log_file = None + metpluslog = '' if config.has_option('config', 'LOG_METPLUS'): user_defined_log_file = True # strinterp will set metpluslog to '' if LOG_METPLUS = is unset. @@ -316,34 +321,6 @@ def _set_logvars(config, logger=None): if metpluslog: if os.path.basename(metpluslog) == metpluslog: metpluslog = os.path.join(log_dir, metpluslog) - else: - # No LOG_METPLUS in conf file, so let the code try to set it, - # if the user defined the variable LOG_FILENAME_TEMPLATE. - # LOG_FILENAME_TEMPLATE is an 'unpublished' variable - no one knows - # about it unless you are reading this. Why does this exist ? - # It was from my first cycle implementation. I did not want to pull - # it out, in case the group wanted a stand alone metplus log filename - # template variable. - - # If metpluslog_filename includes a path, python joins it intelligently - # Set the metplus log filename. - # strinterp will set metpluslog_filename to '' if template is empty - if config.has_option('config', 'LOG_FILENAME_TEMPLATE'): - metpluslog_filename = config.strinterp( - 'config', - '{LOG_FILENAME_TEMPLATE}', - LOG_TIMESTAMP_TEMPLATE=log_filenametimestamp - ) - else: - metpluslog_filename = '' - if metpluslog_filename: - metpluslog = os.path.join(log_dir, metpluslog_filename) - else: - metpluslog = '' - - # Adding LOG_TIMESTAMP to the final configuration file. - logger.info('Adding LOG_TIMESTAMP=%s' % repr(log_filenametimestamp)) - config.set('config', 'LOG_TIMESTAMP', log_filenametimestamp) # Setting LOG_METPLUS in the configuration object # At this point LOG_METPLUS will have a value or '' the empty string. diff --git a/metplus/util/diff_util.py b/metplus/util/diff_util.py index 009939333b..d6625454cd 100644 --- a/metplus/util/diff_util.py +++ b/metplus/util/diff_util.py @@ -78,8 +78,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): if not os.path.isfile(filepath_a): continue - # skip metplus_final.conf - if filepath_a.endswith('metplus_final.conf'): + # skip final conf file + if 'metplus_final.conf' in os.path.basename(filepath_a): continue filepath_b = filepath_a.replace(dir_a, dir_b) @@ -109,8 +109,8 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): for filename in files: filepath_b = os.path.join(root, filename) - # skip metplus_final.conf - if filepath_b.endswith('metplus_final.conf'): + # skip final conf file + if 'metplus_final.conf' in os.path.basename(filepath_b): continue filepath_a = filepath_b.replace(dir_b, dir_a) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 221f164330..f16dc01eff 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -901,12 +901,11 @@ def write_list_file(self, filename, file_list, output_dir=None): @param filename name of ascii file to write @param file_list list of files to write to ascii file @param output_dir (Optional) directory to write files. If None, - ascii files are written to {STAGING_DIR}/file_lists + ascii files are written to {FILE_LIST_DIR} @returns path to output file """ if output_dir is None: - list_dir = os.path.join(self.config.getdir('STAGING_DIR'), - 'file_lists') + list_dir = self.config.getdir('FILE_LISTS_DIR') else: list_dir = output_dir diff --git a/parm/metplus_config/defaults.conf b/parm/metplus_config/defaults.conf index 2122f41e34..d6bb256d39 100644 --- a/parm/metplus_config/defaults.conf +++ b/parm/metplus_config/defaults.conf @@ -32,10 +32,11 @@ INPUT_BASE = /path/to OUTPUT_BASE = /path/to -METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf.{LOG_TIMESTAMP} TMP_DIR = {OUTPUT_BASE}/tmp STAGING_DIR = {OUTPUT_BASE}/stage +FILE_LISTS_DIR = {STAGING_DIR}/file_lists.{LOG_TIMESTAMP} ############################################################################### @@ -75,7 +76,7 @@ OMP_NUM_THREADS = 1 ############################################################################### -LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP_TEMPLATE} +LOG_METPLUS = {LOG_DIR}/metplus.log.{LOG_TIMESTAMP} LOG_DIR = {OUTPUT_BASE}/logs