From 5a9d7a21ac2ca3bd8c6d582b558e9362aff44e90 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 5 Dec 2023 17:18:57 +0000 Subject: [PATCH 01/30] Draft replacements in scripts. --- scripts/exregional_make_grid.sh | 51 ++++++++++-------- scripts/exregional_make_ics.sh | 96 +++++++++++++++++---------------- scripts/exregional_make_lbcs.sh | 30 ++++++----- 3 files changed, 97 insertions(+), 80 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 88472a8343..6823c2581c 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -265,29 +265,36 @@ generation executable (exec_fp): # namelist file. # settings=" -'regional_grid_nml': { - 'plon': ${LON_CTR}, - 'plat': ${LAT_CTR}, - 'delx': ${DEL_ANGLE_X_SG}, - 'dely': ${DEL_ANGLE_Y_SG}, - 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO}, - 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO}, - 'pazi': ${PAZI}, - } +'regional_grid_nml': + 'plon': ${LON_CTR} + 'plat': ${LAT_CTR} + 'delx': ${DEL_ANGLE_X_SG} + 'dely': ${DEL_ANGLE_Y_SG} + 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO} + 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO} + 'pazi': ${PAZI} " -# -# Call the python script to create the namelist file. -# - ${USHdir}/set_namelist.py -q -u "$settings" -o ${rgnl_grid_nml_fp} || \ - print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the -regional_esg_grid namelist file failed. Parameters passed to this script -are: - Full path to output namelist file: - rgnl_grid_nml_fp = \"${rgnl_grid_nml_fp}\" - Namelist settings specified on command line (these have highest precedence): - settings = -$settings" + + # Store the settings in a temporary file + tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") + cat > $tmpfile << EOF +$settings +EOF + + uw config realize \ + -i ${tmpfile} \ + -o ${rgnl_grid_nml_fp} \ + -v \ + --values-file ${tmpfile} + + err=$? + if [ $err -ne 0 ]; then + rm $tmpfile + print_err_msg_exit "\ + Error creating regional_esg_grid namelist. + Contents of input are: + $settings" + fi # # Call the executable that generates the grid file. # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index d52bf25ec4..e161b69f16 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -555,62 +555,68 @@ fi # to remove it from the namelist! Which is better to use?? # settings=" -'config': { - 'fix_dir_target_grid': ${FIXlam}, - 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXlam}, - 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${VCOORD_FILE}, - 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, - 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm}, - 'sfc_files_input_grid': ${fn_sfc}, - 'grib2_file_input_grid': \"${fn_grib2}\", - 'cycle_mon': $((10#${mm})), - 'cycle_day': $((10#${dd})), - 'cycle_hour': $((10#${hh})), - 'convert_atm': True, - 'convert_sfc': True, - 'convert_nst': ${convert_nst}, - 'regional': 1, - 'halo_bndy': $((10#${NH4})), - 'halo_blend': $((10#${HALO_BLEND})), - 'input_type': ${input_type}, - 'external_model': ${external_model}, - 'tracers_input': ${tracers_input}, - 'tracers': ${tracers}, - 'nsoill_out': $((10#${nsoill_out})), - 'geogrid_file_input_grid': ${geogrid_file_input_grid}, - 'vgtyp_from_climo': ${vgtyp_from_climo}, - 'sotyp_from_climo': ${sotyp_from_climo}, - 'vgfrc_from_climo': ${vgfrc_from_climo}, - 'minmax_vgfrc_from_climo': ${minmax_vgfrc_from_climo}, - 'lai_from_climo': ${lai_from_climo}, - 'tg3_from_soil': ${tg3_from_soil}, - 'thomp_mp_climo_file': ${thomp_mp_climo_file}, -} +'config': + 'fix_dir_target_grid': ${FIXlam} + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc + 'orog_dir_target_grid': ${FIXlam} + 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc + 'vcoord_file_target_grid': ${VCOORD_FILE} + 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file} + 'data_dir_input_grid': ${extrn_mdl_staging_dir} + 'atm_files_input_grid': ${fn_atm} + 'sfc_files_input_grid': ${fn_sfc} + 'grib2_file_input_grid': \"${fn_grib2}\" + 'cycle_mon': $((10#${mm})) + 'cycle_day': $((10#${dd})) + 'cycle_hour': $((10#${hh})) + 'convert_atm': True + 'convert_sfc': True + 'convert_nst': ${convert_nst} + 'regional': 1 + 'halo_bndy': $((10#${NH4})) + 'halo_blend': $((10#${HALO_BLEND})) + 'input_type': ${input_type} + 'external_model': ${external_model} + 'tracers_input': ${tracers_input} + 'tracers': ${tracers} + 'nsoill_out': $((10#${nsoill_out})) + 'geogrid_file_input_grid': ${geogrid_file_input_grid} + 'vgtyp_from_climo': ${vgtyp_from_climo} + 'sotyp_from_climo': ${sotyp_from_climo} + 'vgfrc_from_climo': ${vgfrc_from_climo} + 'minmax_vgfrc_from_climo': ${minmax_vgfrc_from_climo} + 'lai_from_climo': ${lai_from_climo} + 'tg3_from_soil': ${tg3_from_soil} + 'thomp_mp_climo_file': ${thomp_mp_climo_file} " -# -# Call the python script to create the namelist file. -# + +# Store the settings in a temporary file +tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") +cat > $tmpfile << EOF +$settings +EOF + + nml_fn="fort.41" -${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} +uw config realize \ + -i ${tmpfile} \ + -o ${nml_fn} \ + -v \ + --values-file "${tmpfile}" + err=$? if [ $err -ne 0 ]; then - message_txt="Call to python script set_namelist.py to set the variables -in the namelist file read in by the ${exec_fn} executable failed. Parameters -passed to this script are: - Name of output namelist file: - nml_fn = \"${nml_fn}\" - Namelist settings specified on command line (these have highest precedence): - settings = + message_txt="Error creating namelist read by ${exec_fn} failed. + Contents of input are: $settings" + rm $tmpfile if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then err_exit "${message_txt}" else print_err_msg_exit "${message_txt}" fi fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 08cd0dda18..1666ec9723 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -476,7 +476,7 @@ FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_N # to remove it from the namelist! Which is better to use?? # settings=" -'config': { +'config': 'fix_dir_target_grid': ${FIXlam}, 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, 'orog_dir_target_grid': ${FIXlam}, @@ -498,23 +498,27 @@ settings=" 'tracers_input': ${tracers_input}, 'tracers': ${tracers}, 'thomp_mp_climo_file': ${thomp_mp_climo_file}, -} " -# -# Call the python script to create the namelist file. -# + + # Store the settings in a temporary file + tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") + cat > $tmpfile << EOF +$settings +EOF + nml_fn="fort.41" - ${USHdir}/set_namelist.py -q -u "$settings" -o ${nml_fn} + uw config realize \ + -i ${tmpfile} \ + -o ${nml_fn} \ + -v \ + --values-file ${tmpfile} + export err=$? if [ $err -ne 0 ]; then - message_txt="Call to python script set_namelist.py to set the variables -in the namelist file read in by the ${exec_fn} executable failed. Parameters -passed to this script are: - Name of output namelist file: - nml_fn = \"${nml_fn}\" - Namelist settings specified on command line (these have highest precedence): - settings = + message_txt="Error creating namelist read by ${exec_fn} failed. + Contents of input are: $settings" + rm $tmpfile if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then err_exit "${message_txt}" else From d135c68bc8e97cef446b50fc89e93ee866cb6181 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 5 Dec 2023 18:34:01 +0000 Subject: [PATCH 02/30] Replace calls to set_namelist with uw CLI. --- parm/FV3.input.yml | 7 +- scripts/exregional_make_ics.sh | 40 +-- ush/generate_FV3LAM_wflow.py | 57 +++-- ush/set_FV3nml_ens_stoch_seeds.py | 31 ++- ush/set_FV3nml_sfc_climo_filenames.py | 36 +-- ush/set_namelist.py | 355 -------------------------- ush/update_input_nml.py | 37 +-- 7 files changed, 110 insertions(+), 453 deletions(-) delete mode 100755 ush/set_namelist.py diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index cf71e957db..36334fcc2f 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -4,12 +4,7 @@ # parm/input.nml.FV3 # # to obtain the namelist for each physics suite that the SRW App can -# run with. To build a namelist for one of these configurations, use -# the Python helper script -# -# ush/set_namelist.py -# -# and provide this file and the desired section via the -c option. +# run with. FV3_RRFS_v1beta: diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index e161b69f16..6a358ba511 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -590,32 +590,32 @@ settings=" 'thomp_mp_climo_file': ${thomp_mp_climo_file} " -# Store the settings in a temporary file -tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") -cat > $tmpfile << EOF + # Store the settings in a temporary file + tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") + cat > $tmpfile << EOF $settings EOF -nml_fn="fort.41" -uw config realize \ - -i ${tmpfile} \ - -o ${nml_fn} \ - -v \ - --values-file "${tmpfile}" + nml_fn="fort.41" + uw config realize \ + -i ${tmpfile} \ + -o ${nml_fn} \ + -v \ + --values-file "${tmpfile}" -err=$? -if [ $err -ne 0 ]; then - message_txt="Error creating namelist read by ${exec_fn} failed. - Contents of input are: -$settings" - rm $tmpfile - if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then - err_exit "${message_txt}" - else - print_err_msg_exit "${message_txt}" + err=$? + if [ $err -ne 0 ]; then + message_txt="Error creating namelist read by ${exec_fn} failed. + Contents of input are: + $settings" + rm $tmpfile + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" + fi fi -fi # #----------------------------------------------------------------------- diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 516cbf94cb..24254ea771 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -19,6 +19,7 @@ export_vars, cp_vrfy, ln_vrfy, + load_config_file, mkdir_vrfy, mv_vrfy, create_symlink_to_file, @@ -31,7 +32,6 @@ from setup import setup from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames from get_crontab_contents import add_crontab_line -from set_namelist import set_namelist from check_python_version import check_python_version # These come from ush/python_utils/workflow-tools @@ -495,24 +495,28 @@ def generate_FV3LAM_wflow( # # ----------------------------------------------------------------------- # - # Call the set_namelist.py script to create a new FV3 namelist file (full - # path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as - # the base (i.e. starting) namelist file, with physics-suite-dependent - # modifications to the base file specified in the yaml configuration file - # FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), - # and with additional physics-suite-independent modifications specified - # in the variable "settings" set above. + # Create a new FV3 namelist file # # ----------------------------------------------------------------------- # - args=[ "-n", FV3_NML_BASE_SUITE_FP, - "-c", FV3_NML_YAML_CONFIG_FP, CCPP_PHYS_SUITE, - "-u", settings_str, - "-o", FV3_NML_FP, - ] - if not debug: - args.append("-q") - set_namelist(args) + + physics_cfg = load_config_file(FV3_NML_YAML_CONFIG_FP)[CCPP_PHYS_SUITE] + update_dict(settings, physics_cfg) + + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(physics_cfg)) + tmpfile.seek(0) + subprocess.run(["uw config realize", + "-i", FV3_NML_BASE_SUITE_FP, + "-o", FV3_NML_FP, + "-v", + "--values-file", tmpfile, + ] + ) # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -642,13 +646,20 @@ def generate_FV3LAM_wflow( # if any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): - args=[ "-n", FV3_NML_FP, - "-u", settings_str, - "-o", FV3_NML_STOCH_FP, - ] - if not debug: - args.append("-q") - set_namelist(args) + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(settings)) + tmpfile.seek(0) + subprocess.run(["uw config realize", + "-i", FV3_NML_FP, + "-o", FV3_NML_STOCH_FP, + "-v", + "--values-file", tmpfile, + ] + ) # # ----------------------------------------------------------------------- diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index c8a90e2797..2e190f292d 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -23,7 +23,6 @@ flatten_dict, ) -from set_namelist import set_namelist def set_FV3nml_ens_stoch_seeds(cdate): @@ -110,22 +109,28 @@ def set_FV3nml_ens_stoch_seeds(cdate): ) try: - set_namelist( - ["-q", "-n", fv3_nml_ensmem_fp, "-u", settings_str, "-o", fv3_nml_ensmem_fp] - ) + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(settings)) + tmpfile.seek(0) + subprocess.run(["uw config realize", + "-i", fv3_nml_ensmem_fp, + "-o", fv3_nml_ensmem_fp, + "-v", + "--values-file", tmpfile, + ] + ) except: print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Full path to output namelist file: - fv3_nml_ensmem_fp = '{fv3_nml_ensmem_fp}' - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Updating the FV3 namelist with stochastic seed parameters + failed. + Values to be updated: + """ ) + settings_str ) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index a1ffaa57ef..8e5e9fa957 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -24,7 +24,6 @@ cfg_to_yaml_str, ) -from set_namelist import set_namelist def set_FV3nml_sfc_climo_filenames(debug=False): @@ -95,32 +94,33 @@ def set_FV3nml_sfc_climo_filenames(debug=False): verbose=debug, ) - # Rename the FV3 namelist and call set_namelist - fv3_nml_base_fp = f"{FV3_NML_FP}.base" - mv_vrfy(f"{FV3_NML_FP} {fv3_nml_base_fp}") - + # Update the namelist file try: - set_namelist( - ["-q", "-n", fv3_nml_base_fp, "-u", settings_str, "-o", FV3_NML_FP] - ) + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(settings)) + tmpfile.seek(0) + subprocess.run(["uw config realize", + "-i", FV3_NML_FP, + "-o", FV3_NML_FP, + "-v", + "--values-file", tmpfile, + ] + ) except: print_err_msg_exit( dedent( f""" - Call to python script set_namelist.py to set the variables in the FV3 - namelist file that specify the paths to the surface climatology files - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_nml_base_fp = '{fv3_nml_base_fp}' - Full path to output namelist file: - FV3_NML_FP = '{FV3_NML_FP}' - Namelist settings specified on command line (these have highest precedence):\n - settings =\n\n""" + Updating the FV3 namelist with paths to the surface climatology + files failed. + Values to be updated:\n\n""" ) + settings_str ) - rm_vrfy(f"{fv3_nml_base_fp}") def parse_args(argv): diff --git a/ush/set_namelist.py b/ush/set_namelist.py deleted file mode 100755 index e578d3201f..0000000000 --- a/ush/set_namelist.py +++ /dev/null @@ -1,355 +0,0 @@ -#!/usr/bin/env python3 - -""" -This utility updates a Fortran namelist file using the f90nml package. The -settings that are modified are supplied via command line YAML-formatted string -and/or YAML configuration files. - -Additionally, the tool can be used to create a YAML file from an input namelist, -or the difference between two namelists. - -The user configuration file should contain a heirarchy that follows the -heirarchy for the Fortran namelist. An example of modifying an FV3 namelist: - - Configuration file contains: - - fv_core_nml: - k_split: 4 - n_split: 5 - - gfs_physics_nml: - do_sppt: True - -The output namelist will differ from the input namelist by only these three -settings. If one of these sections and/or variables did not previously exist, it -will be automatically created. It is up to the user to ensure that configuration -settings are provided under the correct sections and variable names. - -The optional base configuration file (provided via the -c command line argument) -contains the known set of configurations used and supported by the community, if -using the one provided in parm/FV3.input.yml. If maintaining this file -for a different set of configurations, ensure that the heirarchy is such that it -names the configuration at the top level (section), and the subsequent sections -match those in the F90 namelist that will be updated. - -Examples - - To show help options: - - set_namelist.py -h - - To produce a namelist (fv3_expt.nml) by specifying a physics package: - - set_namelist.py -n ../parm/input.nml.FV3 -c ../parm/FV3.input.yml FV3_HRRR - -o fv3_expt.nml - - To produce a YAML file (fv3_namelist.yml) from a user namelist: - - set_namelist.py -i my_namelist.nml -o fv3_namelist.nml -t yaml - - To produce a YAML file (fv3_my_namelist.yml) with differences from base nml: - - set_namelist.py -n ../parm/input.nml.FV3 -i my_namelist.nml -t yaml - -o fv3_my_namelist.nml - -Expected behavior: - - - A Fortran namelist that contains only user-defined settings will be - generated if no input namelist is provided. - - An unmodified copy of an input namelist will be generated in the - designated output location if no user-settings are provided. - - Command-line-entered settings over-ride settings in YAML configuration - file. - - Given a user namelist, the script can dump a YAML file. - - Given a user namelist and a base namelist, the script can dump the - difference in the two to a YAML file that can be included as a section - in the supported configs. -""" - -import argparse -import collections -import os -import sys - -import f90nml -import yaml - - -def config_exists(arg): - - """ - Checks whether the config file exists and if it contains the input - section. Returns the arg as provided if checks are passed. - """ - - # Agument is expected to be a 2-item list of file name and internal section - # name. - file_name = arg[0] - section_name = arg[1] - - file_exists(file_name) - - # Load the YAML file into a dictionary - with open(file_name, "r") as fn: - cfg = yaml.load(fn, Loader=yaml.Loader) - - # Grab only the section that is specified by the user - try: - cfg = cfg[section_name] - except KeyError: - msg = f"Section {section_name} does not exist in top level of {file_name}" - raise argparse.ArgumentTypeError(msg) - - return [cfg, section_name] - - -def file_exists(arg): - - """Check for existence of file""" - - if not os.path.exists(arg): - msg = f"{arg} does not exist!" - raise argparse.ArgumentTypeError(msg) - - return arg - - -def load_config(arg): - - """ - Check to ensure that the provided config file exists. If it does, load it - with YAML's safe loader and return the resulting dict. - """ - - return yaml.safe_load(arg) - - -def path_ok(arg): - - """ - Check whether the path to the file exists, and is writeable. Return the path - if it passes all checks, otherwise raise an error. - """ - - # Get the absolute path provided by arg - dir_name = os.path.abspath(os.path.dirname(arg)) - - # Ensure the arg path exists, and is writable. Raise error if not. - if os.path.lexists(dir_name) and os.access(dir_name, os.W_OK): - return arg - - msg = f"{arg} is not a writable path!" - raise argparse.ArgumentTypeError(msg) - - -def parse_args(argv): - - """ - Function maintains the arguments accepted by this script. Please see - Python's argparse documenation for more information about settings of each - argument. - """ - - parser = argparse.ArgumentParser( - description="Update a Fortran namelist with user-defined settings." - ) - - # Required - parser.add_argument( - "-o", - "--outfile", - help="Required: Full path to output file. This is a \ - namelist by default.", - required=True, - type=path_ok, - ) - - # Optional - parser.add_argument( - "-c", - "--config", - help="Full path to a YAML config file containing multiple \ - configurations, and the top-level section to use. Optional.", - metavar=("[FILE,", "SECTION]"), - nargs=2, - ) - parser.add_argument( - "-i", - "--input_nml", - help="Path to a user namelist. Use with -n and \ - -t yaml to get a YAML file to use with workflow.", - type=file_exists, - ) - parser.add_argument( - "-n", - "--basenml", - dest="nml", - help="Full path to the input Fortran namelist. Optional.", - type=file_exists, - ) - parser.add_argument( - "-t", - "--type", - choices=["nml", "yaml"], - default="nml", - help="Output file type.", - ) - parser.add_argument( - "-u", - "--user_config", - help="Command-line user config options in YAML-formatted \ - string. These options will override any provided in an \ - input file. Optional.", - metavar="YAML STRING", - type=load_config, - ) - - # Flags - parser.add_argument( - "-q", - "--quiet", - action="store_true", - help="If provided, suppress all output.", - ) - return parser.parse_args(argv) - - -def dict_diff(dict1, dict2): - - """ - Produces a dictionary of how dict2 differs from dict1 - """ - - diffs = {} - - # Loop through dict1 sections and key/value pairs - for sect, items in dict1.items(): - for key, val in items.items(): - - # If dict 2 has a different value, record the dict2 value - if val != dict2.get(sect, {}).get(key, ""): - if not diffs.get(sect): - diffs[sect] = {} - diffs[sect][key] = dict2.get(sect, {}).get(key) - - # Loop through dict2 sections and key/value pairs to catch any settings that - # may be present in the 2nd dict that weren't in the first. - for sect, items in dict2.items(): - for key, val in items.items(): - - # If dict1 has a diffent value than dict2, record the dict2 value - if val != dict1.get(sect, {}).get(key, ""): - - # Check to make sure it hasn't already been recorded - if diffs.get(sect, {}).get(key, "DNE") == "DNE": - if not diffs.get(sect): - diffs[sect] = {} - diffs[sect][key] = val - return diffs - - -def to_dict(odict): - - """Recursively convert OrderedDict to Python dict.""" - - if not isinstance(odict, collections.OrderedDict): - return odict - - ret = dict(odict) - for key, value in ret.items(): - if isinstance(value, collections.OrderedDict): - ret[key] = to_dict(value) - return ret - - -def update_dict(dest, newdict, quiet=False): - - """ - Overwrites all values in dest dictionary with values from newdict. Turn off - print statements with queit=True. - - Input: - - dest A dict that is to be updated. - newdict A dict containing sections and keys corresponding to - those in dest and potentially additional ones, that will be used to - update the dest dict. - quiet An optional boolean flag to turn off output. - - Output: - - None - - Result: - - The dest dict is updated in place. - """ - - for sect, values in newdict: - # If section is set to None, remove all contents from namelist - if values is None: - dest[sect] = {} - else: - for key, value in values.items(): - if not quiet: - print(f"Setting {sect}.{key} = {value}") - - # Remove key from dict if config is set to None - if value is None: - _ = dest[sect].pop(key, None) - else: - - try: - dest[sect][key] = value - except KeyError: - # Namelist section did not exist. Create it and update the value. - dest[sect] = {} - dest[sect][key] = value - - -def set_namelist(argv): - - """Using input command line arguments (cla), update a Fortran namelist file.""" - - # parse argumetns - cla = parse_args(argv) - if cla.config: - cla.config, _ = config_exists(cla.config) - - # Load base namelist into dict - nml = f90nml.Namelist() - if cla.nml is not None: - nml = f90nml.read(cla.nml) - - # Update namelist settings (nml) with config file settings (cfg) - cfg = {} - if cla.config is not None: - cfg = cla.config - update_dict(nml, cfg.items(), quiet=cla.quiet) - - # Update nml, overriding YAML if needed, with any command-line entries - if cla.user_config: - update_dict(nml, cla.user_config.items(), quiet=cla.quiet) - - # Write the resulting file - with open(cla.outfile, "w") as fn: - if cla.type == "nml": - nml.write(fn, sort=True) - - if cla.type == "yaml": - if cla.input_nml: - input_nml = f90nml.read(cla.input_nml) - - # Determine how input_nml differs from the configured namelist - diff = dict_diff(nml, input_nml) - - # Write diffs to YAML file - yaml.dump(diff, fn) - - else: - # Write the namelist to YAML file - yaml.dump(to_dict(nml.todict()), fn) - - -if __name__ == "__main__": - set_namelist(sys.argv[1:]) diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 0f10c675b2..32fca4a572 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -16,8 +16,6 @@ flatten_dict, ) -from set_namelist import set_namelist - def update_input_nml(run_dir): """Update the FV3 input.nml file in the specified run directory @@ -101,29 +99,32 @@ def update_input_nml(run_dir): fv3_input_nml_fp = os.path.join(run_dir, FV3_NML_FN) try: - set_namelist( - [ - "-q", - "-n", + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="fv3_settings", + suffix=".yaml") as tmpfile: + + tmpfile.write(settings_str) + tmpfile.seek(0) + subprocess.run(['uw config realize', + "-i", fv3_input_nml_fp, - "-u", - settings_str, + "--input-format", + "nml", "-o", fv3_input_nml_fp, - ] - ) + "-v", + "--values-file", + tmpfile, + ] + ) except: logging.exception( dedent( f""" - Call to python script set_namelist.py to generate an FV3 namelist file - failed. Parameters passed to this script are: - Full path to base namelist file: - fv3_input_nml_fp = '{fv3_input_nml_fp}' - Full path to output namelist file: - fv3_input_nml_fp = '{fv3_input_nml_fp}' - Namelist settings specified on command line:\n - settings =\n\n""" + Call to generate an FV3 namelist file failed. + """ ) + settings_str ) From c4eef3aa5a4666e00575e5773fe8c9a27ae58770 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 8 Dec 2023 15:58:08 +0000 Subject: [PATCH 03/30] WIP --- ush/generate_FV3LAM_wflow.py | 47 ++++++++++++++++++---- ush/set_FV3nml_ens_stoch_seeds.py | 58 ++++++++++++++------------- ush/set_FV3nml_sfc_climo_filenames.py | 3 +- 3 files changed, 72 insertions(+), 36 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 24254ea771..6eec9ba726 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -8,10 +8,12 @@ # pylint: disable=invalid-name import argparse -import os import logging -from textwrap import dedent +import os import sys +from subprocess import STDOUT, CalledProcessError, check_output +from textwrap import dedent + from python_utils import ( log_info, @@ -510,13 +512,28 @@ def generate_FV3LAM_wflow( suffix=".yaml") as tmpfile: tmpfile.write(cfg_to_yaml_str(physics_cfg)) tmpfile.seek(0) - subprocess.run(["uw config realize", + cmd = " ".join(["uw config realize", "-i", FV3_NML_BASE_SUITE_FP, "-o", FV3_NML_FP, "-v", "--values-file", tmpfile, - ] - ) + ]) + + indent = " " + try: + logfunc = logging.info + output = check_output(cmd, encoding="utf=8", env=env, shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception("Failed with status: %s", indent, e.returncode) + sys.exit(1) + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) + # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -653,13 +670,27 @@ def generate_FV3LAM_wflow( suffix=".yaml") as tmpfile: tmpfile.write(cfg_to_yaml_str(settings)) tmpfile.seek(0) - subprocess.run(["uw config realize", + cmd = " ".join(["uw config realize", "-i", FV3_NML_FP, "-o", FV3_NML_STOCH_FP, "-v", "--values-file", tmpfile, - ] - ) + ]) + + indent = " " + try: + logfunc = logging.info + output = check_output(cmd, encoding="utf=8", env=env, shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception("Failed with status: %s", indent, e.returncode) + sys.exit(1) + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) # # ----------------------------------------------------------------------- diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 2e190f292d..65e838971c 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 +import argparse import os import sys -import argparse -from textwrap import dedent from datetime import datetime +from subprocess import STDOUT, CalledProcessError, check_output +from textwrap import dedent from python_utils import ( print_input_args, @@ -108,32 +109,35 @@ def set_FV3nml_ens_stoch_seeds(cdate): verbose=VERBOSE, ) - try: - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(settings)) - tmpfile.seek(0) - subprocess.run(["uw config realize", - "-i", fv3_nml_ensmem_fp, - "-o", fv3_nml_ensmem_fp, - "-v", - "--values-file", tmpfile, - ] - ) - except: - print_err_msg_exit( - dedent( - f""" - Updating the FV3 namelist with stochastic seed parameters - failed. - Values to be updated: - """ - ) - + settings_str + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(settings)) + tmpfile.seek(0) + cmd = " ".join(["uw config realize", + "-i", fv3_nml_ensmem_fp, + "-o", fv3_nml_ensmem_fp, + "-v", + "--values-file", tmpfile, + ] ) + indent = " " + try: + logfunc = logging.info + output = check_output(cmd, encoding="utf=8", env=env, shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception("Failed with status: %s", indent, e.returncode) + sys.exit(1) + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) + def parse_args(argv): diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 8e5e9fa957..b6aaabe274 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -1,8 +1,9 @@ #!/usr/bin/env python3 +import argparse import os import sys -import argparse +from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent from python_utils import ( From 538c24d04eae521885d1e39443b6d1969e057b35 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 8 Dec 2023 09:20:40 -0700 Subject: [PATCH 04/30] WIP --- ush/set_FV3nml_sfc_climo_filenames.py | 53 ++++++++++---------- ush/update_input_nml.py | 71 ++++++++++++++------------- 2 files changed, 65 insertions(+), 59 deletions(-) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index b6aaabe274..a8b85df251 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -96,33 +96,34 @@ def set_FV3nml_sfc_climo_filenames(debug=False): ) # Update the namelist file - try: - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(settings)) - tmpfile.seek(0) - subprocess.run(["uw config realize", - "-i", FV3_NML_FP, - "-o", FV3_NML_FP, - "-v", - "--values-file", tmpfile, - ] - ) - except: - print_err_msg_exit( - dedent( - f""" - Updating the FV3 namelist with paths to the surface climatology - files failed. - Values to be updated:\n\n""" - ) - + settings_str + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="namelist_settings", + suffix=".yaml") as tmpfile: + tmpfile.write(cfg_to_yaml_str(settings)) + tmpfile.seek(0) + cmd = " ".join(["uw config realize", + "-i", FV3_NML_FP, + "-o", FV3_NML_FP, + "-v", + "--values-file", tmpfile, + ] ) - - + indent = " " + try: + logfunc = logging.info + output = check_output(cmd, encoding="utf=8", env=env, shell=True, + stderr=STDOUT, text=True) + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception("Failed with status: %s", indent, e.returncode) + sys.exit(1) + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) def parse_args(argv): """Parse command line arguments""" diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 32fca4a572..11d31ae9b6 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -import os -import sys import argparse import logging +import os +import sys +from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent from python_utils import ( @@ -98,39 +99,43 @@ def update_input_nml(run_dir): # fv3_input_nml_fp = os.path.join(run_dir, FV3_NML_FN) - try: - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="fv3_settings", - suffix=".yaml") as tmpfile: - - tmpfile.write(settings_str) - tmpfile.seek(0) - subprocess.run(['uw config realize', - "-i", - fv3_input_nml_fp, - "--input-format", - "nml", - "-o", - fv3_input_nml_fp, - "-v", - "--values-file", - tmpfile, - ] - ) - except: - logging.exception( - dedent( - f""" - Call to generate an FV3 namelist file failed. - """ - ) - + settings_str + with tempfile.NamedTemporaryFile( + dir="./", + mode="w+t", + prefix="fv3_settings", + suffix=".yaml") as tmpfile: + + tmpfile.write(settings_str) + tmpfile.seek(0) + cmd = " ".join(['uw config realize', + "-i", + fv3_input_nml_fp, + "--input-format", + "nml", + "-o", + fv3_input_nml_fp, + "-v", + "--values-file", + tmpfile, + ] ) - return False - return True + indent = " " + try: + logfunc = logging.info + output = check_output(cmd, encoding="utf=8", env=env, shell=True, + stderr=STDOUT, text=True) + ret = True + except CalledProcessError as e: + logfunc = logging.error + output = e.output + logging.exception("Failed with status: %s", indent, e.returncode) + ret = False + finally: + logfunc("Output:") + for line in output.split("\n"): + logfunc("%s%s", indent * 2, line) + return ret def parse_args(argv): From 819a48d446e370554a67d7fbf91b042f1b4ceabf Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 8 Dec 2023 21:12:33 +0000 Subject: [PATCH 05/30] WIP --- parm/FV3.input.yml | 134 +++++++++++++------------- ush/generate_FV3LAM_wflow.py | 11 ++- ush/load_modules_wflow.sh | 4 +- ush/set_FV3nml_ens_stoch_seeds.py | 5 +- ush/set_FV3nml_sfc_climo_filenames.py | 3 +- ush/update_input_nml.py | 1 + 6 files changed, 83 insertions(+), 75 deletions(-) diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index 36334fcc2f..2a59e10e39 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -78,7 +78,7 @@ FV3_HRRR: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: !!python/none + do_sfcperts: gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True @@ -107,7 +107,7 @@ FV3_RAP: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: !!python/none + do_sfcperts: gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True @@ -135,40 +135,40 @@ FV3_GFS_2017_gfdlmp: k_split: 6 n_split: 6 nord: 2 - nord_zs_filter: !!python/none + nord_zs_filter: range_warn: False vtdm4: 0.075 gfs_physics_nml: &gfs_2017_gfdlmp_phys avg_max_length: 3600.0 - bl_mynn_tkeadvect: !!python/none - bl_mynn_edmf: !!python/none - bl_mynn_edmf_mom: !!python/none + bl_mynn_tkeadvect: + bl_mynn_edmf: + bl_mynn_edmf_mom: cdmbgwd: [3.5, 0.01] - cplflx: !!python/none + cplflx: do_deep: False - do_mynnedmf: !!python/none - do_mynnsfclay: !!python/none + do_mynnedmf: + do_mynnsfclay: fhcyc: 0.0 fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - icloud_bl: !!python/none + icloud_bl: imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 lgfdlmprad: True - lheatstrg: !!python/none - lndp_type: !!python/none - lsm: !!python/none - lsoil: !!python/none - lsoil_lsm: !!python/none - ltaerosol: !!python/none - n_var_lndp: !!python/none + lheatstrg: + lndp_type: + lsm: + lsoil: + lsoil_lsm: + ltaerosol: + n_var_lndp: oz_phys: True oz_phys_2015: False - satmedmf: !!python/none + satmedmf: shal_cnv: True - ttendlim: !!python/none + ttendlim: gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp c_cracw: 0.8 c_paut: 0.5 @@ -267,7 +267,7 @@ FV3_GFS_v15p2: kord_wz: 9 n_split: 8 n_sponge: 30 - nord_zs_filter: !!python/none + nord_zs_filter: nudge_qv: True range_warn: False rf_cutoff: 750.0 @@ -278,16 +278,16 @@ FV3_GFS_v15p2: tau_l2v: 225.0 tau_v2l: 150.0 gfs_physics_nml: &gfs_v15_gfs_physics - bl_mynn_edmf: !!python/none - bl_mynn_edmf_mom: !!python/none - bl_mynn_tkeadvect: !!python/none + bl_mynn_edmf: + bl_mynn_edmf_mom: + bl_mynn_tkeadvect: cnvcld: True cnvgwd: True - cplflx: !!python/none + cplflx: do_myjpbl: False do_myjsfc: False - do_mynnedmf: !!python/none - do_mynnsfclay: !!python/none + do_mynnedmf: + do_mynnsfclay: do_tofd: False do_ugwp: False do_ysu: False @@ -295,12 +295,12 @@ FV3_GFS_v15p2: fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - iau_delthrs: !!python/none - iaufhrs: !!python/none + iau_delthrs: + iaufhrs: imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 - icloud_bl: !!python/none + icloud_bl: iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -316,14 +316,14 @@ FV3_GFS_v15p2: iopt_trs: 2 ldiag_ugwp: False lgfdlmprad: True - lradar: !!python/none + lradar: lsm: 1 - lsoil: !!python/none - lsoil_lsm: !!python/none - ltaerosol: !!python/none + lsoil: + lsoil_lsm: + ltaerosol: shal_cnv: True shinhong: False - ttendlim: !!python/none + ttendlim: xkzm_h: 1.0 xkzm_m: 1.0 xkzminv: 0.3 @@ -337,7 +337,7 @@ FV3_GFS_v15_thompson_mynn_lam3km: avg_max_length: 3600.0 fv_core_nml: agrid_vel_rst: True - full_zs_filter: !!python/none + full_zs_filter: n_sponge: 9 npz_type: '' rf_fast: False @@ -377,20 +377,20 @@ FV3_GFS_v15_thompson_mynn_lam3km: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - iopt_trs: !!python/none + iopt_trs: iovr: 3 ldiag_ugwp: False lgfdlmprad: False lsm: 1 - lsoil: !!python/none - lsoil_lsm: !!python/none + lsoil: + lsoil_lsm: ltaerosol: False print_diff_pgr: True - sfclay_compute_flux: !!python/none + sfclay_compute_flux: xkzminv: 0.3 xkzm_m: 1.0 xkzm_h: 1.0 - surf_map_nml: !!python/none + surf_map_nml: FV3_GFS_v16: cires_ugwp_nml: @@ -414,7 +414,7 @@ FV3_GFS_v16: na_init: 0 nudge_dz: False res_latlon_dynamics: '' - rf_fast: !!python/none + rf_fast: tau: 10.0 gfdl_cloud_microphysics_nml: <<: *gfs_gfdl_cloud_mp @@ -426,10 +426,10 @@ FV3_GFS_v16: gfs_physics_nml: <<: *gfs_v15_gfs_physics cdmbgwd: [4.0, 0.15, 1.0, 1.0] - do_myjpbl: !!python/none - do_myjsfc: !!python/none + do_myjpbl: + do_myjsfc: do_tofd: True - do_ysu: !!python/none + do_ysu: hybedmf: False iaer: 5111 icliq_sw: 2 @@ -438,23 +438,23 @@ FV3_GFS_v16: isatmedmf: 1 lgfdlmprad: True lheatstrg: True - lndp_type: !!python/none + lndp_type: lsoil: 4 - n_var_lndp: !!python/none + n_var_lndp: prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] satmedmf: True - shinhong: !!python/none - xkzminv: !!python/none - xkzm_m: !!python/none - xkzm_h: !!python/none + shinhong: + xkzminv: + xkzm_m: + xkzm_h: mpp_io_nml: deflate_level: 1 shuffle: 1 namsfc: landice: True ldebug: False - surf_map_nml: !!python/none + surf_map_nml: FV3_GFS_v17_p8: cires_ugwp_nml: @@ -467,8 +467,8 @@ FV3_GFS_v17_p8: delt_max: 0.002 dnats: 0 do_sat_adj: False - do_vort_damp: !!python/none - full_zs_filter: !!python/none + do_vort_damp: + full_zs_filter: fv_sg_adj: 450 hord_dp: -5 hord_mt: 5 @@ -480,10 +480,10 @@ FV3_GFS_v17_p8: n_sponge: 10 nord: 2 nudge_dz: False - n_zs_filter: !!python/none + n_zs_filter: range_warn: True res_latlon_dynamics: '' - rf_fast: !!python/none + rf_fast: tau: 10.0 gfdl_cloud_microphysics_nml: <<: *gfs_gfdl_cloud_mp @@ -505,8 +505,8 @@ FV3_GFS_v17_p8: do_gsl_drag_ls_bl: False do_gsl_drag_ss: True do_gsl_drag_tofd: False - do_myjpbl: !!python/none - do_myjsfc: !!python/none + do_myjpbl: + do_myjsfc: do_mynnedmf: False do_mynnsfclay: False do_RRTMGP: False @@ -515,7 +515,7 @@ FV3_GFS_v17_p8: do_ugwp_v0_orog_only: False do_ugwp_v1: False do_ugwp_v1_orog_only: False - do_ysu: !!python/none + do_ysu: doGP_cldoptics_LUT: False doGP_lwscat: False dt_inner: 150 @@ -525,7 +525,7 @@ FV3_GFS_v17_p8: hybedmf: False iaer: 1011 ialb: 2 - iau_inc_files: !!python/none + iau_inc_files: icloud_bl: 1 icliq_sw: 2 iems: 2 @@ -538,7 +538,7 @@ FV3_GFS_v17_p8: iopt_stc: 3 iovr: 3 isatmedmf: 1 - ldiag_ugwp: !!python/none + ldiag_ugwp: lgfdlmprad: False lradar: False lseaspray: True @@ -550,7 +550,7 @@ FV3_GFS_v17_p8: lw_file_gas: rrtmgp-data-lw-g128-210809.nc min_lakeice: 0.15 min_seaice: 0.15 - nsfullradar_diag: !!python/none + nsfullradar_diag: prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] qdiag3d: False @@ -561,14 +561,14 @@ FV3_GFS_v17_p8: rrtmgp_nGptsSW: 112 satmedmf: True sedi_semi: True - sfclay_compute_flux: !!python/none - shinhong: !!python/none + sfclay_compute_flux: + shinhong: sw_file_clouds: rrtmgp-cloud-optics-coeffs-sw.nc sw_file_gas: rrtmgp-data-sw-g112-210809.nc ttendlim: -999 - xkzminv: !!python/none - xkzm_m: !!python/none - xkzm_h: !!python/none + xkzminv: + xkzm_m: + xkzm_h: mpp_io_nml: deflate_level: 1 shuffle: 1 @@ -577,5 +577,5 @@ FV3_GFS_v17_p8: fsics: 0 landice: False ldebug: False - surf_map_nml: !!python/none + surf_map_nml: diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 6eec9ba726..18d207c362 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -11,6 +11,7 @@ import logging import os import sys +import tempfile from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent @@ -29,6 +30,7 @@ cfg_to_yaml_str, find_pattern_in_str, flatten_dict, + update_dict, ) from setup import setup @@ -516,13 +518,14 @@ def generate_FV3LAM_wflow( "-i", FV3_NML_BASE_SUITE_FP, "-o", FV3_NML_FP, "-v", - "--values-file", tmpfile, + "--values-file", tmpfile.name, ]) indent = " " + output = "" try: logfunc = logging.info - output = check_output(cmd, encoding="utf=8", env=env, shell=True, + output = check_output(cmd, encoding="utf=8", shell=True, stderr=STDOUT, text=True) except CalledProcessError as e: logfunc = logging.error @@ -674,13 +677,13 @@ def generate_FV3LAM_wflow( "-i", FV3_NML_FP, "-o", FV3_NML_STOCH_FP, "-v", - "--values-file", tmpfile, + "--values-file", tmpfile.name, ]) indent = " " try: logfunc = logging.info - output = check_output(cmd, encoding="utf=8", env=env, shell=True, + output = check_output(cmd, encoding="utf=8", shell=True, stderr=STDOUT, text=True) except CalledProcessError as e: logfunc = logging.error diff --git a/ush/load_modules_wflow.sh b/ush/load_modules_wflow.sh index 67499f4888..9d3b0bfe96 100755 --- a/ush/load_modules_wflow.sh +++ b/ush/load_modules_wflow.sh @@ -31,7 +31,7 @@ if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then usage exit 0 fi - +set -x # Set machine name to lowercase machine=${1,,} @@ -49,8 +49,10 @@ fi # Source modulefile for this machine WFLOW_MOD_FN="wflow_${machine}" +set +x source "${HOMEdir}/etc/lmod-setup.sh" ${machine} module use "${HOMEdir}/modulefiles" +echo MODPATH $MODULEPATH module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || { echo "ERROR: Loading of platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 65e838971c..5bc7dfae71 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -3,6 +3,7 @@ import argparse import os import sys +import tempfile from datetime import datetime from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent @@ -120,13 +121,13 @@ def set_FV3nml_ens_stoch_seeds(cdate): "-i", fv3_nml_ensmem_fp, "-o", fv3_nml_ensmem_fp, "-v", - "--values-file", tmpfile, + "--values-file", tmpfile.name, ] ) indent = " " try: logfunc = logging.info - output = check_output(cmd, encoding="utf=8", env=env, shell=True, + output = check_output(cmd, encoding="utf=8", shell=True, stderr=STDOUT, text=True) except CalledProcessError as e: logfunc = logging.error diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index a8b85df251..e2793834f8 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -3,6 +3,7 @@ import argparse import os import sys +import tempfile from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent @@ -107,7 +108,7 @@ def set_FV3nml_sfc_climo_filenames(debug=False): "-i", FV3_NML_FP, "-o", FV3_NML_FP, "-v", - "--values-file", tmpfile, + "--values-file", tmpfile.name, ] ) indent = " " diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 11d31ae9b6..6b9ef303f0 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -4,6 +4,7 @@ import logging import os import sys +import tempfile from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent From ae106a703f98eb8ac76b289cac2d8a20bc083683 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Wed, 13 Dec 2023 18:07:03 +0000 Subject: [PATCH 06/30] Fix a typo. --- ush/set_FV3nml_sfc_climo_filenames.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index e2793834f8..6a0356539a 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -111,7 +111,7 @@ def set_FV3nml_sfc_climo_filenames(debug=False): "--values-file", tmpfile.name, ] ) - indent = " " + indent = " " try: logfunc = logging.info output = check_output(cmd, encoding="utf=8", env=env, shell=True, From afe40d274c69eb56ac66e211300831288fefc242 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 8 Feb 2024 14:01:08 +0000 Subject: [PATCH 07/30] [develop] Add -n 1 to allow the use of the service partition --- parm/wflow/verify_pre.yaml | 4 ++++ ush/machine/hera.yaml | 1 + ush/machine/jet.yaml | 1 + 3 files changed, 6 insertions(+) diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index bc1ca11078..eb1a7eb796 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -30,6 +30,7 @@ task_get_obs_ccpa: OBS_DIR: '&CCPA_OBS_DIR;' OBTYPE: 'CCPA' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" walltime: 00:45:00 @@ -42,6 +43,7 @@ task_get_obs_nohrsc: OBS_DIR: '&NOHRSC_OBS_DIR;' OBTYPE: 'NOHRSC' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" walltime: 00:45:00 @@ -55,6 +57,7 @@ task_get_obs_mrms: OBTYPE: 'MRMS' VAR: 'REFC RETOP' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" walltime: 00:45:00 @@ -68,6 +71,7 @@ task_get_obs_ndas: FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' queue: "&QUEUE_HPSS;" + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' walltime: 02:00:00 diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index 2ad41e36aa..8d751ae891 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -21,6 +21,7 @@ platform: RUN_CMD_NEXUS: srun -n ${nprocs} --export=ALL RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS} SCHED_NATIVE_CMD: --export=NONE + SCHED_NATIVE_CMD_HPSS: -n 1 --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/input_model_data TEST_AQM_INPUT_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/aqm_data diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index b5cb19cd0b..93d375ee02 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -19,6 +19,7 @@ platform: RUN_CMD_SERIAL: time RUN_CMD_UTILS: srun --export=ALL SCHED_NATIVE_CMD: --export=NONE + SCHED_NATIVE_CMD_HPSS: -n 1 --export=NONE PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }' TEST_EXTRN_MDL_SOURCE_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/input_model_data TEST_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen From 870b3159e30610b9c4d57e834465918ac4da8275 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 18:45:23 +0000 Subject: [PATCH 08/30] Updating and linting. --- ush/set_FV3nml_ens_stoch_seeds.py | 92 +++++++++-------------- ush/set_FV3nml_sfc_climo_filenames.py | 104 ++++++++++---------------- 2 files changed, 75 insertions(+), 121 deletions(-) diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 5bc7dfae71..1d60cb09ac 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -1,33 +1,28 @@ #!/usr/bin/env python3 +""" +Updates stochastic physics parameters in the namelist based on user configuration settings. +""" + import argparse +import datetime as dt import os import sys -import tempfile -from datetime import datetime -from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent +from uwtools.api.config import get_sh_config, realize + from python_utils import ( print_input_args, print_info_msg, - print_err_msg_exit, - date_to_str, - mkdir_vrfy, - cp_vrfy, - cd_vrfy, - str_to_type, import_vars, - set_env_var, - define_macos_utilities, cfg_to_yaml_str, - load_shell_config, - flatten_dict, ) +VERBOSE = os.environ.get("VERBOSE", "true") -def set_FV3nml_ens_stoch_seeds(cdate): +def set_fv3nml_ens_stoch_seeds(cdate, expt_config): """ This function, for an ensemble-enabled experiment (i.e. for an experiment for which the workflow configuration variable @@ -40,15 +35,17 @@ def set_FV3nml_ens_stoch_seeds(cdate): called as part of the TN_RUN_FCST task. Args: - cdate + cdate the cycle + expt_config the in-memory dict representing the experiment configuration Returns: None """ print_input_args(locals()) - # import all environment variables - import_vars() + # set variables important to this function from the experiment definition + import_vars(dictionary=expt_config["global"]) + # pylint: disable=undefined-variable # # ----------------------------------------------------------------------- @@ -60,7 +57,7 @@ def set_FV3nml_ens_stoch_seeds(cdate): # fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" - ensmem_num = int(ENSMEM_INDX) + ensmem_num = int(os.environ["ENSMEM_INDX"]) cdate_i = int(cdate.strftime("%Y%m%d%H")) @@ -96,50 +93,25 @@ def set_FV3nml_ens_stoch_seeds(cdate): settings["nam_sfcperts"] = {"iseed_lndp": [iseed_lsm_spp]} - settings_str = cfg_to_yaml_str(settings) - print_info_msg( dedent( f""" The variable 'settings' specifying seeds in '{fv3_nml_ensmem_fp}' has been set as follows: - settings =\n\n""" - ) - + settings_str, + settings =\n\n + + {cfg_to_yaml_str(settings)}""" + ), verbose=VERBOSE, ) - - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(settings)) - tmpfile.seek(0) - cmd = " ".join(["uw config realize", - "-i", fv3_nml_ensmem_fp, - "-o", fv3_nml_ensmem_fp, - "-v", - "--values-file", tmpfile.name, - ] + realize( + input_config=fv3_nml_ensmem_fp, + input_format="nml", + output_file=fv3_nml_ensmem_fp, + output_format="nml", + supplemental_configs=settings, ) - indent = " " - try: - logfunc = logging.info - output = check_output(cmd, encoding="utf=8", shell=True, - stderr=STDOUT, text=True) - except CalledProcessError as e: - logfunc = logging.error - output = e.output - logging.exception("Failed with status: %s", indent, e.returncode) - sys.exit(1) - finally: - logfunc("Output:") - for line in output.split("\n"): - logfunc("%s%s", indent * 2, line) - - def parse_args(argv): """Parse command line arguments""" @@ -147,7 +119,13 @@ def parse_args(argv): description="Creates stochastic seeds for an ensemble experiment." ) - parser.add_argument("-c", "--cdate", dest="cdate", required=True, help="Date.") + parser.add_argument( + "-c", "--cdate", + dest="cdate", + required=True, + type=lambda d: dt.datetime.strptime(d, '%Y%m%d%H'), + help="Date.", + ) parser.add_argument( "-p", @@ -162,7 +140,5 @@ def parse_args(argv): if __name__ == "__main__": args = parse_args(sys.argv[1:]) - cfg = load_shell_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - set_FV3nml_ens_stoch_seeds(str_to_type(args.cdate)) + config = get_sh_config(args.path_to_defns) + set_fv3nml_ens_stoch_seeds(args.cdate, config) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 6a0356539a..bd7f7daba0 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -1,34 +1,41 @@ #!/usr/bin/env python3 +""" +Update filenames for surface climotology files in the namelist. +""" + import argparse import os +import re import sys -import tempfile -from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent +from uwtools.api.config import get_yaml_config, realize + from python_utils import ( - print_input_args, print_info_msg, - print_err_msg_exit, check_var_valid_value, - mv_vrfy, - mkdir_vrfy, - cp_vrfy, - rm_vrfy, import_vars, - set_env_var, - load_config_file, - load_shell_config, - flatten_dict, - define_macos_utilities, - find_pattern_in_str, cfg_to_yaml_str, ) +VERBOSE = os.environ.get("VERBOSE", "true") + +needed_vars = [ + "CRES", + "DO_ENSEMBLE", + "EXPTDIR", + "FIXlam", + "FV3_NML_FP", + "PARMdir", + "RUN_ENVIR", + ] +import_vars(env_vars=needed_vars) -def set_FV3nml_sfc_climo_filenames(debug=False): +# pylint: disable=undefined-variable + +def set_fv3nml_sfc_climo_filenames(debug=False): """ This function sets the values of the variables in the forecast model's namelist file that specify the paths to the surface @@ -45,12 +52,8 @@ def set_FV3nml_sfc_climo_filenames(debug=False): """ # import all environment variables - import_vars() - # fixed file mapping variables - fixed_cfg = load_config_file(os.path.join(PARMdir, "fixed_files_mapping.yaml")) - IMPORTS = ["SFC_CLIMO_FIELDS", "FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"] - import_vars(dictionary=flatten_dict(fixed_cfg), env_vars=IMPORTS) + fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] # The regular expression regex_search set below will be used to extract # from the elements of the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING @@ -69,18 +72,16 @@ def set_FV3nml_sfc_climo_filenames(debug=False): dummy_run_dir += os.sep + "any_ensmem" namsfc_dict = {} - for mapping in FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - tup = find_pattern_in_str(regex_search, mapping) - nml_var_name = tup[0] - sfc_climo_field_name = tup[1] + for mapping in fixed_cfg["FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING"]: + nml_var_name, sfc_climo_field_name = re.search(regex_search, mapping).groups() - check_var_valid_value(sfc_climo_field_name, SFC_CLIMO_FIELDS) + check_var_valid_value(sfc_climo_field_name, fixed_cfg["SFC_CLIMO_FIELDS"]) - fp = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") + file_path = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") if RUN_ENVIR != "nco": - fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) + file_path = os.path.relpath(os.path.realpath(file_path), start=dummy_run_dir) - namsfc_dict[nml_var_name] = fp + namsfc_dict[nml_var_name] = file_path settings["namsfc_dict"] = namsfc_dict settings_str = cfg_to_yaml_str(settings) @@ -90,41 +91,21 @@ def set_FV3nml_sfc_climo_filenames(debug=False): f""" The variable 'settings' specifying values of the namelist variables has been set as follows:\n - settings =\n\n""" - ) - + settings_str, + settings = + + {settings_str} + """ + ), verbose=debug, ) - # Update the namelist file - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(settings)) - tmpfile.seek(0) - cmd = " ".join(["uw config realize", - "-i", FV3_NML_FP, - "-o", FV3_NML_FP, - "-v", - "--values-file", tmpfile.name, - ] + realize( + input_config=FV3_NML_FP, + input_format="nml", + output_file=FV3_NML_FP, + output_format="nml", + supplemental_configs=settings, ) - indent = " " - try: - logfunc = logging.info - output = check_output(cmd, encoding="utf=8", env=env, shell=True, - stderr=STDOUT, text=True) - except CalledProcessError as e: - logfunc = logging.error - output = e.output - logging.exception("Failed with status: %s", indent, e.returncode) - sys.exit(1) - finally: - logfunc("Output:") - for line in output.split("\n"): - logfunc("%s%s", indent * 2, line) def parse_args(argv): """Parse command line arguments""" @@ -145,7 +126,4 @@ def parse_args(argv): if __name__ == "__main__": args = parse_args(sys.argv[1:]) - cfg = load_shell_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) - set_FV3nml_sfc_climo_filenames(args.debug) + set_fv3nml_sfc_climo_filenames(args.debug) From 81403618fd60210e3836c43de18dc11518b4ce89 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 18:46:25 +0000 Subject: [PATCH 09/30] Renaming for PEP8 standard. --- ...et_FV3nml_ens_stoch_seeds.py => set_fv3nml_ens_stoch_seeds.py} | 0 ...l_sfc_climo_filenames.py => set_fv3nml_sfc_climo_filenames.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename ush/{set_FV3nml_ens_stoch_seeds.py => set_fv3nml_ens_stoch_seeds.py} (100%) rename ush/{set_FV3nml_sfc_climo_filenames.py => set_fv3nml_sfc_climo_filenames.py} (100%) diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py similarity index 100% rename from ush/set_FV3nml_ens_stoch_seeds.py rename to ush/set_fv3nml_ens_stoch_seeds.py diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py similarity index 100% rename from ush/set_FV3nml_sfc_climo_filenames.py rename to ush/set_fv3nml_sfc_climo_filenames.py From 5d314a4f282d69d5d0efe2be488ddcf9b6f2b9e8 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 18:48:16 +0000 Subject: [PATCH 10/30] Linting. --- ush/generate_FV3LAM_wflow.py | 85 ++++++++---------------------------- 1 file changed, 19 insertions(+), 66 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 67a195fe3b..d2d291049b 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -11,17 +11,17 @@ import logging import os import sys -import tempfile from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent +from uwtools.api.config import get_yaml_config, realize + from python_utils import ( log_info, import_vars, export_vars, cp_vrfy, ln_vrfy, - load_config_file, mkdir_vrfy, mv_vrfy, create_symlink_to_file, @@ -29,11 +29,10 @@ cfg_to_yaml_str, find_pattern_in_str, flatten_dict, - update_dict, ) from setup import setup -from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames +from set_fv3nml_sfc_climo_filenames import set_fv3nml_sfc_climo_filenames from get_crontab_contents import add_crontab_line from check_python_version import check_python_version @@ -513,38 +512,14 @@ def generate_FV3LAM_wflow( # ----------------------------------------------------------------------- # - physics_cfg = load_config_file(FV3_NML_YAML_CONFIG_FP)[CCPP_PHYS_SUITE] - update_dict(settings, physics_cfg) - - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(physics_cfg)) - tmpfile.seek(0) - cmd = " ".join(["uw config realize", - "-i", FV3_NML_BASE_SUITE_FP, - "-o", FV3_NML_FP, - "-v", - "--values-file", tmpfile.name, - ]) - - indent = " " - output = "" - try: - logfunc = logging.info - output = check_output(cmd, encoding="utf=8", shell=True, - stderr=STDOUT, text=True) - except CalledProcessError as e: - logfunc = logging.error - output = e.output - logging.exception("Failed with status: %s", indent, e.returncode) - sys.exit(1) - finally: - logfunc("Output:") - for line in output.split("\n"): - logfunc("%s%s", indent * 2, line) + physics_cfg = get_yaml_config(FV3_NML_YAML_CONFIG_FP) + realize( + input_config=FV3_NML_BASE_SUITE_FP, + input_format="nml", + output_file=FV3_NML_FP, + output_format="nml", + supplemental_configs=physics_cfg[CCPP_PHYS_SUITE], + ) # # If not running the TN_MAKE_GRID task (which implies the workflow will @@ -560,7 +535,7 @@ def generate_FV3LAM_wflow( # if not expt_config['rocoto']['tasks'].get('task_make_grid'): - set_FV3nml_sfc_climo_filenames(debug) + set_fv3nml_sfc_climo_filenames(debug) # # ----------------------------------------------------------------------- @@ -674,35 +649,13 @@ def generate_FV3LAM_wflow( #----------------------------------------------------------------------- # if any((DO_SPP, DO_SPPT, DO_SHUM, DO_SKEB, DO_LSM_SPP)): - - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="namelist_settings", - suffix=".yaml") as tmpfile: - tmpfile.write(cfg_to_yaml_str(settings)) - tmpfile.seek(0) - cmd = " ".join(["uw config realize", - "-i", FV3_NML_FP, - "-o", FV3_NML_STOCH_FP, - "-v", - "--values-file", tmpfile.name, - ]) - - indent = " " - try: - logfunc = logging.info - output = check_output(cmd, encoding="utf=8", shell=True, - stderr=STDOUT, text=True) - except CalledProcessError as e: - logfunc = logging.error - output = e.output - logging.exception("Failed with status: %s", indent, e.returncode) - sys.exit(1) - finally: - logfunc("Output:") - for line in output.split("\n"): - logfunc("%s%s", indent * 2, line) + realize( + input_config=FV3_NML_FP, + input_format="nml", + output_file=FV3_NML_STOCH_FP, + output_format="nml", + supplemental_configs=settings, + ) # # ----------------------------------------------------------------------- From 46980b4189332c4fc3dfe947eaf6dd537aa2cda8 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 18:49:04 +0000 Subject: [PATCH 11/30] Linted. --- ush/update_input_nml.py | 144 +++++++++++----------------------------- 1 file changed, 38 insertions(+), 106 deletions(-) diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 6b9ef303f0..fa412a636d 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -1,62 +1,38 @@ #!/usr/bin/env python3 +""" +Update the model namelist for a variety of different settings. +""" + import argparse -import logging import os import sys -import tempfile -from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent +from uwtools.api.config import realize + from python_utils import ( - import_vars, print_input_args, print_info_msg, - print_err_msg_exit, cfg_to_yaml_str, - load_shell_config, - flatten_dict, ) +VERBOSE = os.environ.get("VERBOSE", "true") -def update_input_nml(run_dir): +def update_input_nml(namelist, restart, aqm_na_13km): """Update the FV3 input.nml file in the specified run directory Args: - run_dir: run directory + namelist: path to the namelist Returns: Boolean """ print_input_args(locals()) - - # import all environment variables - import_vars() - - # - # ----------------------------------------------------------------------- - # - # Update the FV3 input.nml file in the specified run directory. - # - # ----------------------------------------------------------------------- - # - print_info_msg( - f""" - Updating the FV3 input.nml file in the specified run directory (run_dir): - run_dir = '{run_dir}'""", - verbose=VERBOSE, - ) - # - # ----------------------------------------------------------------------- - # - # Set new values of the specific parameters to be updated. - # - # ----------------------------------------------------------------------- - # settings = {} # For restart run - if args.restart: + if restart: settings["fv_core_nml"] = { "external_ic": False, "make_nh": False, @@ -69,112 +45,68 @@ def update_input_nml(run_dir): settings["gfs_physics_nml"] = { "nstf_name": [2, 0, 0, 0, 0], } - + # For AQM_NA_13km domain for air quality modeling - if args.aqm_na_13km: + if aqm_na_13km: settings["fv_core_nml"] = { "k_split": 1, "n_split": 8, } - settings_str = cfg_to_yaml_str(settings) - print_info_msg( dedent( f""" - The variable 'settings' specifying values to be used in the FV3 'input.nml' - file for restart has been set as follows:\n - settings =\n\n""" - ) - + settings_str, + Updating {namelist} + + The updated values are: + + {cfg_to_yaml_str(settings)} + + """ + ), verbose=VERBOSE, ) - # - # ----------------------------------------------------------------------- - # - # Call a python script to update the experiment's actual FV3 INPUT.NML - # file for restart. - # - # ----------------------------------------------------------------------- - # - fv3_input_nml_fp = os.path.join(run_dir, FV3_NML_FN) - - with tempfile.NamedTemporaryFile( - dir="./", - mode="w+t", - prefix="fv3_settings", - suffix=".yaml") as tmpfile: - - tmpfile.write(settings_str) - tmpfile.seek(0) - cmd = " ".join(['uw config realize', - "-i", - fv3_input_nml_fp, - "--input-format", - "nml", - "-o", - fv3_input_nml_fp, - "-v", - "--values-file", - tmpfile, - ] - ) - - indent = " " - try: - logfunc = logging.info - output = check_output(cmd, encoding="utf=8", env=env, shell=True, - stderr=STDOUT, text=True) - ret = True - except CalledProcessError as e: - logfunc = logging.error - output = e.output - logging.exception("Failed with status: %s", indent, e.returncode) - ret = False - finally: - logfunc("Output:") - for line in output.split("\n"): - logfunc("%s%s", indent * 2, line) - return ret + # Update the experiment's FV3 INPUT.NML file + realize( + input_config=namelist, + input_format="nml", + output_file=namelist, + output_format="nml", + supplemental_configs=settings, + ) def parse_args(argv): """Parse command line arguments""" parser = argparse.ArgumentParser(description="Update FV3 input.nml file for restart.") parser.add_argument( - "-r", "--run_dir", - dest="run_dir", + "-n", "--namelist", + dest="namelist", required=True, - help="Run directory." - ) - - parser.add_argument( - "-p", "--path-to-defns", - dest="path_to_defns", - required=True, - help="Path to var_defns file.", + help="Run directory.", ) parser.add_argument( "--restart", action='store_true', - help='Update for restart') + help='Update for restart', + ) parser.add_argument( "--aqm_na_13km", action='store_true', - help='Update for AQM_NA_13km in air quality modeling') + help='Update for AQM_NA_13km in air quality modeling', + ) return parser.parse_args(argv) if __name__ == "__main__": args = parse_args(sys.argv[1:]) - cfg = load_shell_config(args.path_to_defns) - cfg = flatten_dict(cfg) - import_vars(dictionary=cfg) update_input_nml( - run_dir=args.run_dir, + namelist=args.namelist, + restart=args.restart, + aqm_na_13km=args.aqm_na_13km, ) From f25b5bc98ecab979e9f36b41c665fabb05926e63 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 18:49:26 +0000 Subject: [PATCH 12/30] WIP --- environment.yml | 2 +- parm/fixed_files_mapping.yaml | 29 ----------------------------- scripts/exregional_run_fcst.sh | 6 ++---- 3 files changed, 3 insertions(+), 34 deletions(-) diff --git a/environment.yml b/environment.yml index c574df5e23..6fe2548da9 100644 --- a/environment.yml +++ b/environment.yml @@ -5,4 +5,4 @@ channels: dependencies: - pylint=2.17* - pytest=7.2* - - uwtools=1.0.0 + - uwtools=2.0.0 diff --git a/parm/fixed_files_mapping.yaml b/parm/fixed_files_mapping.yaml index 90fd1870a4..54ddd41a81 100644 --- a/parm/fixed_files_mapping.yaml +++ b/parm/fixed_files_mapping.yaml @@ -139,35 +139,6 @@ fixed_files: !join_str ["FNSMCC | ",*FNSMCC], !join_str ["FNMSKH | ",*FNMSKH] ] - #"FNZORC | $FNZORC", - - # - #----------------------------------------------------------------------- - # - # FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: - # This array is used to set some of the namelist variables in the forecast - # model's namelist file that represent the relative or absolute paths of - # various fixed files (the first column of the array, where columns are - # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXlam directory derived from - # the corresponding surface climatology fields (the second column of the - # array). - # - #----------------------------------------------------------------------- - # - FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING: [ - "FNALBC | snowfree_albedo", - "FNALBC2 | facsf", - "FNTG3C | substrate_temperature", - "FNVEGC | vegetation_greenness", - "FNVETC | vegetation_type", - "FNSOTC | soil_type", - "FNVMNC | vegetation_greenness", - "FNVMXC | vegetation_greenness", - "FNSLPC | slope_type", - "FNABSC | maximum_snow_albedo" - ] - # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 6dfad9673b..e1af7a3bb7 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -492,8 +492,7 @@ fi # if [ "${CPL_AQM}" = "TRUE" ] && [ "${PREDEF_GRID_NAME}" = "AQM_NA_13km" ]; then python3 $USHdir/update_input_nml.py \ - --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run_dir "${DATA}" \ + --namelist "${DATA}/${FV3_NML_FN}" \ --aqm_na_13km || print_err_msg_exit "\ Call to function to update the FV3 input.nml file for air quality modeling using AQM_NA_13km for the current cycle's (cdate) run directory (DATA) failed: @@ -520,8 +519,7 @@ if [ "${DO_FCST_RESTART}" = "TRUE" ] && [ "$(ls -A ${DATA}/RESTART )" ]; then # Update FV3 input.nml for restart python3 $USHdir/update_input_nml.py \ - --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ - --run_dir "${DATA}" \ + --namelist "${DATA}/${FV3_NML_FN}" \ --restart export err=$? if [ $err -ne 0 ]; then From 2a112754fca0cce3f62089cea533456b3180da65 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 8 Feb 2024 20:59:32 +0000 Subject: [PATCH 13/30] WIP --- ush/generate_FV3LAM_wflow.py | 18 ++++++++---------- ush/set_fv3nml_ens_stoch_seeds.py | 2 +- ush/set_fv3nml_sfc_climo_filenames.py | 6 +++--- ush/update_input_nml.py | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index d2d291049b..3f98c4be2c 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -14,7 +14,7 @@ from subprocess import STDOUT, CalledProcessError, check_output from textwrap import dedent -from uwtools.api.config import get_yaml_config, realize +from uwtools.api.config import get_nml_config, get_yaml_config, realize from python_utils import ( log_info, @@ -513,14 +513,12 @@ def generate_FV3LAM_wflow( # physics_cfg = get_yaml_config(FV3_NML_YAML_CONFIG_FP) - realize( - input_config=FV3_NML_BASE_SUITE_FP, - input_format="nml", - output_file=FV3_NML_FP, - output_format="nml", - supplemental_configs=physics_cfg[CCPP_PHYS_SUITE], - ) - + base_namelist = get_nml_config(FV3_NML_BASE_SUITE_FP) + base_namelist.update_values(physics_cfg[CCPP_PHYS_SUITE]) + for sect, values in base_namelist.copy().items(): + if not values: + del base_namelist[sect] + base_namelist.dump(FV3_NML_FP) # # If not running the TN_MAKE_GRID task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying @@ -654,7 +652,7 @@ def generate_FV3LAM_wflow( input_format="nml", output_file=FV3_NML_STOCH_FP, output_format="nml", - supplemental_configs=settings, + supplemental_configs=[settings], ) # diff --git a/ush/set_fv3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py index 1d60cb09ac..2aeb2b7dcd 100644 --- a/ush/set_fv3nml_ens_stoch_seeds.py +++ b/ush/set_fv3nml_ens_stoch_seeds.py @@ -110,7 +110,7 @@ def set_fv3nml_ens_stoch_seeds(cdate, expt_config): input_format="nml", output_file=fv3_nml_ensmem_fp, output_format="nml", - supplemental_configs=settings, + supplemental_configs=[settings], ) def parse_args(argv): diff --git a/ush/set_fv3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py index bd7f7daba0..c5ed5ee0f0 100644 --- a/ush/set_fv3nml_sfc_climo_filenames.py +++ b/ush/set_fv3nml_sfc_climo_filenames.py @@ -21,7 +21,7 @@ VERBOSE = os.environ.get("VERBOSE", "true") -needed_vars = [ +NEEDED_VARS = [ "CRES", "DO_ENSEMBLE", "EXPTDIR", @@ -31,7 +31,6 @@ "RUN_ENVIR", ] -import_vars(env_vars=needed_vars) # pylint: disable=undefined-variable @@ -52,6 +51,7 @@ def set_fv3nml_sfc_climo_filenames(debug=False): """ # import all environment variables + import_vars(env_vars=NEEDED_VARS) fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] @@ -104,7 +104,7 @@ def set_fv3nml_sfc_climo_filenames(debug=False): input_format="nml", output_file=FV3_NML_FP, output_format="nml", - supplemental_configs=settings, + supplemental_configs=[settings], ) def parse_args(argv): diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index fa412a636d..45532e79e4 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -74,7 +74,7 @@ def update_input_nml(namelist, restart, aqm_na_13km): input_format="nml", output_file=namelist, output_format="nml", - supplemental_configs=settings, + supplemental_configs=[settings], ) def parse_args(argv): From 92010a3ec20bb8126fc578c57637c2efde7d76f4 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Thu, 8 Feb 2024 21:52:03 +0000 Subject: [PATCH 14/30] [develop] Add native entry in aqm_prep.yaml for nexus_gfs_sfc --- parm/wflow/aqm_prep.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/wflow/aqm_prep.yaml b/parm/wflow/aqm_prep.yaml index 36d2c8af2a..6cfab161d7 100644 --- a/parm/wflow/aqm_prep.yaml +++ b/parm/wflow/aqm_prep.yaml @@ -23,6 +23,7 @@ default_aqm_task: &default_aqm task_nexus_gfs_sfc: <<: *default_aqm command: '&LOAD_MODULES_RUN_TASK_FP; "nexus_gfs_sfc" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC"' + native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;' memory: 2G From 6dc364f195d0e8538506db971443c276e59081cc Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 27 Feb 2024 15:57:47 +0000 Subject: [PATCH 15/30] hanges for new version and linted ush --- scripts/exregional_make_grid.sh | 6 +++--- scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_run_fcst.sh | 2 +- scripts/exregional_run_met_genensprod_or_ensemblestat.sh | 2 +- scripts/exregional_run_met_gridstat_or_pointstat_vx.sh | 2 +- .../exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh | 2 +- .../exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh | 2 +- scripts/exregional_run_met_pb2nc_obs.sh | 2 +- scripts/exregional_run_met_pcpcombine.sh | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 6823c2581c..7a24c6868d 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -285,7 +285,7 @@ EOF -i ${tmpfile} \ -o ${rgnl_grid_nml_fp} \ -v \ - --values-file ${tmpfile} + ${tmpfile} err=$? if [ $err -ne 0 ]; then @@ -617,7 +617,7 @@ failed." # #----------------------------------------------------------------------- # -# Call a function (set_FV3nml_sfc_climo_filenames) to set the values of +# Call a function (set_fv3nml_sfc_climo_filenames) to set the values of # those variables in the forecast model's namelist file that specify the # paths to the surface climatology files. These files will either already # be avaialable in a user-specified directory (SFC_CLIMO_DIR) or will be @@ -626,7 +626,7 @@ failed." # #----------------------------------------------------------------------- # -python3 $USHdir/set_FV3nml_sfc_climo_filenames.py \ +python3 $USHdir/set_fv3nml_sfc_climo_filenames.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 6a358ba511..65c5c9a973 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -602,7 +602,7 @@ EOF -i ${tmpfile} \ -o ${nml_fn} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? if [ $err -ne 0 ]; then diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 1666ec9723..83d8cda8bb 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -511,7 +511,7 @@ EOF -i ${tmpfile} \ -o ${nml_fn} \ -v \ - --values-file ${tmpfile} + ${tmpfile} export err=$? if [ $err -ne 0 ]; then diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index e1af7a3bb7..27b0efbeec 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -474,7 +474,7 @@ fi #----------------------------------------------------------------------- # if ([ "$STOCH" == "TRUE" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]); then - python3 $USHdir/set_FV3nml_ens_stoch_seeds.py \ + python3 $USHdir/set_fv3nml_ens_stoch_seeds.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --cdate "$CDATE" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index 8fef9cfa21..507c7da39e 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -400,7 +400,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index c9d8baeefe..7b539ed977 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -401,7 +401,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 0ebff19961..bb91888540 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -359,7 +359,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index 06e7e22241..586dddab63 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -311,7 +311,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 1122df77ae..d55ad2ff44 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -285,7 +285,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 6db1ff16ba..ae309b959f 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -363,7 +363,7 @@ uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ -v \ - --values-file "${tmpfile}" + "${tmpfile}" err=$? rm $tmpfile From 1d431e95dacedb11e305bb6b1815d25bbfe5a009 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 6 Mar 2024 19:06:55 +0000 Subject: [PATCH 16/30] WIP --- scripts/exregional_make_ics.sh | 1 + scripts/exregional_make_lbcs.sh | 43 ++++++++++++++------------- ush/generate_FV3LAM_wflow.py | 2 +- ush/set_fv3nml_sfc_climo_filenames.py | 15 ++++++---- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 07e6b1241c..2fd8ac3624 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -601,6 +601,7 @@ EOF uw config realize \ -i ${tmpfile} \ -o ${nml_fn} \ + --output-format nml\ -v \ "${tmpfile}" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index a2b690ca6b..7ee390d4d5 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -477,27 +477,27 @@ FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_N # settings=" 'config': - 'fix_dir_target_grid': ${FIXlam}, - 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXlam}, - 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, - 'vcoord_file_target_grid': ${VCOORD_FILE}, - 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file}, - 'data_dir_input_grid': ${extrn_mdl_staging_dir}, - 'atm_files_input_grid': ${fn_atm}, - 'grib2_file_input_grid': \"${fn_grib2}\", - 'cycle_mon': $((10#${mm})), - 'cycle_day': $((10#${dd})), - 'cycle_hour': $((10#${hh})), - 'convert_atm': True, - 'regional': 2, - 'halo_bndy': $((10#${NH4})), - 'halo_blend': $((10#${HALO_BLEND})), - 'input_type': ${input_type}, - 'external_model': ${external_model}, - 'tracers_input': ${tracers_input}, - 'tracers': ${tracers}, - 'thomp_mp_climo_file': ${thomp_mp_climo_file}, + 'fix_dir_target_grid': ${FIXlam} + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc + 'orog_dir_target_grid': ${FIXlam} + 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc + 'vcoord_file_target_grid': ${VCOORD_FILE} + 'varmap_file': ${PARMdir}/ufs_utils/varmap_tables/${varmap_file} + 'data_dir_input_grid': ${extrn_mdl_staging_dir} + 'atm_files_input_grid': ${fn_atm} + 'grib2_file_input_grid': \"${fn_grib2}\" + 'cycle_mon': $((10#${mm})) + 'cycle_day': $((10#${dd})) + 'cycle_hour': $((10#${hh})) + 'convert_atm': True + 'regional': 2 + 'halo_bndy': $((10#${NH4})) + 'halo_blend': $((10#${HALO_BLEND})) + 'input_type': ${input_type} + 'external_model': ${external_model} + 'tracers_input': ${tracers_input} + 'tracers': ${tracers} + 'thomp_mp_climo_file': ${thomp_mp_climo_file} " # Store the settings in a temporary file @@ -510,6 +510,7 @@ EOF uw config realize \ -i ${tmpfile} \ -o ${nml_fn} \ + --output-format nml \ -v \ ${tmpfile} diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 3f98c4be2c..5238c567a5 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -533,7 +533,7 @@ def generate_FV3LAM_wflow( # if not expt_config['rocoto']['tasks'].get('task_make_grid'): - set_fv3nml_sfc_climo_filenames(debug) + set_fv3nml_sfc_climo_filenames(flatten_dict(expt_config), debug) # # ----------------------------------------------------------------------- diff --git a/ush/set_fv3nml_sfc_climo_filenames.py b/ush/set_fv3nml_sfc_climo_filenames.py index c5ed5ee0f0..417aa0b5ee 100644 --- a/ush/set_fv3nml_sfc_climo_filenames.py +++ b/ush/set_fv3nml_sfc_climo_filenames.py @@ -13,10 +13,12 @@ from uwtools.api.config import get_yaml_config, realize from python_utils import ( - print_info_msg, + cfg_to_yaml_str, check_var_valid_value, + flatten_dict, import_vars, - cfg_to_yaml_str, + load_shell_config, + print_info_msg, ) VERBOSE = os.environ.get("VERBOSE", "true") @@ -34,7 +36,7 @@ # pylint: disable=undefined-variable -def set_fv3nml_sfc_climo_filenames(debug=False): +def set_fv3nml_sfc_climo_filenames(config, debug=False): """ This function sets the values of the variables in the forecast model's namelist file that specify the paths to the surface @@ -50,8 +52,7 @@ def set_fv3nml_sfc_climo_filenames(debug=False): None """ - # import all environment variables - import_vars(env_vars=NEEDED_VARS) + import_vars(dictionary=config, env_vars=NEEDED_VARS) fixed_cfg = get_yaml_config(os.path.join(PARMdir, "fixed_files_mapping.yaml"))["fixed_files"] @@ -126,4 +127,6 @@ def parse_args(argv): if __name__ == "__main__": args = parse_args(sys.argv[1:]) - set_fv3nml_sfc_climo_filenames(args.debug) + cfg = load_shell_config(args.path_to_defns) + cfg = flatten_dict(cfg) + set_fv3nml_sfc_climo_filenames(cfg, args.debug) From b7c9bf3deeedf8190ee944333407014ec5d67dd8 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 11 Mar 2024 01:30:43 +0000 Subject: [PATCH 17/30] Update CLI args for new uwtools version --- ...onal_run_met_genensprod_or_ensemblestat.sh | 4 +- ...gional_run_met_gridstat_or_pointstat_vx.sh | 4 +- ...un_met_gridstat_or_pointstat_vx_ensmean.sh | 4 +- ...un_met_gridstat_or_pointstat_vx_ensprob.sh | 4 +- scripts/exregional_run_met_pb2nc_obs.sh | 60 +++++++++---------- scripts/exregional_run_met_pcpcombine.sh | 4 +- ush/generate_FV3LAM_wflow.py | 5 ++ 7 files changed, 45 insertions(+), 40 deletions(-) diff --git a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh index a527c23aa2..fe0e119b19 100755 --- a/scripts/exregional_run_met_genensprod_or_ensemblestat.sh +++ b/scripts/exregional_run_met_genensprod_or_ensemblestat.sh @@ -399,8 +399,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh index a89fd7e555..7eb1ce4605 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx.sh @@ -400,8 +400,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh index 4375b0dca0..458dcec33f 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensmean.sh @@ -358,8 +358,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh index c81a0f2724..fc735845c9 100755 --- a/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh +++ b/scripts/exregional_run_met_gridstat_or_pointstat_vx_ensprob.sh @@ -310,8 +310,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 9998fea51a..9782501e52 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -233,46 +233,46 @@ settings="\ # # MET/METplus information. # - 'metplus_tool_name': '${metplus_tool_name}' - 'MetplusToolName': '${MetplusToolName}' - 'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' - 'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' +metplus_tool_name: '${metplus_tool_name}' +MetplusToolName: '${MetplusToolName}' +METPLUS_TOOL_NAME: '${METPLUS_TOOL_NAME}' +metplus_verbosity_level: '${METPLUS_VERBOSITY_LEVEL}' # # Date and forecast hour information. # - 'cdate': '$CDATE' - 'fhr_list': '${FHR_LIST}' +cdate: '$CDATE' +fhr_list: '${FHR_LIST}' # # Input and output directory/file information. # - 'metplus_config_fn': '${metplus_config_fn:-}' - 'metplus_log_fn': '${metplus_log_fn:-}' - 'obs_input_dir': '${OBS_INPUT_DIR:-}' - 'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' - 'fcst_input_dir': '${FCST_INPUT_DIR:-}' - 'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' - 'output_base': '${OUTPUT_BASE}' - 'output_dir': '${OUTPUT_DIR}' - 'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' - 'staging_dir': '${STAGING_DIR}' - 'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' +metplus_config_fn: '${metplus_config_fn:-}' +metplus_log_fn: '${metplus_log_fn:-}' +obs_input_dir: '${OBS_INPUT_DIR:-}' +obs_input_fn_template: '${OBS_INPUT_FN_TEMPLATE:-}' +fcst_input_dir: '${FCST_INPUT_DIR:-}' +fcst_input_fn_template: '${FCST_INPUT_FN_TEMPLATE:-}' +output_base: '${OUTPUT_BASE}' +output_dir: '${OUTPUT_DIR}' +output_fn_template: '${OUTPUT_FN_TEMPLATE:-}' +staging_dir: '${STAGING_DIR}' +vx_fcst_model_name: '${VX_FCST_MODEL_NAME}' # # Ensemble and member-specific information. # - 'num_ens_members': '${NUM_ENS_MEMBERS}' - 'ensmem_name': '${ensmem_name:-}' - 'time_lag': '${time_lag:-}' +num_ens_members: '${NUM_ENS_MEMBERS}' +ensmem_name: '${ensmem_name:-}' +time_lag: '${time_lag:-}' # # Field information. # - 'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' - 'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' - 'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' - 'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' - 'obtype': '${OBTYPE}' - 'accum_hh': '${ACCUM_HH:-}' - 'accum_no_pad': '${ACCUM_NO_PAD:-}' - 'field_thresholds': '${FIELD_THRESHOLDS:-}' +fieldname_in_obs_input: '${FIELDNAME_IN_OBS_INPUT}' +fieldname_in_fcst_input: '${FIELDNAME_IN_FCST_INPUT}' +fieldname_in_met_output: '${FIELDNAME_IN_MET_OUTPUT}' +fieldname_in_met_filedir_names: '${FIELDNAME_IN_MET_FILEDIR_NAMES}' +obtype: '${OBTYPE}' +accum_hh: '${ACCUM_HH:-}' +accum_no_pad: '${ACCUM_NO_PAD:-}' +field_thresholds: '${FIELD_THRESHOLDS:-}' " # Render the template to create a METplus configuration file @@ -284,8 +284,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/scripts/exregional_run_met_pcpcombine.sh b/scripts/exregional_run_met_pcpcombine.sh index 523e4ef175..7eabe02901 100755 --- a/scripts/exregional_run_met_pcpcombine.sh +++ b/scripts/exregional_run_met_pcpcombine.sh @@ -362,8 +362,8 @@ EOF uw template render \ -i ${metplus_config_tmpl_fp} \ -o ${metplus_config_fp} \ - -v \ - "${tmpfile}" + --verbose \ + --values-file "${tmpfile}" err=$? rm $tmpfile diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 5238c567a5..ec2b95c3f3 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -515,9 +515,14 @@ def generate_FV3LAM_wflow( physics_cfg = get_yaml_config(FV3_NML_YAML_CONFIG_FP) base_namelist = get_nml_config(FV3_NML_BASE_SUITE_FP) base_namelist.update_values(physics_cfg[CCPP_PHYS_SUITE]) + base_namelist.update_values(settings) for sect, values in base_namelist.copy().items(): if not values: del base_namelist[sect] + continue + for k, v in values.copy().items(): + if v is None: + del base_namelist[sect][k] base_namelist.dump(FV3_NML_FP) # # If not running the TN_MAKE_GRID task (which implies the workflow will From c166a7788cf2ef14479457435521dd2118dba161 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 17:21:19 +0000 Subject: [PATCH 18/30] Update uwtools version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 6fe2548da9..faeb19d466 100644 --- a/environment.yml +++ b/environment.yml @@ -5,4 +5,4 @@ channels: dependencies: - pylint=2.17* - pytest=7.2* - - uwtools=2.0.0 + - uwtools=2.1* From ca2a2680a107d05ed9f55b356d5e2dbb84343c3c Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 17:39:53 +0000 Subject: [PATCH 19/30] Add new linted ush files --- .github/workflows/python_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index 0e71f8d72d..26957281c2 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -41,6 +41,7 @@ jobs: pylint --ignore-imports=yes tests/test_python/ pylint ush/create_*.py pylint ush/generate_FV3LAM_wflow.py + pylint ush/set_fv3nml*.py - name: Run python unittests run: | From 8eedbd285eace59ab45f981e77fdc44969a1e49e Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 18:05:41 +0000 Subject: [PATCH 20/30] Remove debugging additions. --- ush/load_modules_wflow.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ush/load_modules_wflow.sh b/ush/load_modules_wflow.sh index 1d2ee71e15..cf33a43f3f 100755 --- a/ush/load_modules_wflow.sh +++ b/ush/load_modules_wflow.sh @@ -31,7 +31,7 @@ if [ "$1" == "--help" ] || [ "$1" == "-h" ]; then usage exit 0 fi -set -x + # Set machine name to lowercase machine=${1,,} @@ -49,10 +49,8 @@ fi # Source modulefile for this machine WFLOW_MOD_FN="wflow_${machine}" -set +x source "${HOMEdir}/etc/lmod-setup.sh" ${machine} module use "${HOMEdir}/modulefiles" -echo MODPATH $MODULEPATH module load "${WFLOW_MOD_FN}" > /dev/null 2>&1 || { echo "ERROR: Loading of platform-specific module file (WFLOW_MOD_FN) for the workflow task failed: From d379eed1a970bdd87a2de3a0e98f5966737f534b Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 18:35:51 +0000 Subject: [PATCH 21/30] Fixing some doc strings in additional linted file. --- .github/workflows/python_tests.yaml | 1 + ush/update_input_nml.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_tests.yaml b/.github/workflows/python_tests.yaml index 26957281c2..fb0de16910 100644 --- a/.github/workflows/python_tests.yaml +++ b/.github/workflows/python_tests.yaml @@ -42,6 +42,7 @@ jobs: pylint ush/create_*.py pylint ush/generate_FV3LAM_wflow.py pylint ush/set_fv3nml*.py + pylint ush/update_input_nml.py - name: Run python unittests run: | diff --git a/ush/update_input_nml.py b/ush/update_input_nml.py index 45532e79e4..e975d9bc08 100644 --- a/ush/update_input_nml.py +++ b/ush/update_input_nml.py @@ -23,7 +23,10 @@ def update_input_nml(namelist, restart, aqm_na_13km): """Update the FV3 input.nml file in the specified run directory Args: - namelist: path to the namelist + namelist: path to the namelist + restart: should forecast start from restart? + aqm_na_13km: should the 13km AQM config be used? + Returns: Boolean """ @@ -85,7 +88,7 @@ def parse_args(argv): "-n", "--namelist", dest="namelist", required=True, - help="Run directory.", + help="Path to namelist to update.", ) parser.add_argument( From aa38beb01b1c8e3d0165af463f898b0446450ee5 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 18:41:05 +0000 Subject: [PATCH 22/30] Linting tests. --- ...ml_ens_stoch_seeds.py => test_set_fv3nml_ens_stoch_seeds.py} | 2 +- ...limo_filenames.py => test_set_fv3nml_sfc_climo_filenames.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/test_python/{test_set_FV3nml_ens_stoch_seeds.py => test_set_fv3nml_ens_stoch_seeds.py} (97%) rename tests/test_python/{test_set_FV3nml_sfc_climo_filenames.py => test_set_fv3nml_sfc_climo_filenames.py} (96%) diff --git a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py similarity index 97% rename from tests/test_python/test_set_FV3nml_ens_stoch_seeds.py rename to tests/test_python/test_set_fv3nml_ens_stoch_seeds.py index f87d57d53b..3eda0a32c1 100644 --- a/tests/test_python/test_set_FV3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py @@ -15,7 +15,7 @@ set_env_var, ) -from set_FV3nml_ens_stoch_seeds import set_FV3nml_ens_stoch_seeds +from set_fv3nml_ens_stoch_seeds import set_fv3nml_ens_stoch_seeds class Testing(unittest.TestCase): """ Define the tests """ diff --git a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py similarity index 96% rename from tests/test_python/test_set_FV3nml_sfc_climo_filenames.py rename to tests/test_python/test_set_fv3nml_sfc_climo_filenames.py index 131af70506..42889b60b5 100644 --- a/tests/test_python/test_set_FV3nml_sfc_climo_filenames.py +++ b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py @@ -12,7 +12,7 @@ mkdir_vrfy, set_env_var, ) -from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames +from set_fv3nml_sfc_climo_filenames import set_fv3nml_sfc_climo_filenames class Testing(unittest.TestCase): """ Define the tests """ From b866c688d95b3de368ef55e22f0bb56398c1d461 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 18:44:09 +0000 Subject: [PATCH 23/30] Missed the updated name calls. --- tests/test_python/test_set_fv3nml_ens_stoch_seeds.py | 6 +++--- tests/test_python/test_set_fv3nml_sfc_climo_filenames.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py index 3eda0a32c1..6d7bad5571 100644 --- a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py @@ -1,4 +1,4 @@ -""" Tests for set_FV3nml_ens_stoch_seeds.py """ +""" Tests for set_fv3nml_ens_stoch_seeds.py """ #pylint: disable=invalid-name @@ -19,10 +19,10 @@ class Testing(unittest.TestCase): """ Define the tests """ - def test_set_FV3nml_ens_stoch_seeds(self): + def test_set_fv3nml_ens_stoch_seeds(self): """ Call the function and make sure it doesn't fail""" os.chdir(self.mem_dir) - set_FV3nml_ens_stoch_seeds(cdate=self.cdate) + set_fv3nml_ens_stoch_seeds(cdate=self.cdate) def setUp(self): define_macos_utilities() diff --git a/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py index 42889b60b5..c5483a3aa1 100644 --- a/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py +++ b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py @@ -1,4 +1,4 @@ -""" Tests for set_FV3nml_sfc_climo_filenames.py """ +""" Tests for set_fv3nml_sfc_climo_filenames.py """ #pylint: disable=invalid-name @@ -16,9 +16,9 @@ class Testing(unittest.TestCase): """ Define the tests """ - def test_set_FV3nml_sfc_climo_filenames(self): + def test_set_fv3nml_sfc_climo_filenames(self): """ Call the function and don't raise an Exception. """ - set_FV3nml_sfc_climo_filenames() + set_fv3nml_sfc_climo_filenames() def setUp(self): define_macos_utilities() From 67d827f047fa1d5e4ea827cbf30df5e36deae6b7 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Mar 2024 20:35:18 +0000 Subject: [PATCH 24/30] Fix unlinted tests and make them pass. --- .../test_set_fv3nml_ens_stoch_seeds.py | 23 ++++++++++--------- .../test_set_fv3nml_sfc_climo_filenames.py | 18 ++++++++------- ush/set_fv3nml_ens_stoch_seeds.py | 1 + 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py index 6d7bad5571..5f0eeb0c5a 100644 --- a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py @@ -22,11 +22,10 @@ class Testing(unittest.TestCase): def test_set_fv3nml_ens_stoch_seeds(self): """ Call the function and make sure it doesn't fail""" os.chdir(self.mem_dir) - set_fv3nml_ens_stoch_seeds(cdate=self.cdate) + set_fv3nml_ens_stoch_seeds(cdate=self.cdate, expt_config=self.config) def setUp(self): define_macos_utilities() - set_env_var("DEBUG", True) set_env_var("VERBOSE", True) self.cdate = datetime(2021, 1, 1) test_dir = os.path.dirname(os.path.abspath(__file__)) @@ -55,17 +54,19 @@ def setUp(self): ) - set_env_var("USHdir", USHdir) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") - set_env_var("FV3_NML_FP", os.path.join(self.mem_dir, "input.nml")) - set_env_var("DO_SHUM", True) - set_env_var("DO_SKEB", True) - set_env_var("DO_SPPT", True) - set_env_var("DO_SPP", True) - set_env_var("DO_LSM_SPP", True) - ISEED_SPP = [4, 5, 6, 7, 8] - set_env_var("ISEED_SPP", ISEED_SPP) + + self.config = { + "global": { + "DO_SHUM": True, + "DO_SKEB": True, + "DO_SPPT": True, + "DO_SPP": True, + "DO_LSM_SPP": True, + "ISEED_SPP": [4, 5, 6, 7, 8], + } + } def tearDown(self): self.tmp_dir.cleanup() diff --git a/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py index c5483a3aa1..b0daf50fea 100644 --- a/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py +++ b/tests/test_python/test_set_fv3nml_sfc_climo_filenames.py @@ -18,7 +18,7 @@ class Testing(unittest.TestCase): """ Define the tests """ def test_set_fv3nml_sfc_climo_filenames(self): """ Call the function and don't raise an Exception. """ - set_fv3nml_sfc_climo_filenames() + set_fv3nml_sfc_climo_filenames(config=self.config) def setUp(self): define_macos_utilities() @@ -42,13 +42,15 @@ def setUp(self): os.path.join(PARMdir, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - set_env_var("PARMdir", PARMdir) - set_env_var("EXPTDIR", EXPTDIR) - set_env_var("FIXlam", FIXlam) - set_env_var("DO_ENSEMBLE", False) - set_env_var("CRES", "C3357") - set_env_var("RUN_ENVIR", "nco") - set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) + self.config = { + "CRES": "C3357", + "DO_ENSEMBLE": False, + "EXPTDIR": EXPTDIR, + "FIXlam": FIXlam, + "FV3_NML_FP": os.path.join(EXPTDIR, "input.nml"), + "PARMdir": PARMdir, + "RUN_ENVIR": "nco", + } def tearDown(self): self.tmp_dir.cleanup() diff --git a/ush/set_fv3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py index 2aeb2b7dcd..7390e9fde2 100644 --- a/ush/set_fv3nml_ens_stoch_seeds.py +++ b/ush/set_fv3nml_ens_stoch_seeds.py @@ -20,6 +20,7 @@ ) VERBOSE = os.environ.get("VERBOSE", "true") +FV3_NML_FN = os.environ.get("FV3_NML_FN", "input.nml") def set_fv3nml_ens_stoch_seeds(cdate, expt_config): From ea605c0c511e1dcaad7cd9bbbb07e0d9f05a1e67 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 14 Mar 2024 15:32:51 +0000 Subject: [PATCH 25/30] Changes from Mike K's PR comments. --- parm/FV3.input.yml | 140 ++++++++++++------------ scripts/exregional_make_ics.sh | 8 +- scripts/exregional_make_lbcs.sh | 8 +- scripts/exregional_run_met_pb2nc_obs.sh | 58 +++++----- 4 files changed, 101 insertions(+), 113 deletions(-) diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index 3fb97f7d07..e96a181882 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -78,11 +78,11 @@ FV3_HRRR: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: + do_sfcperts: null gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True - do_gsl_drag_ls_bl: True + do_gsl_drag_ls_bl: True iaer: 5111 icliq_sw: 2 iovr: 3 @@ -97,8 +97,8 @@ FV3_HRRR: mosaic_lu: 0 mosaic_soil: 0 thsfc_loc: False - nst_anl: - nstf_name: + nst_anl: null + nstf_name: null FV3_RAP: fv_core_nml: @@ -107,7 +107,7 @@ FV3_RAP: <<: *RRFS_v1beta_phys cdmbgwd: [3.5, 1.0] do_mynnsfclay: True - do_sfcperts: + do_sfcperts: null gwd_opt: 3 do_gsl_drag_ss: True do_gsl_drag_tofd: True @@ -135,40 +135,40 @@ FV3_GFS_2017_gfdlmp: k_split: 6 n_split: 6 nord: 2 - nord_zs_filter: + nord_zs_filter: null range_warn: False vtdm4: 0.075 gfs_physics_nml: &gfs_2017_gfdlmp_phys avg_max_length: 3600.0 - bl_mynn_tkeadvect: - bl_mynn_edmf: - bl_mynn_edmf_mom: + bl_mynn_tkeadvect: null + bl_mynn_edmf: null + bl_mynn_edmf_mom: null cdmbgwd: [3.5, 0.01] - cplflx: + cplflx: null do_deep: False - do_mynnedmf: - do_mynnsfclay: + do_mynnedmf: null + do_mynnsfclay: null fhcyc: 0.0 fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - icloud_bl: + icloud_bl: null imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 lgfdlmprad: True - lheatstrg: - lndp_type: - lsm: - lsoil: - lsoil_lsm: - ltaerosol: - n_var_lndp: + lheatstrg: null + lndp_type: null + lsm: null + lsoil: null + lsoil_lsm: null + ltaerosol: null + n_var_lndp: null oz_phys: True oz_phys_2015: False - satmedmf: + satmedmf: null shal_cnv: True - ttendlim: + ttendlim: null gfdl_cloud_microphysics_nml: &gfs_gfdl_cloud_mp c_cracw: 0.8 c_paut: 0.5 @@ -267,7 +267,7 @@ FV3_GFS_v15p2: kord_wz: 9 n_split: 8 n_sponge: 30 - nord_zs_filter: + nord_zs_filter: null nudge_qv: True range_warn: False rf_cutoff: 750.0 @@ -278,16 +278,16 @@ FV3_GFS_v15p2: tau_l2v: 225.0 tau_v2l: 150.0 gfs_physics_nml: &gfs_v15_gfs_physics - bl_mynn_edmf: - bl_mynn_edmf_mom: - bl_mynn_tkeadvect: + bl_mynn_edmf: null + bl_mynn_edmf_mom: null + bl_mynn_tkeadvect: null cnvcld: True cnvgwd: True - cplflx: + cplflx: null do_myjpbl: False do_myjsfc: False - do_mynnedmf: - do_mynnsfclay: + do_mynnedmf: null + do_mynnsfclay: null do_tofd: False do_ugwp: False do_ysu: False @@ -295,12 +295,12 @@ FV3_GFS_v15p2: fhlwr: 3600.0 fhswr: 3600.0 hybedmf: True - iau_delthrs: - iaufhrs: + iau_delthrs: null + iaufhrs: null imfdeepcnv: 2 imfshalcnv: 2 imp_physics: 11 - icloud_bl: + icloud_bl: null iopt_alb: 2 iopt_btr: 1 iopt_crs: 1 @@ -316,28 +316,28 @@ FV3_GFS_v15p2: iopt_trs: 2 ldiag_ugwp: False lgfdlmprad: True - lradar: + lradar: null lsm: 1 - lsoil: - lsoil_lsm: - ltaerosol: + lsoil: null + lsoil_lsm: null + ltaerosol: null shal_cnv: True shinhong: False - ttendlim: + ttendlim: null xkzm_h: 1.0 xkzm_m: 1.0 xkzminv: 0.3 namsfc: landice: True ldebug: False - surf_map_nml: + surf_map_nml: null FV3_GFS_v15_thompson_mynn_lam3km: atmos_model_nml: avg_max_length: 3600.0 fv_core_nml: agrid_vel_rst: True - full_zs_filter: + full_zs_filter: null n_sponge: 9 npz_type: '' rf_fast: False @@ -377,20 +377,20 @@ FV3_GFS_v15_thompson_mynn_lam3km: iopt_snf: 4 iopt_stc: 1 iopt_tbot: 2 - iopt_trs: + iopt_trs: null iovr: 3 ldiag_ugwp: False lgfdlmprad: False lsm: 1 - lsoil: - lsoil_lsm: + lsoil: null + lsoil_lsm: null ltaerosol: False print_diff_pgr: True - sfclay_compute_flux: + sfclay_compute_flux: null xkzminv: 0.3 xkzm_m: 1.0 xkzm_h: 1.0 - surf_map_nml: + surf_map_nml: null FV3_GFS_v16: cires_ugwp_nml: @@ -414,7 +414,7 @@ FV3_GFS_v16: na_init: 0 nudge_dz: False res_latlon_dynamics: '' - rf_fast: + rf_fast: null tau: 10.0 gfdl_cloud_microphysics_nml: <<: *gfs_gfdl_cloud_mp @@ -426,10 +426,10 @@ FV3_GFS_v16: gfs_physics_nml: <<: *gfs_v15_gfs_physics cdmbgwd: [4.0, 0.15, 1.0, 1.0] - do_myjpbl: - do_myjsfc: + do_myjpbl: null + do_myjsfc: null do_tofd: True - do_ysu: + do_ysu: null hybedmf: False iaer: 5111 icliq_sw: 2 @@ -438,23 +438,23 @@ FV3_GFS_v16: isatmedmf: 1 lgfdlmprad: True lheatstrg: True - lndp_type: + lndp_type: null lsoil: 4 - n_var_lndp: + n_var_lndp: null prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] satmedmf: True - shinhong: - xkzminv: - xkzm_m: - xkzm_h: + shinhong: null + xkzminv: null + xkzm_m: null + xkzm_h: null mpp_io_nml: deflate_level: 1 shuffle: 1 namsfc: landice: True ldebug: False - surf_map_nml: + surf_map_nml: null FV3_GFS_v17_p8: cires_ugwp_nml: @@ -467,8 +467,8 @@ FV3_GFS_v17_p8: delt_max: 0.002 dnats: 0 do_sat_adj: False - do_vort_damp: - full_zs_filter: + do_vort_damp: null + full_zs_filter: null fv_sg_adj: 450 hord_dp: -5 hord_mt: 5 @@ -480,10 +480,10 @@ FV3_GFS_v17_p8: n_sponge: 10 nord: 2 nudge_dz: False - n_zs_filter: + n_zs_filter: null range_warn: True res_latlon_dynamics: '' - rf_fast: + rf_fast: null tau: 10.0 gfdl_cloud_microphysics_nml: <<: *gfs_gfdl_cloud_mp @@ -505,8 +505,8 @@ FV3_GFS_v17_p8: do_gsl_drag_ls_bl: False do_gsl_drag_ss: True do_gsl_drag_tofd: False - do_myjpbl: - do_myjsfc: + do_myjpbl: null + do_myjsfc: null do_mynnedmf: False do_mynnsfclay: False do_RRTMGP: False @@ -515,7 +515,7 @@ FV3_GFS_v17_p8: do_ugwp_v0_orog_only: False do_ugwp_v1: False do_ugwp_v1_orog_only: False - do_ysu: + do_ysu: null doGP_cldoptics_LUT: False doGP_lwscat: False dt_inner: 150 @@ -524,7 +524,7 @@ FV3_GFS_v17_p8: hybedmf: False iaer: 1011 ialb: 2 - iau_inc_files: + iau_inc_files: null icloud_bl: 1 icliq_sw: 2 iems: 2 @@ -537,7 +537,7 @@ FV3_GFS_v17_p8: iopt_stc: 3 iovr: 3 isatmedmf: 1 - ldiag_ugwp: + ldiag_ugwp: null lgfdlmprad: False lradar: False lseaspray: True @@ -549,7 +549,7 @@ FV3_GFS_v17_p8: lw_file_gas: rrtmgp-data-lw-g128-210809.nc min_lakeice: 0.15 min_seaice: 0.15 - nsfullradar_diag: + nsfullradar_diag: null prautco: [0.00015, 0.00015] psautco: [0.0008, 0.0005] qdiag3d: False @@ -560,14 +560,14 @@ FV3_GFS_v17_p8: rrtmgp_nGptsSW: 112 satmedmf: True sedi_semi: True - sfclay_compute_flux: - shinhong: + sfclay_compute_flux: null + shinhong: null sw_file_clouds: rrtmgp-cloud-optics-coeffs-sw.nc sw_file_gas: rrtmgp-data-sw-g112-210809.nc ttendlim: -999 - xkzminv: - xkzm_m: - xkzm_h: + xkzminv: null + xkzm_m: null + xkzm_h: null mpp_io_nml: deflate_level: 1 shuffle: 1 diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 2fd8ac3624..2414e36faf 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -546,13 +546,7 @@ fi # IMPORTANT: # If we want a namelist variable to be removed from the namelist file, # in the "settings" variable below, we need to set its value to the -# string "null". This is equivalent to setting its value to -# !!python/none -# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the -# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. -# -# It turns out that setting the variable to an empty string also works -# to remove it from the namelist! Which is better to use?? +# string "null". # settings=" 'config': diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7ee390d4d5..7066713d9a 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -467,13 +467,7 @@ FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_N # IMPORTANT: # If we want a namelist variable to be removed from the namelist file, # in the "settings" variable below, we need to set its value to the -# string "null". This is equivalent to setting its value to -# !!python/none -# in the base namelist file specified by FV3_NML_BASE_SUITE_FP or the -# suite-specific yaml settings file specified by FV3_NML_YAML_CONFIG_FP. -# -# It turns out that setting the variable to an empty string also works -# to remove it from the namelist! Which is better to use?? +# string "null". # settings=" 'config': diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 9782501e52..30f6a93550 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # #----------------------------------------------------------------------- @@ -233,46 +233,46 @@ settings="\ # # MET/METplus information. # -metplus_tool_name: '${metplus_tool_name}' -MetplusToolName: '${MetplusToolName}' -METPLUS_TOOL_NAME: '${METPLUS_TOOL_NAME}' -metplus_verbosity_level: '${METPLUS_VERBOSITY_LEVEL}' + 'metplus_tool_name': '${metplus_tool_name}' + 'MetplusToolName': '${MetplusToolName}' + 'METPLUS_TOOL_NAME': '${METPLUS_TOOL_NAME}' + 'metplus_verbosity_level': '${METPLUS_VERBOSITY_LEVEL}' # # Date and forecast hour information. # -cdate: '$CDATE' -fhr_list: '${FHR_LIST}' + 'cdate': '$CDATE' + 'fhr_list': '${FHR_LIST}' # # Input and output directory/file information. # -metplus_config_fn: '${metplus_config_fn:-}' -metplus_log_fn: '${metplus_log_fn:-}' -obs_input_dir: '${OBS_INPUT_DIR:-}' -obs_input_fn_template: '${OBS_INPUT_FN_TEMPLATE:-}' -fcst_input_dir: '${FCST_INPUT_DIR:-}' -fcst_input_fn_template: '${FCST_INPUT_FN_TEMPLATE:-}' -output_base: '${OUTPUT_BASE}' -output_dir: '${OUTPUT_DIR}' -output_fn_template: '${OUTPUT_FN_TEMPLATE:-}' -staging_dir: '${STAGING_DIR}' -vx_fcst_model_name: '${VX_FCST_MODEL_NAME}' + 'metplus_config_fn': '${metplus_config_fn:-}' + 'metplus_log_fn': '${metplus_log_fn:-}' + 'obs_input_dir': '${OBS_INPUT_DIR:-}' + 'obs_input_fn_template': '${OBS_INPUT_FN_TEMPLATE:-}' + 'fcst_input_dir': '${FCST_INPUT_DIR:-}' + 'fcst_input_fn_template': '${FCST_INPUT_FN_TEMPLATE:-}' + 'output_base': '${OUTPUT_BASE}' + 'output_dir': '${OUTPUT_DIR}' + 'output_fn_template': '${OUTPUT_FN_TEMPLATE:-}' + 'staging_dir': '${STAGING_DIR}' + 'vx_fcst_model_name': '${VX_FCST_MODEL_NAME}' # # Ensemble and member-specific information. # -num_ens_members: '${NUM_ENS_MEMBERS}' -ensmem_name: '${ensmem_name:-}' -time_lag: '${time_lag:-}' + 'num_ens_members': '${NUM_ENS_MEMBERS}' + 'ensmem_name': '${ensmem_name:-}' + 'time_lag': '${time_lag:-}' # # Field information. # -fieldname_in_obs_input: '${FIELDNAME_IN_OBS_INPUT}' -fieldname_in_fcst_input: '${FIELDNAME_IN_FCST_INPUT}' -fieldname_in_met_output: '${FIELDNAME_IN_MET_OUTPUT}' -fieldname_in_met_filedir_names: '${FIELDNAME_IN_MET_FILEDIR_NAMES}' -obtype: '${OBTYPE}' -accum_hh: '${ACCUM_HH:-}' -accum_no_pad: '${ACCUM_NO_PAD:-}' -field_thresholds: '${FIELD_THRESHOLDS:-}' + 'fieldname_in_obs_input': '${FIELDNAME_IN_OBS_INPUT}' + 'fieldname_in_fcst_input': '${FIELDNAME_IN_FCST_INPUT}' + 'fieldname_in_met_output': '${FIELDNAME_IN_MET_OUTPUT}' + 'fieldname_in_met_filedir_names': '${FIELDNAME_IN_MET_FILEDIR_NAMES}' + 'obtype': '${OBTYPE}' + 'accum_hh': '${ACCUM_HH:-}' + 'accum_no_pad': '${ACCUM_NO_PAD:-}' + 'field_thresholds': '${FIELD_THRESHOLDS:-}' " # Render the template to create a METplus configuration file From 1d6cd8643e3b250b0eb132b3337ebc1ea0ffa2c2 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 14 Mar 2024 18:23:23 +0000 Subject: [PATCH 26/30] Use stdin instead of temporary files. --- scripts/exregional_make_grid.sh | 27 +++++++++------------ scripts/exregional_make_ics.sh | 42 +++++++++++++++------------------ scripts/exregional_make_lbcs.sh | 17 +++++-------- 3 files changed, 36 insertions(+), 50 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 8a03804bc3..1f95ea8f91 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -267,33 +267,28 @@ generation executable (exec_fp): # settings=" 'regional_grid_nml': - 'plon': ${LON_CTR} - 'plat': ${LAT_CTR} - 'delx': ${DEL_ANGLE_X_SG} - 'dely': ${DEL_ANGLE_Y_SG} - 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO} - 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO} - 'pazi': ${PAZI} + 'plon': ${LON_CTR} + 'plat': ${LAT_CTR} + 'delx': ${DEL_ANGLE_X_SG} + 'dely': ${DEL_ANGLE_Y_SG} + 'lx': ${NEG_NX_OF_DOM_WITH_WIDE_HALO} + 'ly': ${NEG_NY_OF_DOM_WITH_WIDE_HALO} + 'pazi': ${PAZI} " - # Store the settings in a temporary file - tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") - cat > $tmpfile << EOF + (cat << EOF $settings EOF - - uw config realize \ - -i ${tmpfile} \ +) | uw config realize \ + --input-format yaml \ -o ${rgnl_grid_nml_fp} \ -v \ - ${tmpfile} err=$? if [ $err -ne 0 ]; then - rm $tmpfile print_err_msg_exit "\ Error creating regional_esg_grid namelist. - Contents of input are: + Settings for input are: $settings" fi # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 2414e36faf..60852095ee 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -584,33 +584,29 @@ settings=" 'thomp_mp_climo_file': ${thomp_mp_climo_file} " - # Store the settings in a temporary file - tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") - cat > $tmpfile << EOF -$settings -EOF +nml_fn="fort.41" - nml_fn="fort.41" - uw config realize \ - -i ${tmpfile} \ - -o ${nml_fn} \ - --output-format nml\ - -v \ - "${tmpfile}" +(cat << EOF +$settings +EOF +) | uw config realize \ + --input-format yaml \ + -o ${nml_fn} \ + --output-format nml\ + -v \ - err=$? - if [ $err -ne 0 ]; then - message_txt="Error creating namelist read by ${exec_fn} failed. - Contents of input are: - $settings" - rm $tmpfile - if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then - err_exit "${message_txt}" - else - print_err_msg_exit "${message_txt}" - fi +err=$? +if [ $err -ne 0 ]; then + message_txt="Error creating namelist read by ${exec_fn} failed. + Settings for input are: +$settings" + if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then + err_exit "${message_txt}" + else + print_err_msg_exit "${message_txt}" fi +fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7066713d9a..fcde8e6f46 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -469,7 +469,7 @@ FORTRAN namelist file has not specified for this external LBC model (EXTRN_MDL_N # in the "settings" variable below, we need to set its value to the # string "null". # -settings=" + settings=" 'config': 'fix_dir_target_grid': ${FIXlam} 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc @@ -494,26 +494,21 @@ settings=" 'thomp_mp_climo_file': ${thomp_mp_climo_file} " - # Store the settings in a temporary file - tmpfile=$( $READLINK -f "$(mktemp ./namelist_settings.XXXXXX.yaml)") - cat > $tmpfile << EOF + nml_fn="fort.41" + (cat << EOF $settings EOF - - nml_fn="fort.41" - uw config realize \ - -i ${tmpfile} \ +) | uw config realize \ + --input-format yaml \ -o ${nml_fn} \ --output-format nml \ -v \ - ${tmpfile} export err=$? if [ $err -ne 0 ]; then message_txt="Error creating namelist read by ${exec_fn} failed. - Contents of input are: + Settings for input are: $settings" - rm $tmpfile if [ "${RUN_ENVIR}" = "nco" ] && [ "${MACHINE}" = "WCOSS2" ]; then err_exit "${message_txt}" else From 6256f1d66c2b6a42aaef4bd33a0a6c2e34844404 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 18 Mar 2024 14:50:54 +0000 Subject: [PATCH 27/30] Fixing accidental revert to /bin/bash --- scripts/exregional_run_met_pb2nc_obs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_run_met_pb2nc_obs.sh b/scripts/exregional_run_met_pb2nc_obs.sh index 30f6a93550..92d39102fc 100755 --- a/scripts/exregional_run_met_pb2nc_obs.sh +++ b/scripts/exregional_run_met_pb2nc_obs.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # #----------------------------------------------------------------------- From 6c020cfbb129254304eaa0df63c40376395a8603 Mon Sep 17 00:00:00 2001 From: "michael.lueken" Date: Tue, 19 Mar 2024 08:10:42 -0500 Subject: [PATCH 28/30] [replace_set_namelist] Update Hera spack-stack locations to use Rocky8 --- modulefiles/build_hera_gnu.lua | 5 +---- modulefiles/build_hera_intel.lua | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/modulefiles/build_hera_gnu.lua b/modulefiles/build_hera_gnu.lua index 90bd671b5a..d5f78f397b 100644 --- a/modulefiles/build_hera_gnu.lua +++ b/modulefiles/build_hera_gnu.lua @@ -5,10 +5,7 @@ the NOAA RDHPC machine Hera using GNU 9.2.0 whatis([===[Loads libraries needed for building the UFS SRW App on Hera using GNU 9.2.0 ]===]) --- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now, use line below ---prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") load("stack-gcc/9.2.0") diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua index 314fd89183..2121d303dc 100644 --- a/modulefiles/build_hera_intel.lua +++ b/modulefiles/build_hera_intel.lua @@ -8,10 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===]) prepend_path("MODULEPATH","/contrib/sutils/modulefiles") load("sutils") --- When Hera switches from CentOS to Rocky, replace line withh correct path to spack-stack --- If you want to use Rocky OS now, use line below ---prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-rocky8/install/modulefiles/Core") prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" From 1b12beb848fb4cdff8647cf886a8bc52293b14b4 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 19 Mar 2024 17:44:48 +0000 Subject: [PATCH 29/30] Fix issue with stochastic nml settings. --- ush/set_fv3nml_ens_stoch_seeds.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ush/set_fv3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py index 7390e9fde2..f34b5c1af0 100644 --- a/ush/set_fv3nml_ens_stoch_seeds.py +++ b/ush/set_fv3nml_ens_stoch_seeds.py @@ -13,15 +13,14 @@ from uwtools.api.config import get_sh_config, realize from python_utils import ( + cfg_to_yaml_str, + flatten_dict, + import_vars, + load_shell_config, print_input_args, print_info_msg, - import_vars, - cfg_to_yaml_str, ) -VERBOSE = os.environ.get("VERBOSE", "true") -FV3_NML_FN = os.environ.get("FV3_NML_FN", "input.nml") - def set_fv3nml_ens_stoch_seeds(cdate, expt_config): """ @@ -44,6 +43,9 @@ def set_fv3nml_ens_stoch_seeds(cdate, expt_config): print_input_args(locals()) + FV3_NML_FN = expt_config["workflow"]["FV3_NML_FN"] + VERBOSE = expt_config["workflow"]["VERBOSE"] + # set variables important to this function from the experiment definition import_vars(dictionary=expt_config["global"]) # pylint: disable=undefined-variable @@ -141,5 +143,5 @@ def parse_args(argv): if __name__ == "__main__": args = parse_args(sys.argv[1:]) - config = get_sh_config(args.path_to_defns) - set_fv3nml_ens_stoch_seeds(args.cdate, config) + cfg = load_shell_config(args.path_to_defns) + set_fv3nml_ens_stoch_seeds(args.cdate, cfg) From ac6fe2dbd525f84fcd988ebad0362d5d04a2b3bb Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 19 Mar 2024 17:52:12 +0000 Subject: [PATCH 30/30] Update test and lint. --- tests/test_python/test_set_fv3nml_ens_stoch_seeds.py | 7 +++++-- ush/set_fv3nml_ens_stoch_seeds.py | 11 +++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py index 5f0eeb0c5a..17bf74c04b 100644 --- a/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py +++ b/tests/test_python/test_set_fv3nml_ens_stoch_seeds.py @@ -55,9 +55,12 @@ def setUp(self): set_env_var("ENSMEM_INDX", 2) - set_env_var("FV3_NML_FN", "input.nml") self.config = { + "workflow": { + "VERBOSE": True, + "FV3_NML_FN": "input.nml", + }, "global": { "DO_SHUM": True, "DO_SKEB": True, @@ -65,7 +68,7 @@ def setUp(self): "DO_SPP": True, "DO_LSM_SPP": True, "ISEED_SPP": [4, 5, 6, 7, 8], - } + }, } def tearDown(self): diff --git a/ush/set_fv3nml_ens_stoch_seeds.py b/ush/set_fv3nml_ens_stoch_seeds.py index f34b5c1af0..3459fa8707 100644 --- a/ush/set_fv3nml_ens_stoch_seeds.py +++ b/ush/set_fv3nml_ens_stoch_seeds.py @@ -10,11 +10,10 @@ import sys from textwrap import dedent -from uwtools.api.config import get_sh_config, realize +from uwtools.api.config import realize from python_utils import ( cfg_to_yaml_str, - flatten_dict, import_vars, load_shell_config, print_input_args, @@ -43,8 +42,8 @@ def set_fv3nml_ens_stoch_seeds(cdate, expt_config): print_input_args(locals()) - FV3_NML_FN = expt_config["workflow"]["FV3_NML_FN"] - VERBOSE = expt_config["workflow"]["VERBOSE"] + fv3_nml_fn = expt_config["workflow"]["FV3_NML_FN"] + verbose = expt_config["workflow"]["VERBOSE"] # set variables important to this function from the experiment definition import_vars(dictionary=expt_config["global"]) @@ -58,7 +57,7 @@ def set_fv3nml_ens_stoch_seeds(cdate, expt_config): # # ----------------------------------------------------------------------- # - fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" + fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{fv3_nml_fn}" ensmem_num = int(os.environ["ENSMEM_INDX"]) @@ -106,7 +105,7 @@ def set_fv3nml_ens_stoch_seeds(cdate, expt_config): {cfg_to_yaml_str(settings)}""" ), - verbose=VERBOSE, + verbose=verbose, ) realize( input_config=fv3_nml_ensmem_fp,