From 9a3864f082307ce094ab33858695070411d14153 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:22:17 -0600 Subject: [PATCH] Update develop-ref after dtcenter/MET#2655 and #2311 (#2331) Co-authored-by: Lisa Goodrich Co-authored-by: Julie Prestopnik Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Hank Fisher Co-authored-by: Dan Adriaansen Co-authored-by: johnhg Co-authored-by: John Halley Gotway Co-authored-by: jprestop Co-authored-by: Tracy Hertneky Co-authored-by: Giovanni Rosa Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz Co-authored-by: Daniel Adriaansen Co-authored-by: Jonathan Vigh Co-authored-by: root Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Giovanni Rosa Co-authored-by: mrinalbiswas Co-authored-by: Christina Kalb Co-authored-by: jason-english <73247785+jason-english@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> fix GitHub Actions warnings (#1864) fix #1884 develop PCPCombine {custom} in subtract method (#1887) fix #1939 develop - failure reading obs when zipped file also exists (#1941) Closes https://github.com/dtcenter/METplus/issues/1986 fix develop Fix broken documentation links (#2004) fix #2026 develop StatAnalysis looping (#2028) fix priority of obs_window config variables so that wrapper-specific version is preferred over generic OBS_WINDOW_BEGIN/END (#2062) fix #2070 var list numeric order (#2072) fix #2087 develop docs_pdf (#2091) fix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run (#2099) Fix for Dockerfile smell DL4000 (#2112) fix #2082 develop regrid.convert/censor_thresh/censor_val (#2140) fix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) fix #2137 develop PointStat -obs_valid_beg/end (#2141) fix failured introduced by urllib3 (see https://github.com/urllib3/urllib3/issues/2168) fix #2161 develop PCPCombine additional field arguments in -subtract mode (#2162) fix #2168 develop - StatAnalysis time shift (#2169) fix releases. (#2183) fix #2189 develop - spaces in complex thresholds (#2191) fix #2179 develop TCPairs fix -diag argument (#2187) fixes (#2200) fix diff tests (#2217) fix automated tests (#2237) fix #2235 rename multivar_itensity to multivar_intensity_flag (#2236) fix #2241 Create directory containing -out_stat file (#2242) fix #2245 use unique run ID to name logger instance (#2247) fix #2244 develop fix diff tests (#2254) fixture to set pytest tmpdir (#2261) fix #1853 develop - PointStat don't require mask variables to be set (#2262) fix #2279 develop - buoy station file from 2022 (#2280) --- .coveragerc | 3 + .github/actions/run_tests/entrypoint.sh | 42 +- .github/jobs/get_use_cases_to_run.sh | 17 - .github/jobs/set_job_controls.sh | 3 +- .github/labels/common_labels.txt | 2 + .github/labels/delete_labels.sh | 42 +- .github/labels/get_labels.sh | 11 +- .github/labels/post_patch_labels.sh | 42 +- .github/labels/process_labels.sh | 23 +- .github/pull_request_template.md | 2 +- .github/workflows/testing.yml | 46 +- docs/Contributors_Guide/basic_components.rst | 207 +++++++-- docs/Contributors_Guide/create_wrapper.rst | 108 ++--- docs/Contributors_Guide/deprecation.rst | 71 ++- docs/Contributors_Guide/github_workflow.rst | 8 +- docs/Release_Guide/coordinated.rst | 23 + docs/Release_Guide/index.rst | 87 ++-- docs/Release_Guide/metplus_official.rst | 1 - .../common/update_dtc_website.rst | 46 +- .../coordinated/announce_release.rst | 6 + .../finalize_release_on_github.rst | 5 + .../coordinated/update_dtc_website.rst | 66 +++ .../update_zenodo.rst | 0 .../finalize_release_on_github_official.rst | 10 +- .../release_steps/met/update_dtc_website.rst | 2 +- .../set_beta_deletion_reminder_official.rst | 16 +- docs/Users_Guide/release-notes.rst | 2 +- ...GFS_obsGFS_FeatureRelative_SeriesByLead.py | 1 - ...FS_obsGDAS_UpperAir_MultiField_PrepBufr.py | 1 - ..._fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py | 1 - .../dev_tools/add_met_config_helper.py | 9 +- internal/tests/pytests/requirements.txt | 19 + .../util/run_util/no_install_run_util.conf | 19 + .../tests/pytests/util/run_util/run_util.conf | 22 + .../pytests/util/run_util/sed_run_util.conf | 25 ++ .../pytests/util/run_util/test_run_util.py | 100 +++++ .../util/system_util/test_system_util.py | 128 +++++- .../pytests/util/time_util/test_time_util.py | 38 +- .../command_builder/test_command_builder.py | 90 ++++ .../compare_gridded/test_compare_gridded.py | 1 + .../wrappers/gen_vx_mask/test_gen_vx_mask.py | 4 +- .../grid_stat/test_grid_stat_wrapper.py | 1 - .../pytests/wrappers/mtd/test_mtd_wrapper.py | 182 ++++---- .../pcp_combine/test_pcp_combine_wrapper.py | 10 +- .../test_regrid_data_plane.py | 9 +- .../wrappers/tc_gen/test_tc_gen_wrapper.py | 1 - .../pytests/wrappers/usage/test_usage.py | 15 + metplus/util/__init__.py | 1 - metplus/util/run_util.py | 4 +- metplus/util/system_util.py | 165 +------ metplus/util/time_util.py | 238 ++++++----- metplus/wrappers/ascii2nc_wrapper.py | 60 +-- metplus/wrappers/command_builder.py | 9 +- metplus/wrappers/compare_gridded_wrapper.py | 41 +- metplus/wrappers/cyclone_plotter_wrapper.py | 7 - metplus/wrappers/ensemble_stat_wrapper.py | 4 + metplus/wrappers/example_wrapper.py | 80 ++-- metplus/wrappers/extract_tiles_wrapper.py | 53 +-- metplus/wrappers/gempak_to_cf_wrapper.py | 34 +- metplus/wrappers/gen_ens_prod_wrapper.py | 4 + metplus/wrappers/gen_vx_mask_wrapper.py | 40 +- metplus/wrappers/gfdl_tracker_wrapper.py | 28 +- metplus/wrappers/grid_diag_wrapper.py | 7 +- metplus/wrappers/grid_stat_wrapper.py | 3 + metplus/wrappers/ioda2nc_wrapper.py | 27 +- metplus/wrappers/loop_times_wrapper.py | 4 - metplus/wrappers/met_db_load_wrapper.py | 5 + metplus/wrappers/mode_wrapper.py | 4 + metplus/wrappers/mtd_wrapper.py | 404 +++++++----------- metplus/wrappers/pb2nc_wrapper.py | 41 +- metplus/wrappers/pcp_combine_wrapper.py | 35 +- metplus/wrappers/plot_data_plane_wrapper.py | 34 +- metplus/wrappers/plot_point_obs_wrapper.py | 26 +- metplus/wrappers/point2grid_wrapper.py | 27 +- metplus/wrappers/point_stat_wrapper.py | 3 + metplus/wrappers/py_embed_ingest_wrapper.py | 39 +- metplus/wrappers/reformat_gridded_wrapper.py | 50 +-- metplus/wrappers/regrid_data_plane_wrapper.py | 21 +- metplus/wrappers/runtime_freq_wrapper.py | 163 +++++-- metplus/wrappers/series_analysis_wrapper.py | 3 + metplus/wrappers/stat_analysis_wrapper.py | 41 +- metplus/wrappers/tc_diag_wrapper.py | 8 +- metplus/wrappers/tc_gen_wrapper.py | 44 +- metplus/wrappers/tc_pairs_wrapper.py | 143 +++---- metplus/wrappers/tc_stat_wrapper.py | 8 +- metplus/wrappers/tcrmw_wrapper.py | 152 +++---- metplus/wrappers/usage_wrapper.py | 1 + metplus/wrappers/user_script_wrapper.py | 6 +- parm/README | 17 - parm/README.md | 6 + .../TCPairs/TCPairs_extra_tropical.conf | 3 +- .../TCPairs/TCPairs_tropical.conf | 4 +- ...S_obsGFS_FeatureRelative_SeriesByLead.conf | 6 +- ..._obsGDAS_UpperAir_MultiField_PrepBufr.conf | 1 - ...S_obsGFS_FeatureRelative_SeriesByLead.conf | 5 +- ...esByLead_PyEmbed_Multiple_Diagnostics.conf | 5 +- ...dbLoad_fcstFV3_obsGoes_BrightnessTemp.conf | 4 +- .../read_ascii_storm.py | 3 + ...ter_fcstGFS_obsGFS_UserScript_ExtraTC.conf | 9 +- .../Plotter_fcstGFS_obsGFS_ExtraTC.conf | 10 +- ..._fcstADECK_obsBDECK_ATCF_BasicExample.conf | 13 +- 101 files changed, 1901 insertions(+), 1819 deletions(-) create mode 100644 .coveragerc create mode 100644 docs/Release_Guide/coordinated.rst create mode 100644 docs/Release_Guide/release_steps/coordinated/announce_release.rst create mode 100644 docs/Release_Guide/release_steps/coordinated/finalize_release_on_github.rst create mode 100644 docs/Release_Guide/release_steps/coordinated/update_dtc_website.rst rename docs/Release_Guide/release_steps/{metplus => coordinated}/update_zenodo.rst (100%) rename metplus/util/doc_util.py => internal/scripts/dev_tools/add_met_config_helper.py (97%) create mode 100644 internal/tests/pytests/requirements.txt create mode 100644 internal/tests/pytests/util/run_util/no_install_run_util.conf create mode 100644 internal/tests/pytests/util/run_util/run_util.conf create mode 100644 internal/tests/pytests/util/run_util/sed_run_util.conf create mode 100644 internal/tests/pytests/util/run_util/test_run_util.py create mode 100644 internal/tests/pytests/wrappers/usage/test_usage.py delete mode 100644 parm/README create mode 100644 parm/README.md diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 000000000..d561722b1 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +relative_files = True +source = metplus diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 78ce25e08..eecb719a2 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -12,7 +12,7 @@ source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh # get branch name for push or pull request events # add -pull_request if pull request event to keep separated -branch_name=`${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py` +branch_name=$(${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py) if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then branch_name=${branch_name}-pull_request fi @@ -23,49 +23,17 @@ time_command docker pull $DOCKERHUBTAG # if unsuccessful (i.e. pull request from a fork) # then build image locally -docker inspect --type=image $DOCKERHUBTAG > /dev/null -if [ $? != 0 ]; then +if ! docker inspect --type=image $DOCKERHUBTAG > /dev/null; then # if docker pull fails, build locally echo docker pull failed. Building Docker image locally... ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/docker_setup.sh fi -# running unit tests (pytests) -if [[ "$INPUT_CATEGORIES" == pytests* ]]; then - export METPLUS_ENV_TAG="test.v5.1" - export METPLUS_IMG_TAG=${branch_name} - echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG} - echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG} - - export RUN_TAG=metplus-run-env - - # use BuildKit to build image - export DOCKER_BUILDKIT=1 - - start_seconds=$SECONDS - - # build an image with the pytest conda env and the METplus branch image - # Note: adding --build-arg without any value tells docker to - # use value from local environment (export METPLUS_IMG_TAG) - time_command docker build -t $RUN_TAG \ - --build-arg METPLUS_IMG_TAG \ - --build-arg METPLUS_ENV_TAG \ - -f .github/actions/run_tests/Dockerfile.run \ - . - - echo Running Pytests - command="export METPLUS_TEST_OUTPUT_BASE=/data/output;" - command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest internal/tests/pytests -vv --cov=metplus --cov-append --cov-report=term-missing;" - command+="if [ \$? != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" - time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command" - exit $? -fi - # running use case tests # split apart use case category and subset list from input -CATEGORIES=`echo $INPUT_CATEGORIES | awk -F: '{print $1}'` -SUBSETLIST=`echo $INPUT_CATEGORIES | awk -F: '{print $2}'` +CATEGORIES=$(echo $INPUT_CATEGORIES | awk -F: '{print $1}') +SUBSETLIST=$(echo $INPUT_CATEGORIES | awk -F: '{print $2}') # run all cases if no subset list specified if [ -z "${SUBSETLIST}" ]; then @@ -73,7 +41,7 @@ if [ -z "${SUBSETLIST}" ]; then fi # get METviewer if used in any use cases -all_requirements=`./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}` +all_requirements=$(./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}) echo All requirements: $all_requirements NETWORK_ARG="" if [[ "$all_requirements" =~ .*"metviewer".* ]]; then diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh index 0d06a85df..d937f1f7a 100755 --- a/.github/jobs/get_use_cases_to_run.sh +++ b/.github/jobs/get_use_cases_to_run.sh @@ -7,11 +7,9 @@ matrix="[]" run_use_cases=$1 run_all_use_cases=$2 -run_unit_tests=$3 echo Run use cases: $run_use_cases echo Run all use cases: $run_all_use_cases -echo Run unit tests: $run_unit_tests # if running use cases, generate JQ filter to use if [ "$run_use_cases" == "true" ]; then @@ -28,21 +26,6 @@ if [ "$run_use_cases" == "true" ]; then fi -# if unit tests will be run, add "pytests" to beginning of matrix list -if [ "$run_unit_tests" == "true" ]; then - echo Adding unit tests to list to run - - pytests="\"pytests\"," - - # if matrix is empty, set to an array that only includes pytests - if [ "$matrix" == "[]" ]; then - matrix="[${pytests:0: -1}]" - # otherwise prepend item to list - else - matrix="[${pytests}${matrix:1}" - fi -fi - echo Array of groups to run is: $matrix # if matrix is still empty, exit 1 to fail step and skip rest of workflow if [ "$matrix" == "[]" ]; then diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index f03c4809a..032197ff4 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -87,6 +87,7 @@ fi echo "run_get_image=$run_get_image" >> $GITHUB_OUTPUT echo "run_get_input_data=$run_get_input_data" >> $GITHUB_OUTPUT echo "run_diff=$run_diff" >> $GITHUB_OUTPUT +echo "run_unit_tests=$run_unit_tests" >> $GITHUB_OUTPUT echo "run_save_truth_data=$run_save_truth_data" >> $GITHUB_OUTPUT echo "external_trigger=$external_trigger" >> $GITHUB_OUTPUT @@ -96,7 +97,7 @@ branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py` echo "branch_name=$branch_name" >> $GITHUB_OUTPUT # get use cases to run -.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests +.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases # echo output variables to review in logs echo branch_name: $branch_name diff --git a/.github/labels/common_labels.txt b/.github/labels/common_labels.txt index 749775cac..055bf53bf 100644 --- a/.github/labels/common_labels.txt +++ b/.github/labels/common_labels.txt @@ -7,7 +7,9 @@ {"name": "component: documentation","color": "1d76db","description": "Documentation issue"} {"name": "component: external dependency","color": "1d76db","description": "External dependency issue"} {"name": "component: code optimization","color": "1d76db","description": "Code optimization issue"} +{"name": "component: input data","color": "1d76db","description": "Input data issue"} {"name": "component: release engineering","color": "1d76db","description": "Release engineering issue"} +{"name": "component: repository maintenance","color": "1d76db","description": "Repository maintenance issue"} {"name": "component: testing","color": "1d76db","description": "Software testing issue"} {"name": "component: training","color": "1d76db","description": "Training issue"} {"name": "component: user support","color": "1d76db","description": "User support issue"} diff --git a/.github/labels/delete_labels.sh b/.github/labels/delete_labels.sh index 3756f322e..4ff004ebe 100755 --- a/.github/labels/delete_labels.sh +++ b/.github/labels/delete_labels.sh @@ -15,7 +15,10 @@ else repo=$3 fi -# Constants +# Verbose output +VERBOSE=0 + +# GitHub label URL URL="https://api.github.com/repos/dtcenter/${repo}/labels" COMMON_LABELS="`dirname $0`/common_labels.txt" @@ -28,7 +31,12 @@ SCRIPT_DIR=`dirname $0` TMP_FILE="${repo}_labels.tmp" CMD="${SCRIPT_DIR}/get_labels.sh ${user} ${auth} ${repo}" echo "CALLING: ${CMD}" -${CMD} > ${TMP_FILE} +${CMD} > ${TMP_FILE} 2>/dev/null + +# Initialize counts +n_common=0 +n_custom=0 +n_delete=0 # Check each of the existing labels against the common list while read -r line; do @@ -36,19 +44,30 @@ while read -r line; do # Parse the label name name=`echo $line | sed -r 's/,/\n/g' | grep '"name":' | cut -d':' -f2-10 | cut -d'"' -f2` - # Check if it's a common label and a component label + # Check if it appears in the list of common labels is_common=`egrep -i "\"${name}\"" ${COMMON_LABELS} | wc -l` - is_custom=`echo ${name} | egrep "component:|type:" | wc -l` + + # Check if its a custom label that beginning with component, type, or repository name + is_custom=`echo ${name} | egrep -r -i "component:|type:|${repo}" | wc -l` # Keep COMMON labels if [[ $is_common -gt 0 ]]; then - echo "[COMMON] ${repo} label ... ${name}" + ((n_common+=1)) + if [[ $VERBOSE -gt 0 ]]; then + echo "[COMMON] ${repo} label ... ${name}" + fi # Keep CUSTOM, repo-specific labels elif [[ $is_custom -gt 0 ]]; then - echo "[CUSTOM] ${repo} label ... ${name}" + ((n_custom+=1)) + if [[ $VERBOSE -gt 0 ]]; then + echo "[CUSTOM] ${repo} label ... ${name}" + fi # DELETE non-common, non-custom labels - else - echo "[DELETE] ${repo} label ... ${name}" + else + ((n_delete+=1)) + if [[ $VERBOSE -gt 0 ]]; then + echo "[DELETE] ${repo} label ... ${name}" + fi DELETE_URL="${URL}/`echo ${name} | sed -r 's/ /%20/g'`" echo "curl -u \"${user}:${auth}\" -X DELETE \ -H \"Accept: application/vnd.github.v3+json\" \ @@ -62,5 +81,8 @@ rm -f ${TMP_FILE} # Make the run command file executable chmod +x ${CMD_FILE} -echo "To make these changes, execute the run command file:" -echo "./${CMD_FILE}" + +# Print summary +echo "For the ${repo} repository, found $n_common common, $n_custom custom, and $n_delete labels to be deleted." +echo "To delete $n_delete existing ${repo} labels, run:" +echo " ${CMD_FILE}" diff --git a/.github/labels/get_labels.sh b/.github/labels/get_labels.sh index 103877e2f..b6bd6b0c5 100755 --- a/.github/labels/get_labels.sh +++ b/.github/labels/get_labels.sh @@ -16,8 +16,11 @@ else fi # Pull and format existing records for existing labels -curl -u "${user}:${auth}" -H "Accept: application/vnd.github.v3+json" \ -"https://api.github.com/repos/dtcenter/${repo}/labels?page=1&per_page=100" | \ -egrep '"name":|"color":|"description":|{|}' | \ -tr -d '\n' | sed -r 's/ +/ /g' | sed 's/}/}\n/g' | sed 's/,* {/{/g' +# Run twice for page 1 and page 2 to support up to 200 existing labels +for page_number in 1 2; do + curl -u "${user}:${auth}" -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/dtcenter/${repo}/labels?page=${page_number}&per_page=100" | \ + egrep '"name":|"color":|"description":|{|}' | \ + tr -d '\n' | sed -r 's/ +/ /g' | sed 's/}/}\n/g' | sed 's/,* {/{/g' +done diff --git a/.github/labels/post_patch_labels.sh b/.github/labels/post_patch_labels.sh index 7c4811f2d..5c3cc5344 100755 --- a/.github/labels/post_patch_labels.sh +++ b/.github/labels/post_patch_labels.sh @@ -17,19 +17,29 @@ else labels=$4 fi +# Verbose output +VERBOSE=0 + # GitHub label URL URL="https://api.github.com/repos/dtcenter/${repo}/labels" -# Output command file -CMD_FILE="`dirname $0`/commands/post_patch_labels_${repo}_cmd.sh" -echo "#!/bin/sh -v" > ${CMD_FILE} +# Output command files +POST_CMD_FILE="`dirname $0`/commands/post_labels_${repo}_cmd.sh" +echo "#!/bin/sh -v" > ${POST_CMD_FILE} + +PATCH_CMD_FILE="`dirname $0`/commands/patch_labels_${repo}_cmd.sh" +echo "#!/bin/sh -v" > ${PATCH_CMD_FILE} + +# Initialize counts +n_post=0 +n_patch=0 # Get the current repo labels SCRIPT_DIR=`dirname $0` TMP_FILE="${repo}_labels.tmp" CMD="${SCRIPT_DIR}/get_labels.sh ${user} ${auth} ${repo}" echo "CALLING: ${CMD}" -${CMD} > ${TMP_FILE} +${CMD} > ${TMP_FILE} 2>/dev/null # Read the lines of the label file while read -r line; do @@ -42,17 +52,23 @@ while read -r line; do # POST a new label if [[ $exists -eq 0 ]]; then - echo "[POST ] ${repo} label ... ${name}" + ((n_post+=1)) + if [[ $VERBOSE -gt 0 ]]; then + echo "[POST ] ${repo} label ... ${name}" + fi echo "curl -u \"${user}:${auth}\" -X POST \ -H \"Accept: application/vnd.github.v3+json\" \ - -d '${line}' '${URL}'" >> ${CMD_FILE} + -d '${line}' '${URL}'" >> ${POST_CMD_FILE} # PATCH an existing label else + ((n_patch+=1)) old_name=`egrep -i "\"${name}\"" ${TMP_FILE} | sed -r 's/,/\n/g' | grep '"name":' | cut -d':' -f2-10 | cut -d'"' -f2 | sed -r 's/ /%20/g'` - echo "[PATCH] ${repo} label ... ${old_name} -> ${name}" + if [[ $VERBOSE -gt 0 ]]; then + echo "[PATCH] ${repo} label ... ${old_name} -> ${name}" + fi echo "curl -u \"${user}:${auth}\" -X PATCH \ -H \"Accept: application/vnd.github.v3+json\" \ - -d '${line}' '${URL}/${old_name}'" >> ${CMD_FILE} + -d '${line}' '${URL}/${old_name}'" >> ${PATCH_CMD_FILE} fi done < $labels @@ -61,7 +77,11 @@ done < $labels rm -f ${TMP_FILE} # Make the run command file executable -chmod +x ${CMD_FILE} -echo "To make these changes, execute the run command file:" -echo "./${CMD_FILE}" +chmod +x ${POST_CMD_FILE} ${PATCH_CMD_FILE} +# Print summary +echo "For the ${repo} repository, found $n_patch existing labels to be updated and $n_post new labels to be added." +echo "To add $n_post new ${repo} labels, run:" +echo " ${POST_CMD_FILE}" +echo "To update $n_patch existing ${repo} labels, run:" +echo " ${PATCH_CMD_FILE}" diff --git a/.github/labels/process_labels.sh b/.github/labels/process_labels.sh index 1c37a444b..a23bf7c90 100755 --- a/.github/labels/process_labels.sh +++ b/.github/labels/process_labels.sh @@ -19,14 +19,21 @@ SCRIPT_DIR=`dirname $0` REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \ metexpress metplus-training"; -# Build commands to add/update common labels +# Process each repository for REPO in ${REPO_LIST}; do - echo $REPO - ${SCRIPT_DIR}/post_patch_labels.sh $USER $AUTH $REPO ${SCRIPT_DIR}/common_labels.txt -done -# Build commands to delete extra labels -for REPO in ${REPO_LIST}; do - echo $REPO; - ${SCRIPT_DIR}/delete_labels.sh $USER $AUTH $REPO + echo + echo "Processing repository: ${REPO}" + echo + + # Build commands to add/update common labels + CMD="${SCRIPT_DIR}/post_patch_labels.sh $USER $AUTH $REPO ${SCRIPT_DIR}/common_labels.txt" + echo "CALLING: ${CMD}" + ${CMD} + + # Build commands to delete extra labels + CMD="${SCRIPT_DIR}/delete_labels.sh $USER $AUTH $REPO" + echo "CALLING: ${CMD}" + ${CMD} + done diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 48c6621f2..dda54835f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,6 +23,6 @@ See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors Select: **Reviewer(s)** Select: **Organization** level software support **Project** or **Repository** level development cycle **Project** Select: **Milestone** as the version that will include these changes -- [ ] fter submitting the PR, select the :gear: icon in the **Development** section of the right hand sidebar. Search for the issue that this PR will close and select it, if it is not already selected. +- [ ] After submitting the PR, select the :gear: icon in the **Development** section of the right hand sidebar. Search for the issue that this PR will close and select it, if it is not already selected. - [ ] After the PR is approved, merge your changes. If permissions do not allow this, request that the reviewer do the merge. - [ ] Close the linked issue and delete your feature or bugfix branch from GitHub. diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index aff2d9a49..6dc8e4acb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -79,6 +79,7 @@ jobs: run_get_image: ${{ steps.job_status.outputs.run_get_image }} run_get_input_data: ${{ steps.job_status.outputs.run_get_input_data }} run_diff: ${{ steps.job_status.outputs.run_diff }} + run_unit_tests: ${{ steps.job_status.outputs.run_unit_tests }} run_save_truth_data: ${{ steps.job_status.outputs.run_save_truth_data }} external_trigger: ${{ steps.job_status.outputs.external_trigger }} branch_name: ${{ steps.job_status.outputs.branch_name }} @@ -87,7 +88,7 @@ jobs: name: Docker Setup - Get METplus Image runs-on: ubuntu-latest needs: job_control - if: ${{ needs.job_control.outputs.run_get_image == 'true' }} + if: ${{ needs.job_control.outputs.run_get_image == 'true' && needs.job_control.outputs.run_some_tests == 'true' }} steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -105,10 +106,10 @@ jobs: name: Docker Setup - Update Data Volumes runs-on: ubuntu-latest needs: job_control - if: ${{ needs.job_control.outputs.run_get_input_data == 'true' }} + if: ${{ needs.job_control.outputs.run_get_input_data == 'true' && needs.job_control.outputs.run_some_tests == 'true' }} continue-on-error: true steps: - - uses: dtcenter/metplus-action-data-update@v2 + - uses: dtcenter/metplus-action-data-update@v3 with: docker_name: ${{ secrets.DOCKER_USERNAME }} docker_pass: ${{ secrets.DOCKER_PASSWORD }} @@ -121,10 +122,39 @@ jobs: use_feature_data: true tag_max_pages: 15 + unit_tests: + name: Unit Tests + runs-on: ubuntu-latest + needs: [job_control] + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + cache: 'pip' + - name: Install Python Test Dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r internal/tests/pytests/requirements.txt + - name: Run Pytests + run: coverage run -m pytest internal/tests/pytests + env: + METPLUS_TEST_OUTPUT_BASE: ${{ runner.workspace }}/pytest_output + - name: Generate coverage report + run: coverage report -m + if: always() + - name: Run Coveralls + uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 + if: always() + continue-on-error: true + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + use_case_tests: name: Use Case Tests runs-on: ubuntu-latest - needs: [get_image, update_data_volumes, job_control] + needs: [get_image, update_data_volumes, job_control, unit_tests] if: ${{ needs.job_control.outputs.run_some_tests == 'true' }} strategy: fail-fast: false @@ -158,24 +188,24 @@ jobs: # copy logs with errors to error_logs directory to save as artifact - name: Save error logs id: save-errors - if: ${{ always() && steps.run_tests.conclusion == 'failure' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion == 'failure' }} run: .github/jobs/save_error_logs.sh # run difference testing - name: Run difference tests id: run-diff - if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success' && !startsWith(matrix.categories,'pytests') }} + if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success' }} run: .github/jobs/run_difference_tests.sh ${{ matrix.categories }} ${{ steps.get-artifact-name.outputs.artifact_name }} # copy output data to save as artifact - name: Save output data id: save-output - if: ${{ always() && steps.run_tests.conclusion != 'skipped' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion != 'skipped' }} run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }} - name: Upload output data artifact uses: actions/upload-artifact@v3 - if: ${{ always() && steps.run_tests.conclusion != 'skipped' && !startsWith(matrix.categories,'pytests') }} + if: ${{ always() && steps.run_tests.conclusion != 'skipped' }} with: name: ${{ steps.get-artifact-name.outputs.artifact_name }} path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }} diff --git a/docs/Contributors_Guide/basic_components.rst b/docs/Contributors_Guide/basic_components.rst index c386c03f7..0b24209d8 100644 --- a/docs/Contributors_Guide/basic_components.rst +++ b/docs/Contributors_Guide/basic_components.rst @@ -4,15 +4,13 @@ Basic Components of METplus Python Wrappers ******************************************* -CommandBuilder -============== +.. _bc_class_hierarchy: + +Class Hierarchy +=============== -CommandBuilder is the parent class of all METplus wrappers. -Every wrapper is a subclass of CommandBuilder or -another subclass of CommandBuilder. -For example, GridStatWrapper, PointStatWrapper, EnsembleStatWrapper, -and MODEWrapper are all subclasses of CompareGriddedWrapper. -CompareGriddedWrapper is a subclass of CommandBuilder. +**CommandBuilder** is the parent class of all METplus wrappers. +Every wrapper is a subclass of CommandBuilder or a subclass of CommandBuilder. CommandBuilder contains instance variables that are common to every wrapper, such as config (METplusConfig object), errors (a counter of the number of errors that have occurred in the wrapper), and @@ -20,6 +18,107 @@ c_dict (a dictionary containing common information). CommandBuilder also contains use class functions that can be called within each wrapper, such as create_c_dict, clear, and find_data. +**RuntimeFreqWrapper** is a subclass of **CommandBuilder** that contains all +of the logic to handle time looping. +See :ref:`Runtime_Freq` for more information on time looping. +Unless a wrapper is very basic and does not need to loop over time, then +the wrapper should inherit directly or indirectly from **RuntimeFreqWrapper**. + +**LoopTimesWrapper** is a subclass of **RuntimeFreqWrapper**. +This wrapper simply sets the default runtime frequency to **RUN_ONCE_FOR_EACH** +for its subclasses. + +**CompareGriddedWrapper** is a subclass of **LoopTimesWrapper** that contains +functions that are common to multiple wrappers that compare forecast (FCST) +and observation (OBS) data. Subclasses of this wrapper include +**GridStatWrapper**, **PointStatWrapper**, **EnsembleStatWrapper**, +**MODEWrapper**, and **MTDWrapper**. + +**MTDWrapper** in an exception from the rest of the **CompareGriddeWrapper** +subclasses because it typically runs once for each init or valid time and +reads and processes all forecast leads at once. This wrapper inherits from +**CompareGriddedWrapper** because it still uses many of its functions. + + +.. _bc_class_vars: + +Class Variables +=============== + +RUNTIME_FREQ_DEFAULT +-------------------- + +Wrappers that inherit from **RuntimeFreqWrapper** should include a class +variable called **RUNTIME_FREQ_DEFAULT** that lists the default runtime +frequency that should be used if it is not explicitly defined in the METplus +configuration. + +Example:: + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + +If no clear default value exists, then *None* can be set in place of a string. +This means that a use case will report an error if the frequency is not +defined in the METplus configuration file. +The **UserScriptWrapper** wrapper is an example:: + + RUNTIME_FREQ_DEFAULT = None + + +RUNTIME_FREQ_SUPPORTED +---------------------- + +Wrappers that inherit from **RuntimeFreqWrapper** should include a class +variable called **RUNTIME_FREQ_SUPPORTED** that defines a list of the +runtime frequency settings that are supported by the wrapper. Example:: + + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_PER_INIT_OR_VALID'] + +If all runtime frequency values are supported by the wrapper, then the string +*'ALL'* can be set instead of a list of strings:: + + RUNTIME_FREQ_SUPPORTED = 'ALL' + + +WRAPPER_ENV_VAR_KEYS +-------------------- + +This class variable lists all of the environment variables that are set by +the wrapper. These variables are typically referenced in the wrapped MET +config file for the tool and are named with a *METPLUS\_* prefix. +All of the variables that are referenced in the wrapped MET config file must +be listed here so that they will always be set to prevent an error when MET +reads the config file. An empty string will be set if they are not set to +another value by the wrapper. + +DEPRECATED_WRAPPER_ENV_VAR_KEYS +-------------------------------- + +(Optional) +This class variable lists any environment variables that were +previously set by the wrapper and referenced in an old version of the +wrapped MET config file. +This list serves as a developer reference of the variables that were +previously used but are now deprecated. When support for setting these +variables are eventually removed, then the values in this list should also +be removed. + +Flags +----- + +(Optional) +For wrappers that set a dictionary of flags in the wrapped MET config file, +class variables that contain a list of variable names can be defined. +This makes it easier to add/change these variables. + +The list is read by the **self.handle_flags** function. +The name of the variable corresponds to the argument passed to the function. +For example, **EnsembleStatWrapper** includes **OUTPUT_FLAGS** and a call +to **self.handle_flags('OUTPUT')**. + +Existing \*_FLAG class variables include **OUTPUT_FLAGS**, **NC_PAIRS_FLAGS**, **NC_ORANK_FLAGS**, and **ENSEMBLE_FLAGS**. + + .. _bc_init_function: Init Function @@ -64,16 +163,17 @@ create_c_dict (ExampleWrapper):: def create_c_dict(self): c_dict = super().create_c_dict() # get values from config object and set them to be accessed by wrapper - c_dict['INPUT_TEMPLATE'] = self.config.getraw('filename_templates', - 'EXAMPLE_INPUT_TEMPLATE', '') + c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', + 'EXAMPLE_INPUT_TEMPLATE') c_dict['INPUT_DIR'] = self.config.getdir('EXAMPLE_INPUT_DIR', '') - if c_dict['INPUT_TEMPLATE'] == '': - self.logger.info('[filename_templates] EXAMPLE_INPUT_TEMPLATE was not set. ' - 'You should set this variable to see how the runtime is ' - 'substituted. For example: {valid?fmt=%Y%m%d%H}.ext') + if not c_dict['INPUT_TEMPLATE']: + self.logger.info('EXAMPLE_INPUT_TEMPLATE was not set. ' + 'You should set this variable to see how the ' + 'runtime is substituted. ' + 'For example: {valid?fmt=%Y%m%d%H}.ext') - if c_dict['INPUT_DIR'] == '': + if not c_dict['INPUT_DIR']: self.logger.debug('EXAMPLE_INPUT_DIR was not set') return c_dict @@ -95,7 +195,7 @@ create_c_dict (CommandBuilder):: isOK class variable =================== -isOK is defined in CommandBuilder (ush/command_builder.py). +isOK is defined in CommandBuilder (metplus/wrappers/command_builder.py). Its function is to note a failed process while not stopping a parent process. Instead of instantly exiting a larger wrapper script once one subprocess has @@ -106,55 +206,74 @@ At the end of the wrapper initialization step, all isOK=false will be collected and reported. Execution of the wrappers will not occur unless all wrappers in the process list are initialized correctly. +The **self.log_error** function logs an error and sets self.isOK to False, so +it is not necessary to set *self.isOK = False* if this function is called. + .. code-block:: python c_dict['CONFIG_FILE'] = self.config.getstr('config', 'MODE_CONFIG_FILE', '') if not c_dict['CONFIG_FILE']: self.log_error('MODE_CONFIG_FILE must be set') + if something_else_goes_wrong: self.isOK = False -See MODEWrapper (ush/mode_wrapper.py) for other examples. +.. _bc_run_at_time_once: +run_at_time_once function +========================= -run_at_time function -==================== - -run_at_time runs a process for one specific time. -This is defined in CommandBuilder. +**run_at_time_once** runs a process for one specific time. The time depends +on the value of {APP_NAME}_RUNTIME_FREQ. Most wrappers run once per each +init or valid and forecast lead time. This function is often defined in each +wrapper to handle command setup specific to the wrapper. There is a generic +version of the function in **runtime_freq_wrapper.py** that can be used by +other wrappers: .. code-block:: python - def run_at_time(self, input_dict): - """! Loop over each forecast lead and build pb2nc command """ - # loop of forecast leads and process each - lead_seq = util.get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead + def run_at_time_once(self, time_info): + """! Process runtime and try to build command to run. Most wrappers + should be able to call this function to perform all of the actions + needed to build the commands using this template. This function can + be overridden if necessary. - lead_string = time_util.ti_calculate(input_dict)['lead_string'] - self.logger.info("Processing forecast lead {}".format(lead_string)) + @param time_info dictionary containing timing information + @returns True if command was built/run successfully or + False if something went wrong + """ + # get input files + if not self.find_input_files(time_info): + return False - # Run for given init/valid time and forecast lead combination - self.run_at_time_once(input_dict) + # get output path + if not self.find_and_check_output_file(time_info): + return False -See ush/pb2nc_wrapper.py for an example. + # get other configurations for command + self.set_command_line_arguments(time_info) + + # set environment variables if using config file + self.set_environment_variables(time_info) + + # build command and run + return self.build() + +Typically the **find_input_files** and **set_command_line_arguments** +functions need to be implemented in the wrapper to handle the wrapper-specific +functionality. run_all_times function ====================== -run_all_times loops over a series of times calling run_at_time for one -process for each time. Defined in CommandBuilder but overridden in -wrappers that process all of the data from every run time at once. - -See SeriesByLeadWrapper (ush/series_by_lead_wrapper.py) for an example of -overriding the function. +If a wrapper is not inheriting from RuntimeFreqWrapper or one of its child +classes, then the **run_all_times** function can be implemented in the wrapper. +This function is called when the wrapper is called. get_command function ==================== -get_command assembles a MET command with arguments that can be run via the -shell or the wrapper. +**get_command** assembles the command that will be run. It is defined in CommandBuilder but is overridden in most wrappers because the command line arguments differ for each MET tool. @@ -252,13 +371,13 @@ used to easily add support for overriding MET configuration variables that were not previously supported in METplus configuration files. There is a utility that can be used to easily see what changes are needed to -add support for a new variable. The doc_util.py script can be run from the +add support for a new variable. The add_met_config_helper.py script can be run from the command line to output a list of instructions to add new support. It can be run from the top level of the METplus repository. The script can be called to add a single MET configuration variable by supplying the MET tool name and the variable name:: - ./metplus/util/doc_util.py point_stat sid_exc + ./internal/scripts/dev_tools/add_met_config_helper.py point_stat sid_exc This command will provide guidance for adding support for the sid_exc variable found in the PointStatConfig file. @@ -266,7 +385,7 @@ found in the PointStatConfig file. The script can also be called with the name of a dictionary and the names of each dictionary variable:: - ./metplus/util/doc_util.py grid_stat distance_map baddeley_p baddeley_max_dist fom_alpha zhu_weight beta_value_n + ./internal/scripts/dev_tools/add_met_config_helper.py grid_stat distance_map baddeley_p baddeley_max_dist fom_alpha zhu_weight beta_value_n This command will provide guidance for adding support for the distance_map dictionary found in the GridStatConfig file. The list of variables found inside diff --git a/docs/Contributors_Guide/create_wrapper.rst b/docs/Contributors_Guide/create_wrapper.rst index d9849dcc2..f5280e3a1 100644 --- a/docs/Contributors_Guide/create_wrapper.rst +++ b/docs/Contributors_Guide/create_wrapper.rst @@ -7,7 +7,7 @@ Naming File Name ^^^^^^^^^ -Create the new wrapper in the *METplus/metplus/wrappers* directory and +Create the new wrapper in the *metplus/wrappers* directory and name it to reflect the wrapper's function, e.g.: new_tool_wrapper.py is a wrapper around an application named "new_tool." Copy the **example_wrapper.py** to start the process. @@ -65,12 +65,12 @@ Naming ^^^^^^ Rename the class to match the wrapper's class from the above sections. -Most wrappers should be a subclass of the CommandBuilder wrapper:: +Most wrappers should be a subclass of the RuntimeFreqWrapper:: - class NewToolWrapper(CommandBuilder) + class NewToolWrapper(RuntimeFreqWrapper) -The text 'CommandBuilder' in parenthesis makes NewToolWrapper a subclass -of CommandBuilder. +The text *RuntimeFreqWrapper* in parenthesis makes NewToolWrapper a subclass +of RuntimeFreqWrapper. Find and replace can be used to rename all instances of the wrapper name in the file. For example, to create IODA2NC wrapper from ASCII2NC, replace @@ -85,7 +85,18 @@ Parent Class If the new tool falls under one of the existing tool categories, then make the tool a subclass of one of the existing classes. This should only be done if the functions in the parent class are needed -by the new wrapper. When in doubt, use the CommandBuilder. +by the new wrapper. When in doubt, use the **RuntimeFreqWrapper**. + +See :ref:`bc_class_hierarchy` for more information on existing classes to +determine which class to use as the parent class. + +Class Variables for Runtime Frequency +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**RUNTIME_FREQ_DEFAULT** and **RUNTIME_FREQ_SUPPORTED** should be set for all +wrappers that inherit from **RuntimeFreqWrapper**. + +See :ref:`bc_class_vars` for more information. Init Function ^^^^^^^^^^^^^ @@ -142,69 +153,20 @@ then the wrapper will produce an error and not build the command. Run Functions ^^^^^^^^^^^^^ -* Override the run_at_time method if the wrapper will be called once for each - valid or init time specified in the configuration file. - If the wrapper will loop over each forecast lead - (LEAD_SEQ in the METplus config file) and process once for each, then - override run_at_time with the following method and put the logic to build - the MET command for each run in a run_at_time_once method:: - - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - @param input_dict dictionary containing timing information - @returns None - """ - lead_seq = util.get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = time_util.ti_calculate(input_dict) - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_once(time_info) - - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - @param time_info dictionary containing timing information - """ - # get input files - if self.find_input_files(time_info) is None: - return - - # get output path - if not self.find_and_check_output_file(time_info): - return - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - self.build_and_run_command() - - -If the wrapper will not loop and process for each forecast lead, -put the logic to build the command in the run_at_time method. +* The **run_at_time_once** function or some the functions that it calls will + need to be overridden in the wrapper. + See :ref:`bc_run_at_time_once` for more information. -* It is recommended to divide up the logic into components, as illustrated - above, to make the code more readable and easier to test. +* It is recommended to divide up the logic into small functions to make + the code more readable and easier to test. * The function self.set_environment_variables should be called by all - wrappers even if the MET tool does not have a config file. This is done - to set environment variables that MET expects to be set when running, such - as MET_TMP_DIR and MET_PYTHON_EXE. If no environment variables need to be - set specific to the wrapper, then no - implementation of the function in the wrapper needs to be written. - Call the + wrappers even if the MET tool does not have a config file. + This function is typically called from the run_at_time_once function. + This is done to set environment variables that MET expects to be set when + running, such as MET_TMP_DIR and MET_PYTHON_EXE. + If no environment variables need to be set specific to the wrapper, then no + implementation of the function in the wrapper needs to be written. Call the implementation of the function from CommandBuilder, which sets the environment variables defined in the [user_env_vars] section of the configuration file and outputs DEBUG logs for each environment variable @@ -212,7 +174,7 @@ put the logic to build the command in the run_at_time method. each wrapper. * Once all the necessary information has been provided to create the MET - command, call self.build_and_run_command(). This calls self.get_command() + command, call self.build(). This calls self.get_command() to assemble the command and verify that the command wrapper generated contains all of the required arguments. The get_command() in the wrapper may need to be overridden if the MET application is different from @@ -224,7 +186,9 @@ put the logic to build the command in the run_at_time method. * Call self.clear() at the beginning of each loop iteration that tries to build/run a MET command to prevent inadvertently reusing/re-running - commands that were previously created. + commands that were previously created. This is called in the RuntimeFreq + wrapper before each call to run_at_time_once, but an additional call may be + needed if multiple commands are built and run in this function. * To allow the use case to use the specific wrapper, assign the wrapper name to PROCESS_LIST:: @@ -262,12 +226,12 @@ put the logic to build the command in the run_at_time method. documentation for that use case and a README file to create a header for the documentation page. -This new uuse case/example configuration file is located in a directory structure +This new use case/example configuration file is located in a directory structure like the following:: - METplus/parm/use_cases/met_tool_wrapper/NewTool/NewTool.conf - METplus/docs/use_cases/met_tool_wrapper/NewTool/NewTool.py - METplus/docs/use_cases/met_tool_wrapper/NewTool/README.md + parm/use_cases/met_tool_wrapper/NewTool/NewTool.conf + docs/use_cases/met_tool_wrapper/NewTool/NewTool.py + docs/use_cases/met_tool_wrapper/NewTool/README.rst Note the documentation file is in METplus/docs while the use case conf file is in METplus/parm. diff --git a/docs/Contributors_Guide/deprecation.rst b/docs/Contributors_Guide/deprecation.rst index 6c6d63e2f..1703d7a2c 100644 --- a/docs/Contributors_Guide/deprecation.rst +++ b/docs/Contributors_Guide/deprecation.rst @@ -26,60 +26,53 @@ wrong variable and it is using WGRIB2 = wgrib2. check_for_deprecated_config() ----------------------------- -In **metplus/util/config_metplus.py** there is a function called -check_for_deprecated_config. It contains a dictionary of dictionaries -called deprecated_dict that specifies the old config name, the section -it was found in, and a suggested alternative (None if no alternative -exists). +In **metplus/util/constants.py** there is a dictionary called +DEPRECATED_DICT that specifies the old config name as the key. +The value is a dictionary of info that is used to help users update their +config files. + +* **alt**: optional suggested alternative name for the deprecated config. + This can be a single variable name or text to describe multiple variables + or how to handle it. + Set to None or leave unset to tell the user to just remove the variable. +* **copy**: optional item (defaults to True). Set this to False if one + cannot simply replace the deprecated variable name with the value in *alt*. + If True, easy-to-run sed commands are generated to help replace variables. +* **upgrade**: optional item where the value is a keyword that will output + additional instructions for the user, e.g. *ensemble*. + +If any of these old variables are found in any config file passed to +METplus by the user, an error report will be displayed with the old +variables and suggested new ones if applicable. **Example 1** :: -'WGRIB2_EXE' : {'sec' : 'exe', 'alt' : 'WGRIB2'} +'WGRIB2_EXE' : {'alt' : 'WGRIB2'} -This says that WGRIB2_EXE was found in the [exe] section and should -be replaced with WGRIB2. +This means WGRIB2_EXE was found in the config and should be replaced with WGRIB2. **Example 2** :: -'PREPBUFR_DIR_REGEX' : {'sec' : 'regex_pattern', 'alt' : None} - -This says that [regex_pattern] PREPBUFR_DIR_REGEX is no longer used -and there is no alternative (because the wrapper uses filename -templates instead of regex now). - +'PREPBUFR_DIR_REGEX' : {'alt' : None} -If any of these old variables are found in any config file passed to -METplus by the user, an error report will be displayed with the old -variables and suggested new ones if applicable. +This means PREPBUFR_DIR_REGEX is no longer used and there is no alternative. +The variable can simply be removed from the config file. -If support for an old config variable is temporarily needed, the -user should be warned to update their config file because the -variable will be phased out in the future. In this case, add the -‘req’ item to the dictionary and set it to False. This will provide -a warning to the user but will not stop the execution of the code. -If this is done, be sure to modify the code to check for the new -config variable, and if it is not set, check the old config variable -to see if it is set. - -**Example** +**Example 3** :: -'LOOP_METHOD' : {'sec' : 'config', 'alt' : 'LOOP_ORDER', 'req' : False} - -This says that [config] LOOP_METHOD is deprecated and the user -should use LOOP_ORDER, but it is not required to change -immediately. If this is done, it is important to -check for LOOP_ORDER and then -check for LOOP_METHOD if it is not set. +'SOME_VAR' : {'alt': 'OTHER_VAR', 'copy' : None} -In run_metplus.py: +This means SOME_VAR is no longer used. OTHER_VAR is the variable that should +be set instead, but the value must change slightly. +The variable name SOME_VAR cannot simply be replaced with OTHER_VAR. +**Example 4** :: - loop_order = config.getstr('config', 'LOOP_ORDER', '') - if loop_order == '': - loop_order = config.getstr('config', 'LOOP_METHOD') - +'ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON': {'upgrade': 'ensemble'}, +This means that ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON is no longer used and can +be removed. Additional text will be output to describe how to upgrade. diff --git a/docs/Contributors_Guide/github_workflow.rst b/docs/Contributors_Guide/github_workflow.rst index fd1451778..4b623a69c 100644 --- a/docs/Contributors_Guide/github_workflow.rst +++ b/docs/Contributors_Guide/github_workflow.rst @@ -34,7 +34,7 @@ time window for that cycle or reassigned to a future development cycle. Each development cycle culminates in the creation of a software release. The -:ref:`releaseCycleStages` section describes the various types of software releases +:ref:`releaseTypes` section describes the various types of software releases (development, official, or bugfix). Each development cycle culminates in a beta release, a release candidate, or the official release. Generally, a **beta** development cycle results in a **beta** development release while an **rc** development cycle results in an **rc** @@ -63,7 +63,7 @@ are required to perform the following steps. and select **Settings**. Modify these settings as follows. - Project name: The default project name is **@UserNames's feature**. Rename it as - **{METplus Component}-{Target Version Number} Development** (e.g. **METplus-5.1.0 Development**). + **{METplus Component}-{Target Version Number} Development** (e.g. **METplus-Wrappers-5.1.0 Development**). - Add a description: Add **Development toward {METplus Component} version {Target Version Number}.** @@ -180,7 +180,7 @@ to that support project. Each fix is assigned to the current bugfix milestone o the corresponding source code repository. -The :ref:`releaseCycleStages` section describes the various types of software releases +The :ref:`releaseTypes` section describes the various types of software releases (development, official, or bugfix). The GitHub support project contains issues and pull requests that apply only to bugfix releases. @@ -204,7 +204,7 @@ required to perform the following steps. and select **Settings**. Modify these settings as follows. - Project name: The default project name is **@UserNames's feature**. Rename it as - **METplus Version X.Y Support** (e.g. **METplus Version 5.0 Support**). + **Coorindated METplus-X.Y Support** (e.g. **Coordinated METplus-5.0 Support**). - Add a description: Add **Issues related to support for the METplus X.Y coordinated release.** diff --git a/docs/Release_Guide/coordinated.rst b/docs/Release_Guide/coordinated.rst new file mode 100644 index 000000000..4abd404a1 --- /dev/null +++ b/docs/Release_Guide/coordinated.rst @@ -0,0 +1,23 @@ +******************* +Coordinated Release +******************* + +.. |projectRepo| replace:: Coordinated + +Create a new METplus coordinated release from vX.Y.Z official or +bugfix releases of the METplus components. Typically, a coordinated +release consists entirely of official component releases prior to any +bugfix releases being issued. However, the latter is certainly possible. +In fact, whenever a bugfix release is created for a METplus component, +the corresponding coordinated release is updated to link to the most +recent bugfix version. + +The following instructions assume that all of the official or +bugfix component releases have already been created. Note, however, that +some of these steps can be started prior the completion of the +component releases. + +.. include:: release_steps/coordinated/update_dtc_website.rst +.. include:: release_steps/coordinated/finalize_release_on_github.rst +.. include:: release_steps/coordinated/update_zenodo.rst +.. include:: release_steps/coordinated/announce_release.rst diff --git a/docs/Release_Guide/index.rst b/docs/Release_Guide/index.rst index 6956f5311..bde0abc8d 100644 --- a/docs/Release_Guide/index.rst +++ b/docs/Release_Guide/index.rst @@ -4,13 +4,42 @@ Release Guide This METplus Release Guide provides detailed instructions for METplus developers for creating software releases for the METplus component -repositories. **This Release Guide is intended for developers creating -releases and is not intended for users of the software.** +repositories. -.. _releaseCycleStages: +.. note:: This Release Guide is intended for developers creating + releases and is not intended for users of the software. -Stages of the METplus Release Cycle -=================================== +.. _releaseTypes: + +Release Types +============= + +Coordinated Release +------------------- + +A METplus coordinated release is a group of official or bugfix releases for each +of the METplus components that have been developed and tested in parallel. +Coordinated release announcements on the +`DTC METplus Downloads `_ +page link to the component releases that comprise the coordinated release. +When bugfix releases are issued for any METplus component, the corresponding +coordinated release announcement is updated to link to the most recent bugfix +version. + +Official Release +---------------- + +An official release is a stable release of a METplus component and typically matches +the release candidate, which has passed all tests. It is the version of the +code that has been tested as thoroughly as possible and is reliable enough to be +used in production. + +Bugfix Release +-------------- + +A bugfix release for a METplus component introduces no new features, but fixes +bugs in previous official releases and targets the most critical bugs affecting +users. Development Release ------------------- @@ -18,9 +47,9 @@ Development Release Beta ^^^^ -Beta releases are a pre-release of the software to give a larger group of -users the opportunity to test the recently incorporated new features, -enhancements, and bug fixes. Beta releases allow for continued +Beta releases are a pre-release of a METplus software component to give a +larger group of users the opportunity to test the recently incorporated new +features, enhancements, and bug fixes. Beta releases allow for continued development and bug fixes before an official release. There are many possible configurations of hardware and software that exist and installation of beta releases allow for testing of potential conflicts. @@ -28,28 +57,14 @@ of beta releases allow for testing of potential conflicts. Release Candidate (rc) ^^^^^^^^^^^^^^^^^^^^^^ -A release candidate is a version of the software that is nearly ready for -official release but may still have a few bugs. At this stage, all product -features have been designed, coded, and tested through one or more beta +A release candidate is a version of a METplus software component that is nearly +ready for official release but may still have a few bugs. At this stage, all +product features have been designed, coded, and tested through one or more beta cycles with no known bugs. It is code complete, meaning that no entirely new source code will be added to this release. There may still be source code changes to fix bugs, changes to documentation, and changes to test cases or utilities. -Official Release ----------------- - -An official release is a stable release and is basically the release -candidate, which has passed all tests. It is the version of the code that -has been tested as thoroughly as possible and is reliable enough to be -used in production. - -Bugfix Release --------------- - -A bugfix release introduces no new features, but fixes bugs in previous -official releases and targets the most critical bugs affecting users. - Release Support Policy ====================== @@ -68,24 +83,28 @@ to report any bugs, please contact our dedicated support team in the Instructions Summary ==================== -Instructions are provided for three types of software releases: +Instructions are provided for the following types of software releases: + +#. **Coordinated Release** consisting of a group of software component releases -#. **Official Release** (e.g. vX.Y.Z) from the develop branch (becomes the new main_vX.Y branch) +#. **Official Release** (e.g. vX.Y.0) from the develop branch (becomes the new main_vX.Y branch) #. **Bugfix Release** (e.g. vX.Y.Z) from the corresponding main_vX.Y branch #. **Development Release** (e.g. vX.Y.Z-betaN or vX.Y.Z-rcN) from the develop branch -The instructions that are common to all components are documented only once and then included in the release steps for all components. -However some instructions are specific to individual repositories and documented separately. +The instructions that are common to all components are documented only once and then included +in the release steps for all components. However some instructions are specific to individual +repositories and documented separately. -Release instructions for each of the METplus components are described in the following chapters. +Release instructions are described in the following chapters. .. toctree:: :titlesonly: :maxdepth: 1 :numbered: 4 + coordinated metplus met metdataio @@ -94,11 +113,3 @@ Release instructions for each of the METplus components are described in the fol metviewer metexpress recreate_release - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/Release_Guide/metplus_official.rst b/docs/Release_Guide/metplus_official.rst index f2a9acb2d..a2d77cac9 100644 --- a/docs/Release_Guide/metplus_official.rst +++ b/docs/Release_Guide/metplus_official.rst @@ -24,5 +24,4 @@ Create a new vX.Y.Z official release from the develop branch. .. include:: release_steps/metplus/update_version_on_develop.rst .. include:: release_steps/update_docs_official.rst .. include:: release_steps/metplus/update_web_server_data.rst -.. include:: release_steps/metplus/update_zenodo.rst .. include:: release_steps/set_beta_deletion_reminder_official.rst diff --git a/docs/Release_Guide/release_steps/common/update_dtc_website.rst b/docs/Release_Guide/release_steps/common/update_dtc_website.rst index bbd39bc68..7f8b1be69 100644 --- a/docs/Release_Guide/release_steps/common/update_dtc_website.rst +++ b/docs/Release_Guide/release_steps/common/update_dtc_website.rst @@ -6,24 +6,24 @@ Update DTC Website * Navigate to the downloads page for the |projectRepo| repository at https://dtcenter.org/community-code/metplus/download -* Click on the Edit button to edit the Downloads page. +* Click on the **Edit** button to edit the Downloads page. -* Create a new *Software Release* for the newly released version by clicking - on *Add New Release*. +* Create a new **Software Release** for the newly released version by clicking + on **Add New Release**. - * For *Full Title of Release* type "|projectRepo| Version X.Y.Z". + * For **Full Title of Release** type "|projectRepo| Version X.Y.Z". - * For *Related Community Code* select both the METplus and the |projectName| + * For **Related Community Code** select both the "METplus" and the "|projectName|" options (For Macs, hold the Command key to select both). - * For *Version Label* type "|projectRepo| X.Y.Z-betaN". + * For **Version Label** type "|projectRepo| X.Y.Z-betaN". - * Select the release type (*Recommended* for official or bugfix releases or - *Development* for development versions). + * Select the **Release Type** ("Recommended" for official or bugfix releases or + "Development" for development versions). - * Enter the release date. + * Enter the **Release Date**. - * Click on *Add Code Download* then click *Add Link* to add links for each of the following: + * Click on **Add Code Download** then click **Add Link** to add links for each of the following: * Add Link: |addTarfileStep| @@ -41,38 +41,30 @@ Update DTC Website (If creating a new official release, be sure to add a new *Existing Builds and Docker* page, if one was not already created.) - * Inside the text box in the "Release Notes" section provide a direct link to - the *release-notes.html* file in the User's Guide. + * In the **Release Notes** text box provide a direct link to the + *release-notes.html* file in the User's Guide. - * Click on "Create Release". + * Click on **Create Release**. * Update the existing releases, as needed. * For a development release, ensure the "Release Type" is set to - *Development* and change any previous *Development* versions to - *Other*. + **Development** and change any previous **Development** versions to + **Other**. * For a bugfix or official release, change any previous - *Recommended* versions to *Other*. + **Recommended** versions to **Other**. * For an official release, remove the corresponding development releases. - * Create or edit the "Coordinated METplus Version X.Y" software release. - - * For an official release, create the "Coordinated METplus Version X.Y" - release entry if it doesn't already exist. Ensure the "Release Type" - is set to *Recommended*. Consider changing the "Release Type" of - previous coordinated releases from *Recommended* to *Other*. Add - links for the |projectRepo| X.Y.Z, the METplus "Documentation", - the METplus "Existing Builds and Docker" page, and the "Release Notes". - Make the Release Notes a link to the |projectRepo| Release Notes. + * Edit the "Coordinated METplus Version X.Y" software release. * For a bugfix release, update the existing link and text in - the "Coordinated METplus Version X.Y" release section with the + the "Coordinated METplus-X.Y" release section with the X.Y.Z+1 information. * |otherWebsiteUpdates| - * Click on "Save". + * Click on **Save** at the bottom of the page. diff --git a/docs/Release_Guide/release_steps/coordinated/announce_release.rst b/docs/Release_Guide/release_steps/coordinated/announce_release.rst new file mode 100644 index 000000000..9a29d68c9 --- /dev/null +++ b/docs/Release_Guide/release_steps/coordinated/announce_release.rst @@ -0,0 +1,6 @@ +Announce Release +---------------- + +* Contact the METplus project manager to announce the coordinated release via email. + +* Contact the RAL-IT group to request that the coordinated release components be installed in */usr/local* to be used on all RAL machines. diff --git a/docs/Release_Guide/release_steps/coordinated/finalize_release_on_github.rst b/docs/Release_Guide/release_steps/coordinated/finalize_release_on_github.rst new file mode 100644 index 000000000..94172b01b --- /dev/null +++ b/docs/Release_Guide/release_steps/coordinated/finalize_release_on_github.rst @@ -0,0 +1,5 @@ +Finalize Release on GitHub +-------------------------- + +See :ref:`wo-support-project` to create a support project for +the current METplus coordinated release. diff --git a/docs/Release_Guide/release_steps/coordinated/update_dtc_website.rst b/docs/Release_Guide/release_steps/coordinated/update_dtc_website.rst new file mode 100644 index 000000000..6082e88ca --- /dev/null +++ b/docs/Release_Guide/release_steps/coordinated/update_dtc_website.rst @@ -0,0 +1,66 @@ +Update DTC Website +------------------ + +* Navigate to https://dtcenter.org and sign in to the Drupal interface. + +* Navigate to the METplus downloads page at + https://dtcenter.org/community-code/metplus/download + +* Click on the **Edit** button to edit the Downloads page. + +* Create a new **Software Release** for the new coordinated release by clicking + on **Add New Release**. + + * For **Full Title of Release** type "Coorindated METplus X.Y". + + * For **Related Community Code** select only the "METplus" option. + + * For **Version Label** type "Coordinated METplus X.Y". + + * Select the **Release Type** as "Recommended". + + * Select the **Release Options** as "Coordinated". + + * Enter the **Release Date**. + + * Click on **Add Code Download** then click **Add Link** to add links for each of the following: + + * Add Link: Link text should be "METplus X.Y.Z" and the URL should be a link to the METplus component DTC release page. + + * Add Link: Link text should be "MET X.Y.Z" and the URL should be a link to the MET component DTC release page. + + * Add Link: Link text should be "METviewer X.Y.Z" and the URL should be a link to the METviewer component DTC release page. + + * Add Link: Link text should be "METexpress X.Y.Z" and the URL should be a link to the METexpress component DTC release page. + + * Add Link: Link text should be "METplotpy X.Y.Z" and the URL should be a link to the METplotpy component DTC release page. + + * Add Link: Link text should be "METcalcpy X.Y.Z" and the URL should be a link to the METcalcpy component DTC release page. + + * Add Link: Link text should be "METdataio X.Y.Z" and the URL should be a link to the METdataio component DTC release page. + + * Add Link: Link text should be "Documentation" and the URL should be the top + level directory of the main_vX.Y branch of the METplus User's Guide hosted on the web. + For example, use + "https://metplus.readthedocs.io/en/main_vX.Y/Users_Guide/" and NOT + "https://metplus.readthedocs.io/en/vX.Y.Z/Users_Guide/" + + * Add Link: Link text should be "Existing Builds and Docker" and the URL + should be the latest Existing Builds page, i.e. + https://dtcenter.org/community-code/metplus/metplus-X-Y-existing-builds + + * In the **Release Notes** text box provide direct links to the *release-notes.html* + files on the main_vX.Y branch of the User's Guide for each component. + + * Click on **Create Release**. + + * Update any existing coordinated releases by changing the **Release Type** from + "Recommended" to "Other" and click the **Update Release** button. + + * Review the existing component releases and remove any remaining development + releases (e.g. beta and rc) for any of the official releases included in this + coordinated release. + + * Click on **Save** at the bottom of the page. + +* Create a new **Existing Builds and Docker** page for the next coordinated release. diff --git a/docs/Release_Guide/release_steps/metplus/update_zenodo.rst b/docs/Release_Guide/release_steps/coordinated/update_zenodo.rst similarity index 100% rename from docs/Release_Guide/release_steps/metplus/update_zenodo.rst rename to docs/Release_Guide/release_steps/coordinated/update_zenodo.rst diff --git a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst index 85c366ea1..0fd714ccf 100644 --- a/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst +++ b/docs/Release_Guide/release_steps/finalize_release_on_github_official.rst @@ -21,9 +21,6 @@ Finalize Release on GitHub * Close the existing development project for the current milestone. - * If necessary, see :ref:`wo-support-project` to create a support project for the current - METplus coordinated release. - * If necessary, see :ref:`wo-development-project` to create a development project for the next milestone. @@ -37,6 +34,7 @@ Finalize Release on GitHub https://github.com/dtcenter/|projectRepo| -> Settings - -> Branches (tab on left) - -> change the drop down to new branch (click on stacked arrows icon next to the edit icon) - -> update branch protection rules + -> Scroll down to the *Default branch* section + -> Click the stacked arrows button next to default branch name + -> Select the new default branch from the dropdown list + -> Click the *Update* button diff --git a/docs/Release_Guide/release_steps/met/update_dtc_website.rst b/docs/Release_Guide/release_steps/met/update_dtc_website.rst index 064eecf01..351ddd8ef 100644 --- a/docs/Release_Guide/release_steps/met/update_dtc_website.rst +++ b/docs/Release_Guide/release_steps/met/update_dtc_website.rst @@ -2,6 +2,6 @@ .. |projectName| replace:: |projectRepo| -.. |addTarfileStep| replace:: Link text should be the file name of the tar file and the URL should be the .tar.gz file created in the "Attach Release Tarfile" step. +.. |addTarfileStep| replace:: Link text should be the name of the release and the URL should be the release page that was just created under the GitHub Releases tab. .. |otherWebsiteUpdates| replace:: Make any other necessary website updates. For example, the flowchart at https://dtcenter.org/community-code/model-evaluation-tools-met/system-architecture. diff --git a/docs/Release_Guide/release_steps/set_beta_deletion_reminder_official.rst b/docs/Release_Guide/release_steps/set_beta_deletion_reminder_official.rst index 5665f7e1b..ef9700638 100644 --- a/docs/Release_Guide/release_steps/set_beta_deletion_reminder_official.rst +++ b/docs/Release_Guide/release_steps/set_beta_deletion_reminder_official.rst @@ -1,8 +1,12 @@ -Set up Reminder to Delete Beta Tags ------------------------------------ +Set up Reminder to Delete Beta/RC Tags and Releases +--------------------------------------------------- -Help keep the GitHub repositories and DockerHub clean by removing beta tags. -Do not delete the beta tags for this release right away. Please set a +Help keep the GitHub repositories and DockerHub clean by removing +beta/rc tags and releases. +Do not delete the tags/releases for this release right away. Please set a calendar reminder or schedule an email to be sent two weeks from the release -date as a reminder to delete the beta tags in both GitHub and DockerHub -(if applicable). +date as a reminder to delete the tags/releases. + +In GitHub, first delete all of the releases that contain beta or rc in the name, +then delete all corresponding tags. +Delete any beta/rc tags in DockerHub if applicable. diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index afbdbf872..586318c04 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -4,7 +4,7 @@ METplus Release Information .. _release-notes: -Users can view the :ref:`releaseCycleStages` section of +Users can view the :ref:`releaseTypes` section of the Release Guide for descriptions of the development releases (including beta releases and release candidates), official releases, and bugfix releases for the METplus Components. diff --git a/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py b/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py index 5cac8f53c..4ff1b3526 100644 --- a/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py +++ b/docs/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.py @@ -190,7 +190,6 @@ # .. note:: # # * MediumRangeAppUseCase -# * TCPairsToolUseCase # * SeriesByLeadUseCase # * MTDToolUseCase # * RegridDataPlaneToolUseCase diff --git a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py index ca76888ba..9aa994467 100644 --- a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py +++ b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.py @@ -144,7 +144,6 @@ # * NOAAEMCOrgUseCase # * RegriddinginToolUseCase # * ObsTimeSummaryUseCase -# * ClimatologyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py index fe8d4a3ec..7d3229ab8 100644 --- a/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py +++ b/docs/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsNAM_Sfc_MultiField_PrepBufr.py @@ -146,7 +146,6 @@ # * NOAAEMCOrgUseCase # * RegriddinginToolUseCase # * ObsTimeSummaryUseCase -# * ClimatologyUseCase # # Navigate to the :ref:`quick-search` page to discover other similar use cases. # diff --git a/metplus/util/doc_util.py b/internal/scripts/dev_tools/add_met_config_helper.py similarity index 97% rename from metplus/util/doc_util.py rename to internal/scripts/dev_tools/add_met_config_helper.py index 05b826949..6cd8faaef 100755 --- a/metplus/util/doc_util.py +++ b/internal/scripts/dev_tools/add_met_config_helper.py @@ -11,10 +11,13 @@ import os try: - from .string_manip import get_wrapper_name + from metplus.util.string_manip import get_wrapper_name except ImportError: - sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) - from string_manip import get_wrapper_name + # if metplus package is not installed, find util relative to this script + metplus_home = os.path.join(os.path.dirname(__file__), + os.pardir, os.pardir, os.pardir) + sys.path.insert(0, os.path.abspath(metplus_home)) + from metplus.util.string_manip import get_wrapper_name SCRIPT_INFO_TEXT = ( 'This script is intended to help developers add support for setting ' diff --git a/internal/tests/pytests/requirements.txt b/internal/tests/pytests/requirements.txt new file mode 100644 index 000000000..0a4449537 --- /dev/null +++ b/internal/tests/pytests/requirements.txt @@ -0,0 +1,19 @@ +certifi==2023.7.22 +cftime==1.6.2 +coverage==7.2.7 +exceptiongroup==1.1.2 +iniconfig==2.0.0 +netCDF4==1.6.4 +numpy==1.25.2 +packaging==23.1 +pandas==2.0.3 +pdf2image==1.16.3 +Pillow==10.0.0 +pluggy==1.2.0 +pytest==7.4.0 +pytest-cov==4.1.0 +python-dateutil==2.8.2 +pytz==2023.3 +six==1.16.0 +tomli==2.0.1 +tzdata==2023.3 diff --git a/internal/tests/pytests/util/run_util/no_install_run_util.conf b/internal/tests/pytests/util/run_util/no_install_run_util.conf new file mode 100644 index 000000000..f5c8b5e84 --- /dev/null +++ b/internal/tests/pytests/util/run_util/no_install_run_util.conf @@ -0,0 +1,19 @@ +[config] +INPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/input +OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID} +MET_INSTALL_DIR = False + +DO_NOT_RUN_EXE = True + +LOG_LEVEL = DEBUG +LOG_LEVEL_TERMINAL = WARNING +LOG_MET_OUTPUT_TO_METPLUS = no +LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s +LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_DEBUG_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} + +LOG_METPLUS = {LOG_DIR}/metplus.log +LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists diff --git a/internal/tests/pytests/util/run_util/run_util.conf b/internal/tests/pytests/util/run_util/run_util.conf new file mode 100644 index 000000000..6c2145ab7 --- /dev/null +++ b/internal/tests/pytests/util/run_util/run_util.conf @@ -0,0 +1,22 @@ +[config] +INPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/input +OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID} +MET_INSTALL_DIR = {ENV[METPLUS_TEST_OUTPUT_BASE]} + +DO_NOT_RUN_EXE = True + +LOG_LEVEL = DEBUG +LOG_LEVEL_TERMINAL = WARNING +LOG_MET_OUTPUT_TO_METPLUS = no +LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s +LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_DEBUG_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} + +LOG_METPLUS = {LOG_DIR}/metplus.log +LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists + +[user_env_vars] +GODS_OF_WEATHER = Indra_Thor_Zeus \ No newline at end of file diff --git a/internal/tests/pytests/util/run_util/sed_run_util.conf b/internal/tests/pytests/util/run_util/sed_run_util.conf new file mode 100644 index 000000000..a25b16fd0 --- /dev/null +++ b/internal/tests/pytests/util/run_util/sed_run_util.conf @@ -0,0 +1,25 @@ +[config] +INPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/input +OUTPUT_BASE = {ENV[METPLUS_TEST_OUTPUT_BASE]}/test_output/{RUN_ID} +MET_INSTALL_DIR = {ENV[METPLUS_TEST_OUTPUT_BASE]} + +DO_NOT_RUN_EXE = True + +LOG_LEVEL = DEBUG +LOG_LEVEL_TERMINAL = WARNING +LOG_MET_OUTPUT_TO_METPLUS = no +LOG_LINE_FORMAT = (%(filename)s) %(levelname)s: %(message)s +LOG_ERR_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_DEBUG_LINE_FORMAT = {LOG_LINE_FORMAT} +LOG_INFO_LINE_FORMAT = {LOG_LINE_FORMAT} + +LOG_METPLUS = {LOG_DIR}/metplus.log +LOG_TIMESTAMP_TEMPLATE = +METPLUS_CONF = {OUTPUT_BASE}/metplus_final.conf +FILE_LISTS_DIR = {STAGING_DIR}/file_lists + +REGRID_TO_GRID = chessboard +GRID_VX = True + +ENSEMBLE_STAT_NBRHD_PROB_WIDTH = wide +ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON = maybe \ No newline at end of file diff --git a/internal/tests/pytests/util/run_util/test_run_util.py b/internal/tests/pytests/util/run_util/test_run_util.py new file mode 100644 index 000000000..f1d4fb347 --- /dev/null +++ b/internal/tests/pytests/util/run_util/test_run_util.py @@ -0,0 +1,100 @@ +import os +import pytest +from unittest import mock +import metplus.util.run_util as ru + +EXPECTED_CONFIG_KEYS = ['CLOCK_TIME', 'MET_INSTALL_DIR', 'MET_BIN_DIR', + 'INPUT_BASE', 'OUTPUT_BASE', 'METPLUS_CONF', + 'TMP_DIR', 'STAGING_DIR', 'FILE_LISTS_DIR', 'CONVERT', + 'GEMPAKTOCF_JAR', 'GFDL_TRACKER_EXEC', + 'PROCESS_LIST', 'OMP_NUM_THREADS', + 'SCRUB_STAGING_DIR', 'LOG_METPLUS', 'LOG_DIR', + 'LOG_TIMESTAMP_TEMPLATE', 'LOG_TIMESTAMP_USE_DATATIME', + 'LOG_MET_OUTPUT_TO_METPLUS', 'LOG_LEVEL', + 'LOG_LEVEL_TERMINAL', 'LOG_MET_VERBOSITY', + 'LOG_LINE_FORMAT', 'LOG_ERR_LINE_FORMAT', + 'LOG_DEBUG_LINE_FORMAT', 'LOG_INFO_LINE_FORMAT', + 'LOG_LINE_DATE_FORMAT', 'DO_NOT_RUN_EXE', 'CONFIG_INPUT', + 'RUN_ID', 'LOG_TIMESTAMP', 'METPLUS_BASE', 'PARM_BASE', + 'METPLUS_VERSION'] + + +def get_run_util_configs(conf_name): + script_dir = os.path.dirname(__file__) + return [os.path.join(script_dir, conf_name)] + + +@pytest.mark.util +def test_pre_run_setup(): + conf_inputs = get_run_util_configs('run_util.conf') + actual = ru.pre_run_setup(conf_inputs) + + # check all config keys are set correctly + assert sorted(actual.keys('config')) == sorted(EXPECTED_CONFIG_KEYS) + + # spot check a few specific items + expected_stage = os.path.join(actual.get('config', 'OUTPUT_BASE'), 'stage') + assert actual.get('config', 'STAGING_DIR') == expected_stage + assert actual.get('user_env_vars', 'GODS_OF_WEATHER', 'Indra_Thor_Zeus') + + +@pytest.mark.util +def test_pre_run_setup_env_vars(): + with mock.patch.dict('os.environ', {'MY_ENV_VAR': '42','OMP_NUM_THREADS': '4'}): + conf_inputs = get_run_util_configs('run_util.conf') + actual = ru.pre_run_setup(conf_inputs) + assert actual.env['MY_ENV_VAR'] == '42' + assert actual.get('config', 'OMP_NUM_THREADS') == '4' + + +@pytest.mark.util +def test_pre_run_setup_sed_file(capfd): + conf_inputs = get_run_util_configs('sed_run_util.conf') + + with mock.patch.object(ru.sys, 'exit') as mock_sys: + with mock.patch.object(ru, 'validate_config_variables', return_value=(False, ['sed command 1', 'sed command 2'])): + actual = ru.pre_run_setup(conf_inputs) + mock_sys.assert_called_with(1) + + # check sed file is written correctly + sed_file = os.path.join(actual.getdir('OUTPUT_BASE'), 'sed_commands.txt') + assert os.path.exists(sed_file) + with open(sed_file, 'r') as f: + assert f.read() == 'sed command 1\nsed command 2\n' + + # check correct errors logged + out, err = capfd.readouterr() + expected_error_msgs = [f'Find/Replace commands have been generated in {sed_file}', + 'ERROR: Correct configuration variables and rerun. Exiting.'] + for msg in expected_error_msgs: + assert msg in err + + +@pytest.mark.util +def test_pre_run_setup_deprecated(capfd): + conf_inputs = get_run_util_configs('sed_run_util.conf') + + with mock.patch.object(ru.sys, 'exit') as mock_sys: + actual = ru.pre_run_setup(conf_inputs) + mock_sys.assert_called_with(1) + + out, err = capfd.readouterr() + + expected_error_msgs = [ + 'ERROR: DEPRECATED CONFIG ITEMS WERE FOUND. PLEASE FOLLOW THE INSTRUCTIONS TO UPDATE THE CONFIG FILES', + 'ERROR: ENSEMBLE_STAT_ENSEMBLE_FLAG_LATLON should be removed', + 'ERROR: ENSEMBLE_STAT_NBRHD_PROB_WIDTH should be removed', + ] + for msg in expected_error_msgs: + assert msg in err + +@pytest.mark.util +def test_pre_run_setup_no_install(capfd): + conf_inputs = get_run_util_configs('no_install_run_util.conf') + + with mock.patch.object(ru.sys, 'exit') as mock_sys: + actual = ru.pre_run_setup(conf_inputs) + mock_sys.assert_called_with(1) + + out, err = capfd.readouterr() + assert 'MET_INSTALL_DIR must be set correctly to run METplus' in err diff --git a/internal/tests/pytests/util/system_util/test_system_util.py b/internal/tests/pytests/util/system_util/test_system_util.py index 6a829a05a..14f78f9e1 100644 --- a/internal/tests/pytests/util/system_util/test_system_util.py +++ b/internal/tests/pytests/util/system_util/test_system_util.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import os import pytest +from unittest import mock -from metplus.util.system_util import * +import metplus.util.system_util as su @pytest.mark.parametrize( 'filename, expected_result', [ @@ -28,7 +30,7 @@ def test_get_storm_ids(metplus_config, filename, expected_result): 'stat_data', filename) - assert get_storms(filepath, id_only=True) == expected_result + assert su.get_storms(filepath, id_only=True) == expected_result @pytest.mark.parametrize( @@ -57,7 +59,7 @@ def test_get_storms(metplus_config, filename, expected_result): 'stat_data', filename) - storm_dict = get_storms(filepath) + storm_dict = su.get_storms(filepath) print(storm_dict) assert list(storm_dict.keys()) == expected_result for storm_id in expected_result[1:]: @@ -86,7 +88,7 @@ def test_get_storms_mtd(metplus_config): 'mtd', 'fake_mtd_2d.txt') - storm_dict = get_storms(filepath, sort_column=sort_column) + storm_dict = su.get_storms(filepath, sort_column=sort_column) print(storm_dict) assert list(storm_dict.keys()) == expected_result for storm_id in expected_result[1:]: @@ -124,7 +126,7 @@ def test_preprocess_file_stage(metplus_config, filename, ext): else: stagepath = filepath - outpath = preprocess_file(filepath, None, conf) + outpath = su.preprocess_file(filepath, None, conf) assert stagepath == outpath and os.path.exists(outpath) @@ -155,5 +157,119 @@ def test_preprocess_file_options(metplus_config, if filename == 'dir': filename = config.getdir('METPLUS_BASE') expected = filename - result = preprocess_file(filename, data_type, config, allow_dir) + result = su.preprocess_file(filename, data_type, config, allow_dir) assert result == expected + + +@pytest.mark.parametrize( + 'input_exists,expected', [ + (False, '/some/fake/file.bigfoot'), + (True, None) + ] +) +@pytest.mark.util +def test_preprocess_file_not_exist(metplus_config, input_exists, expected): + config = metplus_config + config.set('config', 'INPUT_MUST_EXIST', input_exists) + result = su.preprocess_file('/some/fake/file.bigfoot', None, config) + assert result == expected + + +@pytest.mark.util +def test_preprocess_file_gempack(tmp_path_factory, metplus_config): + config = metplus_config + + # setup files and paths + tmp_dir = tmp_path_factory.mktemp('gempak') + config.set('config', 'STAGING_DIR', '') + file_path = os.path.join(tmp_dir, 'some_file.grd') + open(file_path, 'a').close() + expected = os.path.join(tmp_dir, 'some_file.nc') + + # we need to import so as to mock .build() + from metplus.wrappers import GempakToCFWrapper + + with mock.patch.object(GempakToCFWrapper, 'build') as mock_build: + result = su.preprocess_file(file_path, 'GEMPAK', config) + mock_build.assert_called_once() + + assert result == expected + + +@pytest.mark.parametrize( + 'expected, user_err, id_err', [ + ('James(007)', None, None), + ('007', OSError, None), + ('James', None, AttributeError), + ('', OSError, AttributeError), + ] +) +@pytest.mark.util +def test_get_user_info(expected, user_err, id_err): + with mock.patch.object(su.getpass, 'getuser', return_value='James', side_effect=user_err): + with mock.patch.object(su.os, 'getuid', return_value='007', side_effect=id_err): + actual = su.get_user_info() + assert actual == expected + + +@pytest.mark.util +def test_write_list_to_file(tmp_path_factory): + filename = tmp_path_factory.mktemp('util') / 'temp.txt' + output_list =['some', 'file', 'content'] + su.write_list_to_file(filename, output_list) + + with open(filename, 'r') as f: + actual = f.read() + + assert actual == '\n'.join(output_list) + '\n' + + +@pytest.mark.util +def test_prune_empty(tmp_path_factory): + prune_dir = tmp_path_factory.mktemp('prune') + + dir1 = prune_dir / 'empty_file_dir' + dir2 = prune_dir / 'not_empty_file_dir' + dir3 = prune_dir / 'empty_dir' + for d in [dir1, dir2, dir3]: + os.makedirs(d) + + # make two files, one empty one not. + open(os.path.join(dir1, 'empty.txt'), 'a').close() + file_with_content = os.path.join(dir2, 'not_empty.txt') + with open(file_with_content, 'w') as f: + f.write('Fee fi fo fum.') + + su.prune_empty(prune_dir, mock.Mock()) + + assert not os.path.exists(dir1) + assert os.path.exists(file_with_content) + assert not os.path.exists(dir3) + + +@pytest.mark.parametrize( + 'regex, expected', [ + (r'\d', ['bigfoot/123.txt', 'yeti/234.txt']), + (r'23', ['yeti/234.txt']), + (r'[\s\S]+nc4', ['yeti/sasquatch.nc4']), + ('ha', ['bigfoot/hahaha.nc', 'sasquatch/harry.txt']) + ] +) +@pytest.mark.util +def test_get_files(tmp_path_factory, regex, expected): + search_dir = tmp_path_factory.mktemp('get_files') + + dirs = { + 'bigfoot':['123.txt', 'hahaha.nc'], + 'yeti': ['234.txt', 'sasquatch.nc4'], + 'sasquatch': ['harry.txt', 'hendersons.nc'], + } + + for k, v in dirs.items(): + tmp_dir = os.path.join(search_dir, k) + os.makedirs(tmp_dir) + [open(os.path.join(tmp_dir, f), 'a').close() for f in v] + + actual = su.get_files(search_dir, regex) + assert actual == [os.path.join(search_dir, e) for e in expected] + \ No newline at end of file diff --git a/internal/tests/pytests/util/time_util/test_time_util.py b/internal/tests/pytests/util/time_util/test_time_util.py index cddf3470b..242469c39 100644 --- a/internal/tests/pytests/util/time_util/test_time_util.py +++ b/internal/tests/pytests/util/time_util/test_time_util.py @@ -128,20 +128,46 @@ def test_time_string_to_met_time(time_string, default_unit, met_time): @pytest.mark.parametrize( 'input_dict, expected_time_info', [ - ({'init': datetime(2014, 10, 31, 12), - 'lead': relativedelta(hours=3)}, - {'init': datetime(2014, 10, 31, 12), - 'lead': 10800, - 'valid': datetime(2014, 10, 31, 15)} - ), + # init and lead input + ({'init': datetime(2014, 10, 31, 12), 'lead': relativedelta(hours=3)}, + {'init': datetime(2014, 10, 31, 12), 'lead': 10800, 'valid': datetime(2014, 10, 31, 15)}), + # valid and lead input + ({'valid': datetime(2014, 10, 31, 12), 'lead': relativedelta(hours=3)}, + {'valid': datetime(2014, 10, 31, 12), 'lead': 10800, 'init': datetime(2014, 10, 31, 9)}), + # init/valid/lead input, loop_by init + ({'init': datetime(2014, 10, 31, 12), 'lead': relativedelta(hours=6), 'valid': datetime(2014, 10, 31, 15), 'loop_by': 'init'}, + {'init': datetime(2014, 10, 31, 12), 'lead': 21600, 'valid': datetime(2014, 10, 31, 18)}), + # init/valid/lead input, loop_by valid + ({'valid': datetime(2014, 10, 31, 12), 'lead': relativedelta(hours=6), 'init': datetime(2014, 10, 31, 9), 'loop_by': 'valid'}, + {'valid': datetime(2014, 10, 31, 12), 'lead': 21600, 'init': datetime(2014, 10, 31, 6)}), + # RUN_ONCE: init/valid/lead all wildcards + ({'init': '*', 'valid': '*', 'lead': '*'}, + {'init': '*', 'valid': '*', 'lead': '*', 'date': '*'}), + # RUN_ONCE_PER_INIT_OR_VALID: init/valid is time, wildcard lead/opposite + ({'init': datetime(2014, 10, 31, 12), 'valid': '*', 'lead': '*'}, + {'init': datetime(2014, 10, 31, 12), 'valid': '*', 'lead': '*', 'date': datetime(2014, 10, 31, 12)}), + ({'init': '*', 'valid': datetime(2014, 10, 31, 12), 'lead': '*'}, + {'init': '*', 'valid': datetime(2014, 10, 31, 12), 'lead': '*', 'date': datetime(2014, 10, 31, 12)}), + # RUN_ONCE_PER_LEAD: lead is time interval, init/valid are wildcards + ({'init': '*', 'valid': '*', 'lead': relativedelta(hours=3)}, + {'init': '*', 'valid': '*', 'lead': relativedelta(hours=3), 'date': '*'}), + # case that failed in GFDLTracker wrapper + ({'init': datetime(2021, 7, 13, 0, 0), 'lead': 21600, 'offset_hours': 0}, + {'init': datetime(2021, 7, 13, 0, 0), 'lead': 21600, 'valid': datetime(2021, 7, 13, 6, 0), 'offset': 0}), + # lead is months or years (relativedelta) + # allows lead to remain relativedelta in case init/valid change but still computes lead hours + ({'init': datetime(2021, 7, 13, 0, 0), 'lead': relativedelta(months=1)}, + {'init': datetime(2021, 7, 13, 0, 0), 'lead': relativedelta(months=1), 'valid': datetime(2021, 8, 13, 0, 0), 'lead_hours': 744}), ] ) @pytest.mark.util def test_ti_calculate(input_dict, expected_time_info): + # pass input_dict into ti_calculate and check that expected values are set time_info = time_util.ti_calculate(input_dict) for key, value in expected_time_info.items(): assert time_info[key] == value + # pass output of ti_calculate back into ti_calculate and check values time_info2 = time_util.ti_calculate(time_info) for key, value in expected_time_info.items(): assert time_info[key] == value diff --git a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py index 6fd2d6657..8822e165d 100644 --- a/internal/tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -912,3 +912,93 @@ def test_find_and_check_output_file_skip(metplus_config, exists, skip, is_dir, # cast result to bool because None isn't equal to False assert bool(result) == run + + +@pytest.mark.wrapper +def test_set_met_config_obs_window(metplus_config): + config = metplus_config + cb = CommandBuilder(config) + cb.c_dict['OBS_WINDOW_BEGIN'] = '20230808' + cb.c_dict['OBS_WINDOW_END'] = None + cb.set_met_config_obs_window(cb.c_dict) + + assert cb.env_var_dict['METPLUS_OBS_WINDOW_BEGIN'] == 'begin = 20230808;' + assert cb.env_var_dict['METPLUS_OBS_WINDOW_END'] == '' + + +@pytest.mark.parametrize( + 'time_info, user_envs, expected_env, expected_user_env', [ + ( + {'init':'20230810104356', 'now': '2023081011'}, + True, + 'Foo = 20230810104356', + 'My now = 2023081011', + ), + ( + {'init':'20230810104356', 'now': '2023081011'}, + False, + 'Foo = 20230810104356', + None, + ), + ( + None, + True, + 'Foo = {init?fmt=%Y%m%d%H%M%S}', + None, + ), + + ] +) +@pytest.mark.wrapper +def test_set_environment_variables(metplus_config, + time_info, + user_envs, + expected_env, + expected_user_env): + + config = metplus_config + + if user_envs: + config.set('user_env_vars', + 'CMD_BUILD_USER_TEST', + 'My now = {now?fmt=%Y%m%d%H}' + ) + + cb = CommandBuilder(config) + cb.env_var_dict['CMD_BUILD_TEST'] = 'Foo = {init?fmt=%Y%m%d%H%M%S}' + cb.env_var_keys.append('CMD_BUILD_TEST') + cb.set_environment_variables(time_info=time_info) + + assert cb.env['CMD_BUILD_TEST'] == expected_env + if user_envs and not time_info: + ct = cb.config.getstr('config', 'CLOCK_TIME') + ct = datetime.datetime.strptime(ct, "%Y%m%d%H%M%S").strftime("%Y%m%d%H") + assert cb.env['CMD_BUILD_USER_TEST'] == f'My now = {ct}' + elif user_envs: + assert cb.env['CMD_BUILD_USER_TEST'] == expected_user_env + else: + assert cb.config['user_env_vars'] == {} + + +@pytest.mark.parametrize( + 'shell, expected', [ + ('bash', 'export CMD_BUILD_USER_TEST="User \\"var\\"!";'), + ('csh', 'setenv CMD_BUILD_USER_TEST "User "\\""var"\\""!";'), + ] +) +@pytest.mark.wrapper +def test_get_env_copy(metplus_config, shell, expected): + config = metplus_config + config.set('user_env_vars', + 'CMD_BUILD_USER_TEST', + "User \"var\"!" + ) + + cb = CommandBuilder(config) + cb.c_dict['USER_SHELL']= shell + + cb.set_environment_variables() + actual = cb.get_env_copy({'MET_TMP_DIR', 'OMP_NUM_THREADS'}) + + assert expected in actual + \ No newline at end of file diff --git a/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py b/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py index 0eaf8d5ab..d00857ca9 100644 --- a/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py +++ b/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py @@ -7,6 +7,7 @@ from metplus.wrappers.compare_gridded_wrapper import CompareGriddedWrapper + def compare_gridded_wrapper(metplus_config): """! Returns a default GridStatWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration diff --git a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py index fa1d660f9..d92a79c8e 100644 --- a/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py +++ b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py @@ -37,7 +37,7 @@ def test_run_gen_vx_mask_once(metplus_config): # wrap.c_dict['MASK_INPUT_TEMPLATES'] = ['LAT', 'LON'] # wrap.c_dict['COMMAND_OPTIONS'] = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection"] - wrap.run_at_time_all(time_info) + wrap.run_at_time_once(time_info) expected_cmd = f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_MASK.nc -type lat -thresh 'ge30&&le50' -v 2" @@ -62,7 +62,7 @@ def test_run_gen_vx_mask_twice(metplus_config): cmd_args = ["-type lat -thresh 'ge30&&le50'", "-type lon -thresh 'le-70&&ge-130' -intersection -name lat_lon_mask"] wrap.c_dict['COMMAND_OPTIONS'] = cmd_args - wrap.run_at_time_all(time_info) + wrap.run_at_time_once(time_info) expected_cmds = [f"{wrap.app_path} 2018020100_ZENITH LAT {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc {cmd_args[0]} -v 2", f"{wrap.app_path} {wrap.config.getdir('OUTPUT_BASE')}/stage/gen_vx_mask/temp_0.nc LON {wrap.config.getdir('OUTPUT_BASE')}/GenVxMask_test/2018020100_ZENITH_LAT_LON_MASK.nc {cmd_args[1]} -v 2"] diff --git a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py index 80dfe6117..c971c44ff 100644 --- a/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py +++ b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py @@ -89,7 +89,6 @@ def set_minimum_config_settings(config): ) @pytest.mark.wrapper_b def test_grid_stat_is_prob(metplus_config, config_overrides, expected_values): - config = metplus_config set_minimum_config_settings(config) diff --git a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 2101ccfef..5c54a41ef 100644 --- a/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -21,12 +21,14 @@ f'level="{obs_level_no_quotes}"; cat_thresh=[ gt12.7 ]; }};') -def get_test_data_dir(config, subdir): - return os.path.join(config.getdir('METPLUS_BASE'), - 'internal', 'tests', 'data', subdir) +def get_test_data_dir(subdir): + internal_tests_dir = os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) + ) + return os.path.join(internal_tests_dir, 'data', subdir) -def mtd_wrapper(metplus_config, lead_seq=None): +def mtd_wrapper(metplus_config, config_overrides): """! Returns a default MTDWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration files. Subsequent tests can customize the final METplus configuration @@ -39,8 +41,8 @@ def mtd_wrapper(metplus_config, lead_seq=None): config.set('config', 'LOOP_BY', 'VALID') config.set('config', 'MTD_CONV_THRESH', '>=10') config.set('config', 'MTD_CONV_RADIUS', '15') - if lead_seq: - config.set('config', 'LEAD_SEQ', lead_seq) + for key, value in config_overrides.items(): + config.set('config', key, value) return MTDWrapper(config) @@ -208,16 +210,20 @@ def test_mode_single_field(metplus_config, config_overrides, env_var_values): @pytest.mark.wrapper def test_mtd_by_init_all_found(metplus_config): - mw = mtd_wrapper(metplus_config, '1,2,3') - obs_dir = get_test_data_dir(mw.config, 'obs') - fcst_dir = get_test_data_dir(mw.config, 'fcst') - mw.c_dict['OBS_INPUT_DIR'] = obs_dir - mw.c_dict['FCST_INPUT_DIR'] = fcst_dir - mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" - mw.c_dict['FCST_INPUT_TEMPLATE'] = "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2" - input_dict = {'init': datetime.datetime.strptime("201705100300", '%Y%m%d%H%M')} - - mw.run_at_time(input_dict) + obs_data_dir = get_test_data_dir('obs') + fcst_data_dir = get_test_data_dir('fcst') + overrides = { + 'LEAD_SEQ': '1,2,3', + 'FCST_MTD_INPUT_DIR': fcst_data_dir, + 'OBS_MTD_INPUT_DIR': obs_data_dir, + 'FCST_MTD_INPUT_TEMPLATE': "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2", + 'OBS_MTD_INPUT_TEMPLATE': "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc", + 'LOOP_BY': 'INIT', + 'INIT_TIME_FMT': '%Y%m%d%H%M', + 'INIT_BEG': '201705100300' + } + mw = mtd_wrapper(metplus_config, overrides) + mw.run_all_times() fcst_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510040000_mtd_fcst_APCP.txt') obs_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510040000_mtd_obs_APCP.txt') with open(fcst_list_file) as f: @@ -231,27 +237,31 @@ def test_mtd_by_init_all_found(metplus_config): fcst_list = fcst_list[1:] obs_list = obs_list[1:] - assert(fcst_list[0] == os.path.join(fcst_dir,'20170510', '20170510_i03_f001_HRRRTLE_PHPT.grb2') and - fcst_list[1] == os.path.join(fcst_dir,'20170510', '20170510_i03_f002_HRRRTLE_PHPT.grb2') and - fcst_list[2] == os.path.join(fcst_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') and - obs_list[0] == os.path.join(obs_dir,'20170510', 'qpe_2017051004_A06.nc') and - obs_list[1] == os.path.join(obs_dir,'20170510', 'qpe_2017051005_A06.nc') and - obs_list[2] == os.path.join(obs_dir,'20170510', 'qpe_2017051006_A06.nc') + assert(fcst_list[0] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f001_HRRRTLE_PHPT.grb2') and + fcst_list[1] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f002_HRRRTLE_PHPT.grb2') and + fcst_list[2] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') and + obs_list[0] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051004_A06.nc') and + obs_list[1] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051005_A06.nc') and + obs_list[2] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051006_A06.nc') ) @pytest.mark.wrapper def test_mtd_by_valid_all_found(metplus_config): - mw = mtd_wrapper(metplus_config, '1, 2, 3') - obs_dir = get_test_data_dir(mw.config, 'obs') - fcst_dir = get_test_data_dir(mw.config, 'fcst') - mw.c_dict['OBS_INPUT_DIR'] = obs_dir - mw.c_dict['FCST_INPUT_DIR'] = fcst_dir - mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" - mw.c_dict['FCST_INPUT_TEMPLATE'] = "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2" - input_dict = {'valid' : datetime.datetime.strptime("201705100300", '%Y%m%d%H%M') } - - mw.run_at_time(input_dict) + obs_data_dir = get_test_data_dir('obs') + fcst_data_dir = get_test_data_dir('fcst') + overrides = { + 'LEAD_SEQ': '1, 2, 3', + 'FCST_MTD_INPUT_DIR': fcst_data_dir, + 'OBS_MTD_INPUT_DIR': obs_data_dir, + 'FCST_MTD_INPUT_TEMPLATE': "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2", + 'OBS_MTD_INPUT_TEMPLATE': "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc", + 'LOOP_BY': 'VALID', + 'VALID_TIME_FMT': '%Y%m%d%H%M', + 'VALID_BEG': '201705100300' + } + mw = mtd_wrapper(metplus_config, overrides) + mw.run_all_times() fcst_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510030000_mtd_fcst_APCP.txt') obs_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510030000_mtd_obs_APCP.txt') with open(fcst_list_file) as f: @@ -265,27 +275,31 @@ def test_mtd_by_valid_all_found(metplus_config): fcst_list = fcst_list[1:] obs_list = obs_list[1:] - assert(fcst_list[0] == os.path.join(fcst_dir,'20170510', '20170510_i02_f001_HRRRTLE_PHPT.grb2') and - fcst_list[1] == os.path.join(fcst_dir,'20170510', '20170510_i01_f002_HRRRTLE_PHPT.grb2') and - fcst_list[2] == os.path.join(fcst_dir,'20170510', '20170510_i00_f003_HRRRTLE_PHPT.grb2') and - obs_list[0] == os.path.join(obs_dir,'20170510', 'qpe_2017051003_A06.nc') and - obs_list[1] == os.path.join(obs_dir,'20170510', 'qpe_2017051003_A06.nc') and - obs_list[2] == os.path.join(obs_dir,'20170510', 'qpe_2017051003_A06.nc') + assert(fcst_list[0] == os.path.join(fcst_data_dir,'20170510', '20170510_i02_f001_HRRRTLE_PHPT.grb2') and + fcst_list[1] == os.path.join(fcst_data_dir,'20170510', '20170510_i01_f002_HRRRTLE_PHPT.grb2') and + fcst_list[2] == os.path.join(fcst_data_dir,'20170510', '20170510_i00_f003_HRRRTLE_PHPT.grb2') and + obs_list[0] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051003_A06.nc') and + obs_list[1] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051003_A06.nc') and + obs_list[2] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051003_A06.nc') ) @pytest.mark.wrapper def test_mtd_by_init_miss_fcst(metplus_config): - mw = mtd_wrapper(metplus_config, '3, 6, 9, 12') - obs_dir = get_test_data_dir(mw.config, 'obs') - fcst_dir = get_test_data_dir(mw.config, 'fcst') - mw.c_dict['OBS_INPUT_DIR'] = obs_dir - mw.c_dict['FCST_INPUT_DIR'] = fcst_dir - mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" - mw.c_dict['FCST_INPUT_TEMPLATE'] = "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2" - input_dict = {'init' : datetime.datetime.strptime("201705100300", '%Y%m%d%H%M') } - - mw.run_at_time(input_dict) + obs_data_dir = get_test_data_dir('obs') + fcst_data_dir = get_test_data_dir('fcst') + overrides = { + 'LEAD_SEQ': '3, 6, 9, 12', + 'FCST_MTD_INPUT_DIR': fcst_data_dir, + 'OBS_MTD_INPUT_DIR': obs_data_dir, + 'FCST_MTD_INPUT_TEMPLATE': "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2", + 'OBS_MTD_INPUT_TEMPLATE': "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc", + 'LOOP_BY': 'INIT', + 'INIT_TIME_FMT': '%Y%m%d%H%M', + 'INIT_BEG': '201705100300' + } + mw = mtd_wrapper(metplus_config, overrides) + mw.run_all_times() fcst_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510060000_mtd_fcst_APCP.txt') obs_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510060000_mtd_obs_APCP.txt') with open(fcst_list_file) as f: @@ -299,27 +313,31 @@ def test_mtd_by_init_miss_fcst(metplus_config): fcst_list = fcst_list[1:] obs_list = obs_list[1:] - assert(fcst_list[0] == os.path.join(fcst_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') and - fcst_list[1] == os.path.join(fcst_dir,'20170510', '20170510_i03_f006_HRRRTLE_PHPT.grb2') and - fcst_list[2] == os.path.join(fcst_dir,'20170510', '20170510_i03_f012_HRRRTLE_PHPT.grb2') and - obs_list[0] == os.path.join(obs_dir,'20170510', 'qpe_2017051006_A06.nc') and - obs_list[1] == os.path.join(obs_dir,'20170510', 'qpe_2017051009_A06.nc') and - obs_list[2] == os.path.join(obs_dir,'20170510', 'qpe_2017051015_A06.nc') + assert(fcst_list[0] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') and + fcst_list[1] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f006_HRRRTLE_PHPT.grb2') and + fcst_list[2] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f012_HRRRTLE_PHPT.grb2') and + obs_list[0] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051006_A06.nc') and + obs_list[1] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051009_A06.nc') and + obs_list[2] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051015_A06.nc') ) @pytest.mark.wrapper def test_mtd_by_init_miss_both(metplus_config): - mw = mtd_wrapper(metplus_config, '6, 12, 18') - obs_dir = get_test_data_dir(mw.config, 'obs') - fcst_dir = get_test_data_dir(mw.config, 'fcst') - mw.c_dict['OBS_INPUT_DIR'] = obs_dir - mw.c_dict['FCST_INPUT_DIR'] = fcst_dir - mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" - mw.c_dict['FCST_INPUT_TEMPLATE'] = "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2" - input_dict = {'init' : datetime.datetime.strptime("201705100300", '%Y%m%d%H%M') } - - mw.run_at_time(input_dict) + obs_data_dir = get_test_data_dir('obs') + fcst_data_dir = get_test_data_dir('fcst') + overrides = { + 'LEAD_SEQ': '6, 12, 18', + 'FCST_MTD_INPUT_DIR': fcst_data_dir, + 'OBS_MTD_INPUT_DIR': obs_data_dir, + 'FCST_MTD_INPUT_TEMPLATE': "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2", + 'OBS_MTD_INPUT_TEMPLATE': "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc", + 'LOOP_BY': 'INIT', + 'INIT_TIME_FMT': '%Y%m%d%H%M', + 'INIT_BEG': '201705100300' + } + mw = mtd_wrapper(metplus_config, overrides) + mw.run_all_times() fcst_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510090000_mtd_fcst_APCP.txt') obs_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510090000_mtd_obs_APCP.txt') with open(fcst_list_file) as f: @@ -333,24 +351,28 @@ def test_mtd_by_init_miss_both(metplus_config): fcst_list = fcst_list[1:] obs_list = obs_list[1:] - assert(fcst_list[0] == os.path.join(fcst_dir,'20170510', '20170510_i03_f006_HRRRTLE_PHPT.grb2') and - fcst_list[1] == os.path.join(fcst_dir,'20170510', '20170510_i03_f012_HRRRTLE_PHPT.grb2') and - obs_list[0] == os.path.join(obs_dir,'20170510', 'qpe_2017051009_A06.nc') and - obs_list[1] == os.path.join(obs_dir,'20170510', 'qpe_2017051015_A06.nc') + assert(fcst_list[0] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f006_HRRRTLE_PHPT.grb2') and + fcst_list[1] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f012_HRRRTLE_PHPT.grb2') and + obs_list[0] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051009_A06.nc') and + obs_list[1] == os.path.join(obs_data_dir,'20170510', 'qpe_2017051015_A06.nc') ) @pytest.mark.wrapper def test_mtd_single(metplus_config): - mw = mtd_wrapper(metplus_config, '1, 2, 3') - fcst_dir = get_test_data_dir(mw.config, 'fcst') - mw.c_dict['SINGLE_RUN'] = True - mw.c_dict['SINGLE_DATA_SRC'] = 'FCST' - mw.c_dict['FCST_INPUT_DIR'] = fcst_dir - mw.c_dict['FCST_INPUT_TEMPLATE'] = "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2" - input_dict = {'init': datetime.datetime.strptime("201705100300", '%Y%m%d%H%M') } - - mw.run_at_time(input_dict) + fcst_data_dir = get_test_data_dir('fcst') + overrides = { + 'LEAD_SEQ': '1, 2, 3', + 'MTD_SINGLE_RUN': True, + 'MTD_SINGLE_DATA_SRC': 'FCST', + 'FCST_MTD_INPUT_DIR': fcst_data_dir, + 'FCST_MTD_INPUT_TEMPLATE': "{init?fmt=%Y%m%d}/{init?fmt=%Y%m%d}_i{init?fmt=%H}_f{lead?fmt=%.3H}_HRRRTLE_PHPT.grb2", + 'LOOP_BY': 'INIT', + 'INIT_TIME_FMT': '%Y%m%d%H%M', + 'INIT_BEG': '201705100300' + } + mw = mtd_wrapper(metplus_config, overrides) + mw.run_all_times() single_list_file = os.path.join(mw.config.getdir('STAGING_DIR'), 'file_lists', '20170510040000_mtd_single_APCP.txt') with open(single_list_file) as f: single_list = f.readlines() @@ -359,9 +381,9 @@ def test_mtd_single(metplus_config): # remove file_list line from lists single_list = single_list[1:] - assert(single_list[0] == os.path.join(fcst_dir,'20170510', '20170510_i03_f001_HRRRTLE_PHPT.grb2') and - single_list[1] == os.path.join(fcst_dir,'20170510', '20170510_i03_f002_HRRRTLE_PHPT.grb2') and - single_list[2] == os.path.join(fcst_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') + assert(single_list[0] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f001_HRRRTLE_PHPT.grb2') and + single_list[1] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f002_HRRRTLE_PHPT.grb2') and + single_list[2] == os.path.join(fcst_data_dir,'20170510', '20170510_i03_f003_HRRRTLE_PHPT.grb2') ) diff --git a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index 3cfe0e676..5b4ed1131 100644 --- a/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -114,7 +114,7 @@ def test_get_lowest_forecast_file_dated_subdir(metplus_config): valid_time = datetime.strptime("201802012100", '%Y%m%d%H%M') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir pcw._build_input_accum_list(data_src, {'valid': valid_time}) - out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src) + out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src, custom='') assert(out_file == input_dir+"/20180201/file.2018020118f003.nc" and fcst == 10800) @@ -128,7 +128,7 @@ def test_forecast_constant_init(metplus_config): init_time = datetime.strptime("2018020112", '%Y%m%d%H') valid_time = datetime.strptime("2018020121", '%Y%m%d%H') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir - out_file, fcst = pcw.find_input_file(init_time, valid_time, 0, data_src) + out_file, fcst = pcw.find_input_file(init_time, valid_time, 0, data_src, custom='') assert(out_file == input_dir+"/20180201/file.2018020112f009.nc" and fcst == 32400) @@ -143,7 +143,7 @@ def test_forecast_not_constant_init(metplus_config): valid_time = datetime.strptime("2018020121", '%Y%m%d%H') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir pcw._build_input_accum_list(data_src, {'valid': valid_time}) - out_file, fcst = pcw.find_input_file(init_time, valid_time, 0, data_src) + out_file, fcst = pcw.find_input_file(init_time, valid_time, 0, data_src, custom='') assert(out_file == input_dir+"/20180201/file.2018020118f003.nc" and fcst == 10800) @@ -158,7 +158,7 @@ def test_get_lowest_forecast_file_no_subdir(metplus_config): pcw.c_dict[f'{data_src}_INPUT_TEMPLATE'] = template pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir pcw._build_input_accum_list(data_src, {'valid': valid_time}) - out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src) + out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src, custom='') assert(out_file == input_dir+"/file.2018020118f003.nc" and fcst == 10800) @@ -172,7 +172,7 @@ def test_get_lowest_forecast_file_yesterday(metplus_config): pcw.c_dict[f'{data_src}_INPUT_TEMPLATE'] = template pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir pcw._build_input_accum_list(data_src, {'valid': valid_time}) - out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src) + out_file, fcst = pcw.get_lowest_fcst_file(valid_time, data_src, custom='') assert(out_file == input_dir+"/file.2018013118f012.nc" and fcst == 43200) diff --git a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py index bce689fb2..0ec4edbfb 100644 --- a/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py +++ b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py @@ -156,7 +156,9 @@ def test_run_rdp_once_per_field(metplus_config): wrap.c_dict['FCST_OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'), 'RDP_test') - wrap.run_at_time_once(time_info, var_list, data_type) + wrap.c_dict['VAR_LIST'] = var_list + wrap.c_dict['DATA_SRC'] = data_type + wrap.run_at_time_once(time_info) expected_cmds = [f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME1\"; " "level=\"A06\";' -name FNAME1 2018020100_ZENITH \"VERIF_GRID\" " @@ -205,8 +207,9 @@ def test_run_rdp_all_fields(metplus_config): wrap.c_dict['VERIFICATION_GRID'] = 'VERIF_GRID' wrap.c_dict['FCST_OUTPUT_DIR'] = os.path.join(wrap.config.getdir('OUTPUT_BASE'), 'RDP_test') - - wrap.run_at_time_once(time_info, var_list, data_type) + wrap.c_dict['VAR_LIST'] = var_list + wrap.c_dict['DATA_SRC'] = data_type + wrap.run_at_time_once(time_info) expected_cmds = [f"{wrap.app_path} -v 2 -method BUDGET -width 2 -field 'name=\"FNAME1\"; " "level=\"A06\";' -field 'name=\"FNAME2\"; level=\"A03\";' " diff --git a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index 84d9c9083..610b5573a 100644 --- a/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -319,7 +319,6 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): config.set('config', 'LOOP_BY', 'INIT') config.set('config', 'INIT_TIME_FMT', '%Y') config.set('config', 'INIT_BEG', '2016') - config.set('config', 'LOOP_ORDER', 'processes') config.set('config', 'TC_GEN_TRACK_INPUT_DIR', track_dir) config.set('config', 'TC_GEN_TRACK_INPUT_TEMPLATE', track_template) diff --git a/internal/tests/pytests/wrappers/usage/test_usage.py b/internal/tests/pytests/wrappers/usage/test_usage.py new file mode 100644 index 000000000..4489f8491 --- /dev/null +++ b/internal/tests/pytests/wrappers/usage/test_usage.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +import pytest + +from metplus.wrappers.usage_wrapper import UsageWrapper + + +@pytest.mark.wrapper +def test_usage_wrapper_run(metplus_config): + config = metplus_config + wrapper = UsageWrapper(config) + assert wrapper.isOK + + all_commands = wrapper.run_all_times() + assert not all_commands diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index 752469810..b5e7eeb8a 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -7,7 +7,6 @@ from .config_util import * from .config_metplus import * from .config_validate import * -from .doc_util import * from .run_util import * from .met_config import * from .time_looping import * diff --git a/metplus/util/run_util.py b/metplus/util/run_util.py index efe2ecd57..9bd1ab6a6 100644 --- a/metplus/util/run_util.py +++ b/metplus/util/run_util.py @@ -153,8 +153,8 @@ def _get_wrapper_instance(config, process, instance=None): metplus_wrapper = ( getattr(module, f"{process}Wrapper")(config, instance=instance) ) - except AttributeError: - config.logger.error(f"There was a problem loading {process} wrapper.") + except AttributeError as err: + config.logger.error(f"There was a problem loading {process} wrapper: {err}") return None except ModuleNotFoundError: config.logger.error(f"Could not load {process} wrapper. " diff --git a/metplus/util/system_util.py b/metplus/util/system_util.py index 8ac6f1e80..607aafc6b 100644 --- a/metplus/util/system_util.py +++ b/metplus/util/system_util.py @@ -145,7 +145,8 @@ def prune_empty(output_dir, logger): def get_files(filedir, filename_regex): """! Get all the files (with a particular naming format) by walking - through the directories. + through the directories. Note this uses re.match and will only + find matches at the beginning of the file name. @param filedir The topmost directory from which the search begins. @param filename_regex The regular expression that defines the naming @@ -174,7 +175,10 @@ def preprocess_file(filename, data_type, config, allow_dir=False): """ Decompress gzip, bzip, or zip files or convert Gempak files to NetCDF Args: @param filename: Path to file without zip extensions + @param data_type: str of data_type for filename @param config: Config object + @param allow_dir (optional): bool to allow 'filename' to be a + directory. Default is False. Returns: Path to staged unzipped file or original file if already unzipped """ @@ -284,162 +288,3 @@ def preprocess_file(filename, data_type, config, allow_dir=False): return filename return None - - -def netcdf_has_var(file_path, name, level): - """! Check if name is a variable in the NetCDF file. If not, check if - {name}_{level} (with level prefix letter removed, i.e. 06 from A06) - If the file is not a NetCDF file, OSError occurs. - If the MET_version attribute doesn't exist, AttributeError occurs. - If the netCDF4 package is not available, ImportError should occur. - All of these situations result in the file being considered not - a MET-generated NetCDF file. (CURRENTLY UNUSED) - - @param file_path full path to file to check - @returns True if file is a MET-generated NetCDF file and False if - it is not or it can't be determined. - """ - try: - from netCDF4 import Dataset - - nc_file = Dataset(file_path, 'r') - variables = nc_file.variables.keys() - - # if name is a variable, return that name - if name in variables: - return name - - # if name_level is a variable, return that - name_underscore_level = f"{name}_{split_level(level)[1]}" - if name_underscore_level in variables: - return name_underscore_level - - # requested variable name is not found in file - return None - - except (AttributeError, OSError, ImportError): - return False - - -def is_met_netcdf(file_path): - """! Check if a file is a MET-generated NetCDF file. - If the file is not a NetCDF file, OSError occurs. - If the MET_version attribute doesn't exist, AttributeError occurs. - If the netCDF4 package is not available, ImportError should occur. - All of these situations result in the file being considered not - a MET-generated NetCDF file (CURRENTLY NOT USED) - - @param file_path full path to file to check - @returns True if file is a MET-generated NetCDF file and False if - it is not or it can't be determined. - """ - try: - from netCDF4 import Dataset - nc_file = Dataset(file_path, 'r') - getattr(nc_file, 'MET_version') - except (AttributeError, OSError, ImportError): - return False - - return True - - -def get_filetype(filepath): - """!This function determines if the filepath is a NETCDF or GRIB file - based on the first eight bytes of the file. - It returns the string GRIB, NETCDF, or a None object. - - Note: If it is NOT determined to ba a NETCDF file, - it returns GRIB, regardless. - Unless there is an IOError exception, such as filepath refers - to a non-existent file or filepath is only a directory, than - None is returned, without a system exit. (CURRENTLY NOT USED) - - @param filepath: path/to/filename - @returns The string GRIB, NETCDF or a None object - """ - # Developer Note - # Since we have the impending code-freeze, keeping the behavior the same, - # just changing the implementation. - # The previous logic did not test for GRIB it would just return 'GRIB' - # if you couldn't run ncdump on the file. - # Also note: - # As John indicated ... there is the case when a grib file - # may not start with GRIB ... and if you pass the MET command filtetype=GRIB - # MET will handle it ok ... - - # Notes on file format and determining type. - # https://www.wmo.int/pages/prog/www/WDM/Guides/Guide-binary-2.html - # https://www.unidata.ucar.edu/software/netcdf/docs/faq.html - # http: // www.hdfgroup.org / HDF5 / doc / H5.format.html - - # Interpreting single byte by byte - so ok to ignore endianess - # od command: - # od -An -c -N8 foo.nc - # od -tx1 -N8 foo.nc - # GRIB - # Octet no. IS Content - # 1-4 'GRIB' (Coded CCITT-ITA No. 5) (ASCII); - # 5-7 Total length, in octets, of GRIB message(including Sections 0 & 5); - # 8 Edition number - currently 1 - # NETCDF .. ie. od -An -c -N4 foo.nc which will output - # C D F 001 - # C D F 002 - # 211 H D F - # HDF5 - # Magic numbers Hex: 89 48 44 46 0d 0a 1a 0a - # ASCII: \211 HDF \r \n \032 \n - - # Below is a reference that may be used in the future to - # determine grib version. - # import struct - # with open ("foo.grb2","rb")as binary_file: - # binary_file.seek(7) - # one_byte = binary_file.read(1) - # - # This would return an integer with value 1 or 2, - # B option is an unsigned char. - # struct.unpack('B',one_byte)[0] - - # if filepath is set to None, return None to avoid crash - if filepath == None: - return None - - try: - # read will return up to 8 bytes, if file is 0 bytes in length, - # than first_eight_bytes will be the empty string ''. - # Don't test the file length, just adds more time overhead. - with open(filepath, "rb") as binary_file: - binary_file.seek(0) - first_eight_bytes = binary_file.read(8) - - # From the first eight bytes of the file, unpack the bytes - # of the known identifier byte locations, in to a string. - # Example, if this was a netcdf file than ONLY name_cdf would - # equal 'CDF' the other variables, name_hdf would be 'DF ' - # name_grid 'CDF ' - name_cdf, name_hdf, name_grib = [None] * 3 - if len(first_eight_bytes) == 8: - name_cdf = struct.unpack('3s', first_eight_bytes[:3])[0] - name_hdf = struct.unpack('3s', first_eight_bytes[1:4])[0] - name_grib = struct.unpack('4s', first_eight_bytes[:4])[0] - - # Why not just use a else, instead of elif else if we are going to - # return GRIB ? It allows for expansion, ie. Maybe we pass in a - # logger and log the cases we can't determine the type. - if name_cdf == 'CDF' or name_hdf == 'HDF': - return "NETCDF" - elif name_grib == 'GRIB': - return "GRIB" - else: - # This mimicks previous behavoir, were we at least will always return GRIB. - # It also handles the case where GRIB was not in the first 4 bytes - # of a legitimate grib file, see John. - # logger.info('Can't determine type, returning GRIB - # as default %s'%filepath) - return "GRIB" - - except IOError: - # Skip the IOError, and keep processing data. - # ie. filepath references a file that does not exist - # or filepath is a directory. - return None diff --git a/metplus/util/time_util.py b/metplus/util/time_util.py index 10f53f47f..f1d8aa9ab 100755 --- a/metplus/util/time_util.py +++ b/metplus/util/time_util.py @@ -333,129 +333,40 @@ def _format_time_list(string_value, get_met_format, sort_list=True): return out_list -def ti_calculate(input_dict_preserve): - # copy input dictionary so valid or init can be removed to recalculate it - # without modifying the input to the function - input_dict = input_dict_preserve.copy() - out_dict = input_dict +def ti_calculate(input_dict): + """!Read in input dictionary items and compute missing items. Output from + this function can be passed back into it to re-compute items that have + changed. + Required inputs: init, valid + + @param input_dict dictionary containing time info to use in computations + @returns dictionary with updated items/values + """ + # copy input dictionary to prevent modifying input dictionary + out_dict = input_dict.copy() - # read in input dictionary items and compute missing items - # valid inputs: valid, init, lead, offset + _set_loop_by(out_dict) # look for forecast lead information in input # set forecast lead to 0 if not specified - if 'lead' in input_dict.keys(): - # if lead is relativedelta, pass it through - # if lead is not, treat it as seconds - if isinstance(input_dict['lead'], relativedelta): - out_dict['lead'] = input_dict['lead'] - elif input_dict['lead'] == '*': - out_dict['lead'] = input_dict['lead'] - else: - out_dict['lead'] = relativedelta(seconds=input_dict['lead']) - - elif 'lead_seconds' in input_dict.keys(): - out_dict['lead'] = relativedelta(seconds=input_dict['lead_seconds']) - - elif 'lead_minutes' in input_dict.keys(): - out_dict['lead'] = relativedelta(minutes=input_dict['lead_minutes']) - - elif 'lead_hours' in input_dict.keys(): - lead_hours = int(input_dict['lead_hours']) - lead_days = 0 - # if hours is more than a day, pull out days and relative hours - if lead_hours > 23: - lead_days = lead_hours // 24 - lead_hours = lead_hours % 24 - - out_dict['lead'] = relativedelta(hours=lead_hours, days=lead_days) - - else: - out_dict['lead'] = relativedelta(seconds=0) + _set_lead(out_dict) # set offset to 0 if not specified - if 'offset_hours' in input_dict.keys(): - out_dict['offset'] = datetime.timedelta(hours=input_dict['offset_hours']) - elif 'offset' in input_dict.keys(): - out_dict['offset'] = datetime.timedelta(seconds=input_dict['offset']) - else: - out_dict['offset'] = datetime.timedelta(seconds=0) - - # if init and valid are set, check which was set first via loop_by - # remove the other to recalculate - if 'init' in input_dict.keys() and 'valid' in input_dict.keys(): - if 'loop_by' in input_dict.keys(): - if input_dict['loop_by'] == 'init': - del input_dict['valid'] - elif input_dict['loop_by'] == 'valid': - del input_dict['init'] - - if 'init' in input_dict.keys(): - out_dict['init'] = input_dict['init'] - - if 'valid' in input_dict.keys(): - print("ERROR: Cannot specify both valid and init to time utility") - return None - - # compute valid from init and lead if lead is not wildcard - if out_dict['lead'] == '*': - out_dict['valid'] = '*' - else: - out_dict['valid'] = out_dict['init'] + out_dict['lead'] - - # set loop_by to init or valid to be able to see what was set first - out_dict['loop_by'] = 'init' + _set_offset(out_dict) - # if valid is provided, compute init and da_init - elif 'valid' in input_dict: - out_dict['valid'] = input_dict['valid'] + _set_init_valid_lead(out_dict) - # compute init from valid and lead if lead is not wildcard - if out_dict['lead'] == '*': - out_dict['init'] = '*' - else: - out_dict['init'] = out_dict['valid'] - out_dict['lead'] - - # set loop_by to init or valid to be able to see what was set first - out_dict['loop_by'] = 'valid' - - # if da_init is provided, compute init and valid - elif 'da_init' in input_dict.keys(): - out_dict['da_init'] = input_dict['da_init'] - - if 'valid' in input_dict.keys(): - print("ERROR: Cannot specify both valid and da_init to time utility") - return None - - # compute valid from da_init and offset - out_dict['valid'] = out_dict['da_init'] - out_dict['offset'] - - # compute init from valid and lead if lead is not wildcard - if out_dict['lead'] == '*': - out_dict['init'] = '*' - else: - out_dict['init'] = out_dict['valid'] - out_dict['lead'] - else: - print("ERROR: Need to specify valid, init, or da_init to time utility") - return None - - # calculate da_init from valid and offset - if out_dict['valid'] != '*': + # set valid_fmt and init_fmt if they are not wildcard + if out_dict.get('valid', '*') != '*': + out_dict['valid_fmt'] = out_dict['valid'].strftime('%Y%m%d%H%M%S') + # calculate da_init from valid and offset out_dict['da_init'] = out_dict['valid'] + out_dict['offset'] - - # add common formatted items out_dict['da_init_fmt'] = out_dict['da_init'].strftime('%Y%m%d%H%M%S') - out_dict['valid_fmt'] = out_dict['valid'].strftime('%Y%m%d%H%M%S') - if out_dict['init'] != '*': + if out_dict.get('init', '*') != '*': out_dict['init_fmt'] = out_dict['init'].strftime('%Y%m%d%H%M%S') - # get string representation of forecast lead - if out_dict['lead'] == '*': - out_dict['lead_string'] = 'ALL' - else: - out_dict['lead_string'] = ti_get_lead_string(out_dict['lead']) - + # convert offset to seconds and compute offset hours out_dict['offset'] = int(out_dict['offset'].total_seconds()) out_dict['offset_hours'] = int(out_dict['offset'] // 3600) @@ -466,8 +377,11 @@ def ti_calculate(input_dict_preserve): else: out_dict['date'] = out_dict['init'] - # if lead is wildcard, skip updating other lead values - if out_dict['lead'] == '*': + # if any init/valid/lead are unset or wildcard, + # skip converting lead to total seconds and computing lead hour, min, sec + if (isinstance(out_dict.get('lead', '*'), str) or + out_dict.get('valid', '*') == '*' or + out_dict.get('init', '*') == '*'): return out_dict # get difference between valid and init to get total seconds since relativedelta @@ -479,9 +393,6 @@ def ti_calculate(input_dict_preserve): if out_dict['lead'].months == 0 and out_dict['lead'].years == 0: out_dict['lead'] = total_seconds - # add common uses for relative times - # Specifying integer division // Python 3, - # assuming that was the intent in Python 2. out_dict['lead_hours'] = int(total_seconds // 3600) out_dict['lead_minutes'] = int(total_seconds // 60) out_dict['lead_seconds'] = total_seconds @@ -489,6 +400,103 @@ def ti_calculate(input_dict_preserve): return out_dict +def _set_lead(the_dict): + if 'lead' in the_dict.keys(): + # if lead is relativedelta or wildcard, pass it through + # if not, treat it as seconds + if (not isinstance(the_dict['lead'], relativedelta) and + the_dict['lead'] != '*'): + the_dict['lead'] = relativedelta(seconds=the_dict['lead']) + + elif 'lead_seconds' in the_dict.keys(): + the_dict['lead'] = relativedelta(seconds=the_dict['lead_seconds']) + + elif 'lead_minutes' in the_dict.keys(): + the_dict['lead'] = relativedelta(minutes=the_dict['lead_minutes']) + + elif 'lead_hours' in the_dict.keys(): + lead_hours = int(the_dict['lead_hours']) + lead_days = 0 + # if hours is more than a day, pull out days and relative hours + if lead_hours > 23: + lead_days = lead_hours // 24 + lead_hours = lead_hours % 24 + + the_dict['lead'] = relativedelta(hours=lead_hours, days=lead_days) + else: + # set lead to 0 if it was no specified + the_dict['lead'] = relativedelta(seconds=0) + + # get string representation of forecast lead + if the_dict['lead'] == '*': + the_dict['lead_string'] = 'ALL' + else: + the_dict['lead_string'] = ti_get_lead_string(the_dict['lead']) + + +def _set_offset(the_dict): + if 'offset_hours' in the_dict.keys(): + the_dict['offset'] = datetime.timedelta(hours=the_dict['offset_hours']) + return + + if 'offset' in the_dict.keys(): + if not isinstance(the_dict['offset'], datetime.timedelta): + the_dict['offset'] = datetime.timedelta(seconds=the_dict['offset']) + return + + the_dict['offset'] = datetime.timedelta(seconds=0) + + +def _set_loop_by(the_dict): + # loop_by is already set + if the_dict.get('loop_by'): + return + + init = the_dict.get('init') + valid = the_dict.get('valid') + # if init and valid are both set, don't set loop_by + if init and valid: + return + + # set loop_by to which init or valid is set + if init: + the_dict['loop_by'] = 'init' + elif valid: + the_dict['loop_by'] = 'valid' + + +def _set_init_valid_lead(the_dict): + wildcard_items = [item for item in ('init', 'lead', 'valid') + if the_dict.get(item) == '*'] + + # if 2 or more are wildcards, cannot compute init/valid/lead, so return + if len(wildcard_items) >= 2: + return + + # assumed that 1 or fewer items are wildcard or unset + init = the_dict.get('init') + valid = the_dict.get('valid') + lead = the_dict.get('lead') + loop_by = the_dict.get('loop_by') + + # if init and valid are both set and not wildcard, compute based on loop_by + # note: relativedelta == '*' and != '*' will always return False, so + # check if lead value is a string which implies it is '*' + if init and valid and init != '*' and valid != '*': + if loop_by == 'init': + the_dict['valid'] = init + lead + elif loop_by == 'valid': + the_dict['init'] = valid - lead + elif init and init != '*' and not isinstance(lead, str): + the_dict['valid'] = init + lead + if not loop_by: + the_dict['loop_by'] = 'init' + elif valid and valid != '*' and not isinstance(lead, str): + the_dict['init'] = valid - lead + if not loop_by: + the_dict['loop_by'] = 'valid' + + def add_to_time_input(time_input, clock_time=None, instance=None, custom=None): if clock_time: clock_dt = datetime.datetime.strptime(clock_time, '%Y%m%d%H%M%S') diff --git a/metplus/wrappers/ascii2nc_wrapper.py b/metplus/wrappers/ascii2nc_wrapper.py index 482ce6b9b..4cb65ffb7 100755 --- a/metplus/wrappers/ascii2nc_wrapper.py +++ b/metplus/wrappers/ascii2nc_wrapper.py @@ -13,7 +13,7 @@ import os from ..util import time_util -from . import CommandBuilder +from . import LoopTimesWrapper from ..util import do_string_sub, skip_time, get_lead_sequence '''!@namespace ASCII2NCWrapper @@ -22,7 +22,10 @@ ''' -class ASCII2NCWrapper(CommandBuilder): +class ASCII2NCWrapper(LoopTimesWrapper): + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_TIME_SUMMARY_DICT', @@ -234,59 +237,6 @@ def get_command(self): cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - """ - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = time_util.ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_once(time_info) - - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - Args: - @param time_info dictionary containing timing information - """ - # get input files - if self.find_input_files(time_info) is None: - return - - # get output path - if not self.find_and_check_output_file(time_info): - return - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - - self.build() - def find_input_files(self, time_info): # if using python embedding input, don't check if file exists, # just substitute time info and add to input file list diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 893562d06..1f55e99b8 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -65,6 +65,11 @@ def __init__(self, config, instance=None): self.param = "" self.all_commands = [] + # set app name to empty string if not set by wrapper + # needed to create instance of parent wrapper for unit tests + if not hasattr(self, 'app_name'): + self.app_name = '' + # store values to set in environment variables for each command self.env_var_dict = {} @@ -162,8 +167,6 @@ def create_c_dict(self): c_dict['VERBOSITY'] = self.config.getstr('config', 'LOG_MET_VERBOSITY', '2') - c_dict['ALLOW_MULTIPLE_FILES'] = False - app_name = '' if hasattr(self, 'app_name'): app_name = self.app_name @@ -212,7 +215,7 @@ def clear(self): self.env_list.clear() def set_environment_variables(self, time_info=None): - """!Set environment variables that will be read set when running this tool. + """!Set environment variables that will be read when running this tool. This tool does not have a config file, but environment variables may still need to be set, such as MET_TMP_DIR and MET_PYTHON_EXE. Reformat as needed. Print list of variables that were set and their values. diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 4d242d3ba..f45eb85da 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -16,7 +16,7 @@ from ..util import parse_var_list from ..util import get_lead_sequence, skip_time, sub_var_list from ..util import field_read_prob_info, add_field_info_to_time_info -from . import CommandBuilder +from . import LoopTimesWrapper '''!@namespace CompareGriddedWrapper @brief Common functionality to wrap similar MET applications @@ -27,16 +27,13 @@ @endcode ''' -class CompareGriddedWrapper(CommandBuilder): + +class CompareGriddedWrapper(LoopTimesWrapper): """!Common functionality to wrap similar MET applications that reformat gridded data """ def __init__(self, config, instance=None): - # set app_name if not set by child class for unit tests - if not hasattr(self, 'app_name'): - self.app_name = 'compare_gridded' - super().__init__(config, instance=instance) def create_c_dict(self): @@ -109,38 +106,6 @@ def set_environment_variables(self, time_info): super().set_environment_variables(time_info) - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function loops - over the list of forecast leads and runs the application for each. - - @param input_dict dictionary containing time information - """ - - # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead - - # set current lead time config and environment variables - time_info = ti_calculate(input_dict) - - self.logger.info("Processing forecast lead " - f"{time_info['lead_string']}") - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info("Processing custom string: " - f"{custom_string}") - - time_info['custom'] = custom_string - - # Run for given init/valid time and forecast lead combination - self.run_at_time_once(time_info) - def run_at_time_once(self, time_info): """! Build MET command for a given init/valid time and forecast lead combination. diff --git a/metplus/wrappers/cyclone_plotter_wrapper.py b/metplus/wrappers/cyclone_plotter_wrapper.py index 0e0de9009..8b7ceb53a 100644 --- a/metplus/wrappers/cyclone_plotter_wrapper.py +++ b/metplus/wrappers/cyclone_plotter_wrapper.py @@ -166,7 +166,6 @@ def __init__(self, config, instance=None): self.extent_region = [self.west_lon, self.east_lon, self.south_lat, self.north_lat] self.logger.debug(f"extent region: {self.extent_region}") - def run_all_times(self): """! Calls the defs needed to create the cyclone plots run_all_times() is required by CommandBuilder. @@ -177,7 +176,6 @@ def run_all_times(self): return None self.create_plot() - def retrieve_data(self): """! Retrieve data from track files. Returns: @@ -358,7 +356,6 @@ def retrieve_data(self): return final_sorted_df - def create_plot(self): """ Create the plot, using Cartopy @@ -514,7 +511,6 @@ def create_plot(self): # use Matplotlib's default if no resolution is set in config file plt.savefig(plot_filename) - def get_plot_points(self): """ Get the lon and lat points to be plotted, along with any other plotting-relevant @@ -552,7 +548,6 @@ def get_plot_points(self): return points_list - def get_points_by_track(self): """ Get all the lats and lons for each storm track. Used to generate the line @@ -583,7 +578,6 @@ def get_points_by_track(self): return track_dict - def subset_by_region(self, sanitized_df): """ Args: @@ -618,7 +612,6 @@ def subset_by_region(self, sanitized_df): return masked - @staticmethod def sanitize_lonlist(lon_list): """ diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 5e621792a..21e48747e 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -22,10 +22,14 @@ @endcode """ + class EnsembleStatWrapper(CompareGriddedWrapper): """!Wraps the MET tool ensemble_stat to compare ensemble datasets """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_DESC', diff --git a/metplus/wrappers/example_wrapper.py b/metplus/wrappers/example_wrapper.py index 04f8ddcd0..1802ee01f 100755 --- a/metplus/wrappers/example_wrapper.py +++ b/metplus/wrappers/example_wrapper.py @@ -14,9 +14,14 @@ from ..util import do_string_sub, ti_calculate, get_lead_sequence from ..util import skip_time -from . import CommandBuilder +from . import LoopTimesWrapper + + +class ExampleWrapper(LoopTimesWrapper): + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] -class ExampleWrapper(CommandBuilder): """!Wrapper can be used as a base to develop a new wrapper""" def __init__(self, config, instance=None): self.app_name = 'example' @@ -25,7 +30,7 @@ def __init__(self, config, instance=None): def create_c_dict(self): c_dict = super().create_c_dict() # get values from config object and set them to be accessed by wrapper - c_dict['INPUT_TEMPLATE'] = self.config.getraw('filename_templates', + c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', 'EXAMPLE_INPUT_TEMPLATE') c_dict['INPUT_DIR'] = self.config.getdir('EXAMPLE_INPUT_DIR', '') @@ -38,64 +43,27 @@ def create_c_dict(self): if not c_dict['INPUT_DIR']: self.logger.debug('EXAMPLE_INPUT_DIR was not set') + full_path = os.path.join(c_dict['INPUT_DIR'], c_dict['INPUT_TEMPLATE']) + self.logger.info(f"Input directory is {c_dict['INPUT_DIR']}") + self.logger.info(f"Input template is {c_dict['INPUT_TEMPLATE']}") + self.logger.info(f"Full input template path is {full_path}") + return c_dict - def run_at_time(self, input_dict): + def run_at_time_once(self, time_info): """! Do some processing for the current run time (init or valid) - @param input_dict dictionary with time information of current run + @param time_info dictionary with time information of current run """ - # fill in time info dictionary - time_info = ti_calculate(input_dict) - - # check if looping by valid or init and log time for run - loop_by = time_info['loop_by'] - current_time = time_info[loop_by + '_fmt'] - self.logger.info('Running ExampleWrapper at ' - f'{loop_by} time {current_time}') - # read input directory and template from config dictionary - input_dir = self.c_dict['INPUT_DIR'] - input_template = self.c_dict['INPUT_TEMPLATE'] - self.logger.info(f'Input directory is {input_dir}') - self.logger.info(f'Input template is {input_template}') - - # get forecast leads to loop over - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - - # set forecast lead time in hours - time_info['lead'] = lead - - # recalculate time info items - time_info = ti_calculate(time_info) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info( - f"Processing custom string: {custom_string}" - ) - - time_info['custom'] = custom_string - - # log init/valid/forecast lead times for current loop iteration - self.logger.info( - 'Processing forecast lead ' - f'{time_info["lead_string"]} initialized at ' - f'{time_info["init"].strftime("%Y-%m-%d %HZ")} ' - 'and valid at ' - f'{time_info["valid"].strftime("%Y-%m-%d %HZ")}' - ) - - # perform string substitution to find filename based on - # template and current run time - filename = do_string_sub(input_template, - **time_info) - self.logger.info('Looking in input directory ' - f'for file: {filename}') + full_template = os.path.join(self.c_dict['INPUT_DIR'], + self.c_dict['INPUT_TEMPLATE']) + + # perform string substitution to find filename based on + # template and current run time + filename = do_string_sub(full_template, **time_info) + self.logger.info(f'Looking for file: {filename}') + if os.path.exists(filename): + self.logger.info(f'FOUND FILE: {filename}') return True diff --git a/metplus/wrappers/extract_tiles_wrapper.py b/metplus/wrappers/extract_tiles_wrapper.py index ed11b3835..246e5af68 100755 --- a/metplus/wrappers/extract_tiles_wrapper.py +++ b/metplus/wrappers/extract_tiles_wrapper.py @@ -17,12 +17,16 @@ from ..util import get_lead_sequence, sub_var_list from ..util import parse_var_list, round_0p5, get_storms, prune_empty from .regrid_data_plane_wrapper import RegridDataPlaneWrapper -from . import CommandBuilder +from . import LoopTimesWrapper -class ExtractTilesWrapper(CommandBuilder): + +class ExtractTilesWrapper(LoopTimesWrapper): """! Takes tc-pairs data and regrids paired data to an n x m grid as specified in the config file. """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + COLUMNS_OF_INTEREST = { 'TC_STAT': [ 'INIT', @@ -163,8 +167,7 @@ def regrid_data_plane_init(self): """ rdp = 'REGRID_DATA_PLANE' - overrides = {} - overrides[f'{rdp}_METHOD'] = 'NEAREST' + overrides = {f'{rdp}_METHOD': 'NEAREST'} for data_type in ['FCST', 'OBS']: overrides[f'{data_type}_{rdp}_RUN'] = True @@ -198,41 +201,7 @@ def regrid_data_plane_init(self): rdp_wrapper.c_dict['SHOW_WARNINGS'] = False return rdp_wrapper - def run_at_time(self, input_dict): - """!Loops over loop strings and calls run_at_time_loop_string() to - process data - - @param input_dict dictionary containing initialization time - """ - - # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead - - # set current lead time config and environment variables - time_info = ti_calculate(input_dict) - - self.logger.info( - f"Processing forecast lead {time_info['lead_string']}" - ) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - # loop over custom loop list. If not defined, - # it will run once with an empty string as the custom string - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info( - f"Processing custom string: {custom_string}" - ) - - time_info['custom'] = custom_string - self.run_at_time_loop_string(time_info) - - def run_at_time_loop_string(self, time_info): + def run_at_time_once(self, time_info): """!Read TCPairs track data into TCStat to filter the data. Using the resulting track data, run RegridDataPlane on the model data to create tiles centered on the storm. @@ -384,6 +353,7 @@ def get_object_indices(object_cats): def call_regrid_data_plane(self, time_info, track_data, input_type): # set var list from config using time info var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) + self.regrid_data_plane.c_dict['VAR_LIST'] = var_list for data_type in ['FCST', 'OBS']: grid = self.get_grid(data_type, track_data[data_type], @@ -392,9 +362,8 @@ def call_regrid_data_plane(self, time_info, track_data, input_type): self.regrid_data_plane.c_dict['VERIFICATION_GRID'] = grid # run RegridDataPlane wrapper - ret = self.regrid_data_plane.run_at_time_once(time_info, - var_list, - data_type=data_type) + self.regrid_data_plane.c_dict['DATA_SRC'] = data_type + ret = self.regrid_data_plane.run_at_time_once(time_info) self.all_commands.extend(self.regrid_data_plane.all_commands) self.regrid_data_plane.all_commands.clear() if not ret: diff --git a/metplus/wrappers/gempak_to_cf_wrapper.py b/metplus/wrappers/gempak_to_cf_wrapper.py index 53a5a5cb7..f0ddd57a6 100755 --- a/metplus/wrappers/gempak_to_cf_wrapper.py +++ b/metplus/wrappers/gempak_to_cf_wrapper.py @@ -14,7 +14,7 @@ from ..util import do_string_sub, skip_time, get_lead_sequence from ..util import time_util -from . import CommandBuilder +from . import LoopTimesWrapper '''!@namespace GempakToCFWrapper @brief Wraps the GempakToCF tool to reformat Gempak format to NetCDF Format @@ -22,7 +22,11 @@ ''' -class GempakToCFWrapper(CommandBuilder): +class GempakToCFWrapper(LoopTimesWrapper): + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + def __init__(self, config, instance=None): self.app_name = "GempakToCF" self.app_path = config.getstr('exe', 'GEMPAKTOCF_JAR', '') @@ -66,32 +70,6 @@ def get_command(self): cmd += " " + self.get_output_path() return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. Processing forecast - or observation data is determined by conf variables. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - """ - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - - time_info = time_util.ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - self.run_at_time_once(time_info) - def run_at_time_once(self, time_info): """! Runs the MET application for a given time and forecast lead combination Args: diff --git a/metplus/wrappers/gen_ens_prod_wrapper.py b/metplus/wrappers/gen_ens_prod_wrapper.py index 74c87ffcc..26e4cd659 100755 --- a/metplus/wrappers/gen_ens_prod_wrapper.py +++ b/metplus/wrappers/gen_ens_prod_wrapper.py @@ -10,9 +10,13 @@ from . import LoopTimesWrapper + class GenEnsProdWrapper(LoopTimesWrapper): """! Wrapper for gen_ens_prod MET application """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_DESC', diff --git a/metplus/wrappers/gen_vx_mask_wrapper.py b/metplus/wrappers/gen_vx_mask_wrapper.py index b6aa36abe..6f9018598 100755 --- a/metplus/wrappers/gen_vx_mask_wrapper.py +++ b/metplus/wrappers/gen_vx_mask_wrapper.py @@ -13,7 +13,7 @@ import os from ..util import getlist, get_lead_sequence, skip_time, ti_calculate, mkdir_p -from . import CommandBuilder +from . import LoopTimesWrapper from ..util import do_string_sub '''!@namespace GenVxMaskWrapper @@ -22,7 +22,10 @@ ''' -class GenVxMaskWrapper(CommandBuilder): +class GenVxMaskWrapper(LoopTimesWrapper): + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] def __init__(self, config, instance=None): self.app_name = "gen_vx_mask" @@ -40,12 +43,12 @@ def create_c_dict(self): # input and output files c_dict['INPUT_DIR'] = self.config.getdir('GEN_VX_MASK_INPUT_DIR', '') - c_dict['INPUT_TEMPLATE'] = self.config.getraw('filename_templates', + c_dict['INPUT_TEMPLATE'] = self.config.getraw('config', 'GEN_VX_MASK_INPUT_TEMPLATE') c_dict['OUTPUT_DIR'] = self.config.getdir('GEN_VX_MASK_OUTPUT_DIR', '') - c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('filename_templates', + c_dict['OUTPUT_TEMPLATE'] = self.config.getraw('config', 'GEN_VX_MASK_OUTPUT_TEMPLATE') c_dict['MASK_INPUT_DIR'] = self.config.getdir('GEN_VX_MASK_INPUT_MASK_DIR', @@ -122,34 +125,7 @@ def get_command(self): cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - @returns None - """ - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_all(time_info) - - def run_at_time_all(self, time_info): + def run_at_time_once(self, time_info): """!Loop over list of mask templates and call GenVxMask for each, adding the corresponding command line arguments for each call Args: diff --git a/metplus/wrappers/gfdl_tracker_wrapper.py b/metplus/wrappers/gfdl_tracker_wrapper.py index 770093f4f..1e686109d 100755 --- a/metplus/wrappers/gfdl_tracker_wrapper.py +++ b/metplus/wrappers/gfdl_tracker_wrapper.py @@ -17,11 +17,15 @@ from ..util import do_string_sub, ti_calculate, get_lead_sequence from ..util import remove_quotes, parse_template -from . import CommandBuilder +from . import RuntimeFreqWrapper -class GFDLTrackerWrapper(CommandBuilder): + +class GFDLTrackerWrapper(RuntimeFreqWrapper): """!Configures and runs GFDL Tracker""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE', 'RUN_ONCE_PER_INIT_OR_VALID'] + CONFIG_NAMES = { "DATEIN_INP_MODEL": "int", "DATEIN_INP_MODTYP": "string", @@ -241,20 +245,6 @@ def _read_gfdl_config_variables(self, c_dict): value = get_fct('config', f'GFDL_TRACKER_{name}', '') c_dict[f'REPLACE_CONF_{name}'] = value - def run_at_time(self, input_dict): - """! Do some processing for the current run time (init or valid) - - @param input_dict dictionary containing time information of current run - """ - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - self.run_at_time_once(input_dict) - - self.c_dict['FIRST_RUN'] = False - def run_at_time_once(self, input_dict): """! Do some processing for the current run time (init or valid) @@ -547,8 +537,9 @@ def handle_templates(self, input_dict): # only fill out sgv template file if template is specified # and on a 0Z run that is not the first run time - if (not self.c_dict['SGV_TEMPLATE_FILE'] or - self.c_dict['FIRST_RUN'] or + first_run = self.c_dict['FIRST_RUN'] + self.c_dict['FIRST_RUN'] = False + if (not self.c_dict['SGV_TEMPLATE_FILE'] or first_run or input_dict['init'].strftime('%H') != '00'): return output_path @@ -578,7 +569,6 @@ def sub_template(self, template_file, output_path, sub_dict): for line in output_lines: file_handle.write(f'{line}\n') - def populate_sub_dict(self, time_info): sub_dict = {} diff --git a/metplus/wrappers/grid_diag_wrapper.py b/metplus/wrappers/grid_diag_wrapper.py index eb1c5e98b..9d1f93d02 100755 --- a/metplus/wrappers/grid_diag_wrapper.py +++ b/metplus/wrappers/grid_diag_wrapper.py @@ -24,6 +24,9 @@ class GridDiagWrapper(RuntimeFreqWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = 'ALL' + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_DESC', 'METPLUS_REGRID_DICT', @@ -150,8 +153,6 @@ def get_command(self): return cmd def run_at_time_once(self, time_info): - self.clear() - # subset input files as appropriate input_list_dict = self.subset_input_files(time_info) if not input_list_dict: @@ -231,7 +232,7 @@ def get_files_from_time(self, time_info): files with a key representing a description of that file """ file_dict = super().get_files_from_time(time_info) - input_files = self.find_input_files(time_info) + input_files = self.get_input_files(time_info) if input_files is None: return None diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index 666a148e6..f9b9ca2fb 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -24,6 +24,9 @@ class GridStatWrapper(CompareGriddedWrapper): """!Wraps the MET tool grid_stat to compare gridded datasets""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_DESC', diff --git a/metplus/wrappers/ioda2nc_wrapper.py b/metplus/wrappers/ioda2nc_wrapper.py index dfc75b4c7..323fbbe0c 100755 --- a/metplus/wrappers/ioda2nc_wrapper.py +++ b/metplus/wrappers/ioda2nc_wrapper.py @@ -17,6 +17,9 @@ class IODA2NCWrapper(LoopTimesWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = 'ALL' + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MESSAGE_TYPE', 'METPLUS_MESSAGE_TYPE_GROUP_MAP', @@ -109,30 +112,6 @@ def get_command(self): f" {self.infiles[0]} {self.get_output_path()}" f" {' '.join(self.args)}") - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ioda2nc - - @param time_info dictionary containing timing information - @returns True if command was built/run successfully or - False if something went wrong - """ - # get input files - if not self.find_input_files(time_info): - return False - - # get output path - if not self.find_and_check_output_file(time_info): - return False - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - return self.build() - def find_input_files(self, time_info): """! Get all input files for ioda2nc. Sets self.infiles list. diff --git a/metplus/wrappers/loop_times_wrapper.py b/metplus/wrappers/loop_times_wrapper.py index 9b7b4fae9..09570364c 100755 --- a/metplus/wrappers/loop_times_wrapper.py +++ b/metplus/wrappers/loop_times_wrapper.py @@ -16,10 +16,6 @@ class LoopTimesWrapper(RuntimeFreqWrapper): def __init__(self, config, instance=None): - # set app_name if not set by child class to allow tests to run - if not hasattr(self, 'app_name'): - self.app_name = 'loop_times' - super().__init__(config, instance=instance) def create_c_dict(self): diff --git a/metplus/wrappers/met_db_load_wrapper.py b/metplus/wrappers/met_db_load_wrapper.py index 421d440ce..a837647f2 100755 --- a/metplus/wrappers/met_db_load_wrapper.py +++ b/metplus/wrappers/met_db_load_wrapper.py @@ -22,11 +22,16 @@ @endcode ''' + class METDbLoadWrapper(RuntimeFreqWrapper): """! Config variable names - All names are prepended with MET_DB_LOAD_MV_ and all c_dict values are prepended with MV_. The name is the key and string specifying the type is the value. """ + + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE' + RUNTIME_FREQ_SUPPORTED = 'ALL' + CONFIG_NAMES = { 'HOST': 'string', 'DATABASE': 'string', diff --git a/metplus/wrappers/mode_wrapper.py b/metplus/wrappers/mode_wrapper.py index 57518403d..ece359d64 100755 --- a/metplus/wrappers/mode_wrapper.py +++ b/metplus/wrappers/mode_wrapper.py @@ -15,9 +15,13 @@ from . import CompareGriddedWrapper from ..util import do_string_sub + class MODEWrapper(CompareGriddedWrapper): """!Wrapper for the mode MET tool""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_DESC', diff --git a/metplus/wrappers/mtd_wrapper.py b/metplus/wrappers/mtd_wrapper.py index 432f7f375..b3f3ef630 100755 --- a/metplus/wrappers/mtd_wrapper.py +++ b/metplus/wrappers/mtd_wrapper.py @@ -13,13 +13,17 @@ import os from ..util import get_lead_sequence, sub_var_list -from ..util import ti_calculate +from ..util import ti_calculate, getlist from ..util import do_string_sub, skip_time from ..util import parse_var_list, add_field_info_to_time_info from . import CompareGriddedWrapper + class MTDWrapper(CompareGriddedWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = 'ALL' + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_DESC', @@ -53,63 +57,37 @@ def create_c_dict(self): # set to prevent find_obs from getting multiple files within # a time window. Does not refer to time series of files c_dict['ALLOW_MULTIPLE_FILES'] = False + c_dict['ONCE_PER_FIELD'] = True - c_dict['OUTPUT_DIR'] = self.config.getdir('MTD_OUTPUT_DIR', - self.config.getdir('OUTPUT_BASE')) + c_dict['OUTPUT_DIR'] = ( + self.config.getdir('MTD_OUTPUT_DIR', + self.config.getdir('OUTPUT_BASE')) + ) c_dict['OUTPUT_TEMPLATE'] = ( - self.config.getraw('config', - 'MTD_OUTPUT_TEMPLATE') + self.config.getraw('config', 'MTD_OUTPUT_TEMPLATE') ) # get the MET config file path or use default c_dict['CONFIG_FILE'] = self.get_config_file('MTDConfig_wrapped') # new method of reading/setting MET config values - self.add_met_config(name='min_volume', - data_type='int') + self.add_met_config(name='min_volume', data_type='int') # old approach to reading/setting MET config values - c_dict['MIN_VOLUME'] = self.config.getstr('config', - 'MTD_MIN_VOLUME', '2000') + c_dict['MIN_VOLUME'] = self.config.getstr('config', 'MTD_MIN_VOLUME', '2000') - c_dict['SINGLE_RUN'] = self.config.getbool('config', - 'MTD_SINGLE_RUN', - False) + c_dict['SINGLE_RUN'] = ( + self.config.getbool('config', 'MTD_SINGLE_RUN', False) + ) if c_dict['SINGLE_RUN']: c_dict['SINGLE_DATA_SRC'] = ( - self.config.getstr('config', - 'MTD_SINGLE_DATA_SRC', - '') + self.config.getstr('config', 'MTD_SINGLE_DATA_SRC', '') ) if not c_dict['SINGLE_DATA_SRC']: self.log_error('Must set MTD_SINGLE_DATA_SRC if ' 'MTD_SINGLE_RUN is True') - c_dict['FCST_INPUT_DIR'] = ( - self.config.getdir('FCST_MTD_INPUT_DIR', '') - ) - c_dict['FCST_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'FCST_MTD_INPUT_TEMPLATE') - ) - c_dict['OBS_INPUT_DIR'] = ( - self.config.getdir('OBS_MTD_INPUT_DIR', '') - ) - c_dict['OBS_INPUT_TEMPLATE'] = ( - self.config.getraw('filename_templates', - 'OBS_MTD_INPUT_TEMPLATE') - ) - - c_dict['FCST_FILE_LIST'] = ( - self.config.getraw('config', - 'FCST_MTD_INPUT_FILE_LIST') - ) - c_dict['OBS_FILE_LIST'] = ( - self.config.getraw('config', - 'OBS_MTD_INPUT_FILE_LIST') - ) - if c_dict['FCST_FILE_LIST'] or c_dict['OBS_FILE_LIST']: - c_dict['EXPLICIT_FILE_LIST'] = True + self.get_input_templates(c_dict) # if single run for OBS, read OBS values into FCST keys read_type = 'FCST' @@ -127,6 +105,9 @@ def create_c_dict(self): data_type=c_dict.get('SINGLE_DATA_SRC'), met_tool=self.app_name) ) + if not c_dict['VAR_LIST_TEMP']: + self.log_error('No input fields were specified.' + 'Must set [FCST/OBS]_VAR_[NAME/LEVELS].') return c_dict @@ -172,207 +153,60 @@ def read_field_values(self, c_dict, read_type, write_type): if c_dict['SINGLE_RUN']: c_dict['OBS_CONV_THRESH'] = conf_value - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function loops - over the list of user-defined strings and runs the application - for each. Overrides run_at_time in compare_gridded_wrapper.py - Args: - @param input_dict dictionary containing timing information - """ - - if skip_time(input_dict, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - return - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - self.run_at_time_loop_string(input_dict) - - def run_at_time_loop_string(self, input_dict): - """! Runs the MET application for a given run time. This function loops - over the list of forecast leads and runs the application for each. - Overrides run_at_time in compare_gridded_wrapper.py - Args: - @param input_dict dictionary containing timing information - """ - var_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], input_dict) - - # if only processing a single data set (FCST or OBS) then only read - # that var list and process - if self.c_dict['SINGLE_RUN']: - for var_info in var_list: - self.run_single_mode(input_dict, var_info) - - return - - # if comparing FCST and OBS data, get var list from - # FCST/OBS or BOTH variables - # report error and exit if field info is not set - if not var_list: - self.log_error('No input fields were specified to MTD. You must ' - 'set [FCST/OBS]_VAR_[NAME/LEVELS].') - return None - - for var_info in var_list: - - if self.c_dict.get('EXPLICIT_FILE_LIST', False): - time_info = ti_calculate(input_dict) - add_field_info_to_time_info(time_info, var_info) - model_list_path = do_string_sub(self.c_dict['FCST_FILE_LIST'], - **time_info) - self.logger.debug(f"Explicit FCST file: {model_list_path}") - if not os.path.exists(model_list_path): - self.log_error('FCST file list file does not exist: ' - f'{model_list_path}') - return None - - obs_list_path = do_string_sub(self.c_dict['OBS_FILE_LIST'], - **time_info) - self.logger.debug(f"Explicit OBS file: {obs_list_path}") - if not os.path.exists(obs_list_path): - self.log_error('OBS file list file does not exist: ' - f'{obs_list_path}') - return None - - arg_dict = {'obs_path': obs_list_path, - 'model_path': model_list_path} - - self.process_fields_one_thresh(time_info, var_info, **arg_dict) - continue - - model_list = [] - obs_list = [] - - # find files for each forecast lead time - lead_seq = get_lead_sequence(self.config, input_dict) - - tasks = [] - for lead in lead_seq: - input_dict['lead'] = lead - - time_info = ti_calculate(input_dict) - add_field_info_to_time_info(time_info, var_info) - tasks.append(time_info) - - for current_task in tasks: - # call find_model/obs as needed - model_file = self.find_model(current_task, mandatory=False) - obs_file = self.find_obs(current_task, mandatory=False) - if model_file is None and obs_file is None: + def run_at_time_once(self, time_info): + # calculate valid based on first forecast lead + lead_seq = get_lead_sequence(self.config, time_info) + if not lead_seq: + lead_seq = [0] + first_lead = lead_seq[0] + time_info['lead'] = first_lead + first_valid_time_info = ti_calculate(time_info) + + # get formatted time to use to name file list files + time_fmt = f"{first_valid_time_info['valid_fmt']}" + + # loop through the files found for each field (var_info) + for file_dict in self.c_dict['ALL_FILES']: + var_info = file_dict['var_info'] + inputs = {} + for data_type in ('FCST', 'OBS'): + file_list = file_dict.get(data_type) + if not file_list: continue - - if model_file is None: + if len(file_list) == 1: + if not os.path.exists(file_list[0]): + self.log_error(f'{data_type} file does not exist: ' + f'{file_list[0]}') + continue + inputs[data_type] = file_list[0] continue - if obs_file is None: + file_ext = self.check_for_python_embedding(data_type, var_info) + if not file_ext: continue - self.logger.debug(f"Adding forecast file: {model_file}") - self.logger.debug(f"Adding observation file: {obs_file}") - model_list.append(model_file) - obs_list.append(obs_file) - - # only check model list because obs list should have same size - if not model_list: - self.log_error('Could not find any files to process') - return - - # write ascii file with list of files to process - input_dict['lead'] = lead_seq[0] - time_info = ti_calculate(input_dict) - - # if var name is a python embedding script, check type of python - # input and name file list file accordingly - fcst_file_ext = self.check_for_python_embedding('FCST', var_info) - obs_file_ext = self.check_for_python_embedding('OBS', var_info) - # if check_for_python_embedding returns None, an error occurred - if not fcst_file_ext or not obs_file_ext: - return - - model_outfile = ( - f"{time_info['valid_fmt']}_mtd_fcst_{fcst_file_ext}.txt" - ) - obs_outfile = ( - f"{time_info['valid_fmt']}_mtd_obs_{obs_file_ext}.txt" - ) - model_list_path = self.write_list_file(model_outfile, model_list) - obs_list_path = self.write_list_file(obs_outfile, obs_list) - - arg_dict = {'obs_path': obs_list_path, - 'model_path': model_list_path} + dt = 'single' if self.c_dict['SINGLE_RUN'] else data_type + outfile = f"{time_fmt}_mtd_{dt.lower()}_{file_ext}.txt" + inputs[data_type] = self.write_list_file(outfile, file_list) - self.process_fields_one_thresh(time_info, var_info, **arg_dict) - - - def run_single_mode(self, input_dict, var_info): - single_list = [] - - data_src = self.c_dict.get('SINGLE_DATA_SRC') - - if self.c_dict.get('EXPLICIT_FILE_LIST', False): - time_info = ti_calculate(input_dict) - add_field_info_to_time_info(time_info, var_info) - single_list_path = do_string_sub( - self.c_dict[f'{data_src}_FILE_LIST'], - **time_info - ) - self.logger.debug(f"Explicit file list: {single_list_path}") - if not os.path.exists(single_list_path): - self.log_error(f'{data_src} file list file does not exist: ' - f'{single_list_path}') - return None - - else: - if data_src == 'OBS': - find_method = self.find_obs - else: - find_method = self.find_model - - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead - current_task = ti_calculate(input_dict) - - single_file = find_method(current_task) - if single_file is None: - continue - - single_list.append(single_file) - - if len(single_list) == 0: - return - - # write ascii file with list of files to process - input_dict['lead'] = lead_seq[0] - time_info = ti_calculate(input_dict) - file_ext = self.check_for_python_embedding(data_src, var_info) - if not file_ext: - return - - single_outfile = ( - f"{time_info['valid_fmt']}_mtd_single_{file_ext}.txt" - ) - single_list_path = self.write_list_file(single_outfile, - single_list) - - arg_dict = {} - if data_src == 'OBS': - arg_dict['obs_path'] = single_list_path - arg_dict['model_path'] = None - else: - arg_dict['model_path'] = single_list_path - arg_dict['obs_path'] = None - - self.process_fields_one_thresh(time_info, var_info, **arg_dict) - - def process_fields_one_thresh(self, time_info, var_info, model_path, - obs_path): + if not inputs: + self.log_error('Input files not found') + continue + if len(inputs) < 2 and not self.c_dict['SINGLE_RUN']: + self.log_error('Could not find all required inputs files') + continue + arg_dict = { + 'obs_path': inputs.get('OBS'), + 'model_path': inputs.get('FCST'), + } + self.process_fields_one_thresh(first_valid_time_info, var_info, + **arg_dict) + + def process_fields_one_thresh(self, first_valid_time_info, var_info, + model_path, obs_path): """! For each threshold, set up environment variables and run mode Args: - @param time_info dictionary containing timing information + @param first_valid_time_info dictionary containing timing information @param var_info object containing variable information @param model_path forecast file list path @param obs_path observation file list path @@ -394,7 +228,7 @@ def process_fields_one_thresh(self, time_info, var_info, model_path, if not fcst_thresh_list: fcst_thresh_list = [""] - # loop over thresholds and build field list with one thresh per item + # loop over thresholds and build field list with one thresh per item for fcst_thresh in fcst_thresh_list: fcst_field = ( self.get_field_info(v_name=var_info['fcst_name'], @@ -446,23 +280,18 @@ def process_fields_one_thresh(self, time_info, var_info, model_path, # the lists are the same length obs_field_list = fcst_field_list - # loop through fields and call MTD for fcst_field, obs_field in zip(fcst_field_list, obs_field_list): - self.format_field('FCST', - fcst_field, - is_list=False) - self.format_field('OBS', - obs_field, - is_list=False) + self.format_field('FCST', fcst_field, is_list=False) + self.format_field('OBS', obs_field, is_list=False) self.param = do_string_sub(self.c_dict['CONFIG_FILE'], - **time_info) + **first_valid_time_info) self.set_current_field_config(var_info) - self.set_environment_variables(time_info) + self.set_environment_variables(first_valid_time_info) - if not self.find_and_check_output_file(time_info, + if not self.find_and_check_output_file(first_valid_time_info, is_directory=True): return @@ -510,7 +339,7 @@ def get_command(self): @rtype string @return Returns a MET command with arguments that you can run """ - cmd = '{} -v {} '.format(self.app_path, self.c_dict['VERBOSITY']) + cmd = f"{self.app_path} -v {self.c_dict['VERBOSITY']} " for a in self.args: cmd += a + " " @@ -527,3 +356,88 @@ def get_command(self): cmd += '-outdir {}'.format(self.outdir) return cmd + + def get_input_templates(self, c_dict): + input_types = ['FCST', 'OBS'] + if c_dict.get('SINGLE_RUN', False): + input_types = [c_dict['SINGLE_DATA_SRC']] + + app = self.app_name.upper() + template_dict = {} + for in_type in input_types: + template_path = ( + self.config.getraw('config', + f'{in_type}_{app}_INPUT_FILE_LIST') + ) + if template_path: + c_dict['EXPLICIT_FILE_LIST'] = True + else: + in_dir = self.config.getdir(f'{in_type}_{app}_INPUT_DIR', '') + templates = getlist( + self.config.getraw('config', + f'{in_type}_{app}_INPUT_TEMPLATE') + ) + template_list = [os.path.join(in_dir, template) + for template in templates] + template_path = ','.join(template_list) + + template_dict[in_type] = template_path + + c_dict['TEMPLATE_DICT'] = template_dict + + def get_files_from_time(self, time_info): + """! Create dictionary containing time information (key time_info) and + any relevant files for that runtime. The parent implementation of + this function creates a dictionary and adds the time_info to it. + This wrapper gets all files for the current runtime and adds it to + the dictionary with keys 'FCST' and 'OBS' + + @param time_info dictionary containing time information + @returns dictionary containing time_info dict and any relevant + files with a key representing a description of that file + """ + if self.c_dict.get('ONCE_PER_FIELD', False): + var_list = sub_var_list(self.c_dict.get('VAR_LIST_TEMP'), time_info) + else: + var_list = [None] + + # create a dictionary for each field (var) with time_info and files + file_dict_list = [] + for var_info in var_list: + file_dict = {'var_info': var_info} + if var_info: + add_field_info_to_time_info(time_info, var_info) + + input_files = self.get_input_files(time_info, fill_missing=True) + # only add all input files if none are missing + no_missing = True + if input_files: + for key, value in input_files.items(): + if 'missing' in value: + no_missing = False + file_dict[key] = value + if no_missing: + file_dict_list.append(file_dict) + + return file_dict_list + + def _update_list_with_new_files(self, time_info, list_to_update): + new_files = self.get_files_from_time(time_info) + if not new_files: + return + + # if list to update is empty, copy new items into list + if not list_to_update: + for new_file in new_files: + list_to_update.append(new_file.copy()) + return + + # if list to update is not empty, add new files to each file list, + # make sure new files correspond to the correct field (var) + assert len(list_to_update) == len(new_files) + for new_file, existing_item in zip(new_files, list_to_update): + assert new_file['var_info'] == existing_item['var_info'] + for key, value in new_file.items(): + if key == 'var_info': + continue + existing_item[key].extend(value) diff --git a/metplus/wrappers/pb2nc_wrapper.py b/metplus/wrappers/pb2nc_wrapper.py index 6d3848ca4..44bb7fd97 100755 --- a/metplus/wrappers/pb2nc_wrapper.py +++ b/metplus/wrappers/pb2nc_wrapper.py @@ -16,12 +16,15 @@ from ..util import getlistint, skip_time, get_lead_sequence from ..util import ti_calculate from ..util import do_string_sub -from . import CommandBuilder +from . import LoopTimesWrapper -class PB2NCWrapper(CommandBuilder): + +class PB2NCWrapper(LoopTimesWrapper): """! Wrapper to the MET tool pb2nc which converts prepbufr files to NetCDF for MET's point_stat tool can recognize. """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = 'ALL' WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MESSAGE_TYPE', @@ -252,30 +255,6 @@ def set_valid_window_variables(self, time_info): do_string_sub(end_template, **time_info) - - def run_at_time(self, input_dict): - """! Loop over each forecast lead and build pb2nc command """ - # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - input_dict['lead'] = lead - - lead_string = ti_calculate(input_dict)['lead_string'] - self.logger.info("Processing forecast lead {}".format(lead_string)) - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info( - f"Processing custom string: {custom_string}" - ) - - input_dict['custom'] = custom_string - - # Run for given init/valid time and forecast lead combination - self.clear() - self.run_at_time_once(input_dict) - - def run_at_time_once(self, input_dict): """!Find files needed to run pb2nc and run if found""" # look for input files to process @@ -285,10 +264,6 @@ def run_at_time_once(self, input_dict): if time_info is None: return - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - return - # look for output file path and skip running pb2nc if necessary if not self.find_and_check_output_file(time_info): return @@ -302,11 +277,7 @@ def run_at_time_once(self, input_dict): self.c_dict['CONFIG_FILE'] = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) - # build command and run if successful - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return + # build and run command self.build() def get_command(self): diff --git a/metplus/wrappers/pcp_combine_wrapper.py b/metplus/wrappers/pcp_combine_wrapper.py index a02cacb7d..77b5b0b82 100755 --- a/metplus/wrappers/pcp_combine_wrapper.py +++ b/metplus/wrappers/pcp_combine_wrapper.py @@ -12,17 +12,22 @@ from ..util import get_relativedelta, ti_get_seconds_from_relativedelta from ..util import time_string_to_met_time, seconds_to_met_time from ..util import parse_var_list, template_to_regex, split_level -from ..util import add_field_info_to_time_info +from ..util import add_field_info_to_time_info, sub_var_list from . import ReformatGriddedWrapper '''!@namespace PCPCombineWrapper @brief Wraps the MET tool pcp_combine to combine/divide precipitation accumulations or derive additional fields ''' + + class PCPCombineWrapper(ReformatGriddedWrapper): """! Wraps the MET tool pcp_combine to combine or divide precipitation accumulations """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + # valid values for [FCST/OBS]_PCP_COMBINE_METHOD valid_run_methods = ['ADD', 'SUM', 'SUBTRACT', 'DERIVE', 'USER_DEFINED'] @@ -226,7 +231,9 @@ def set_fcst_or_obs_dict_items(self, d_type, c_dict): return c_dict - def run_at_time_once(self, time_info, var_list, data_src): + def run_at_time_once(self, time_info): + var_list = sub_var_list(self.c_dict['VAR_LIST'], time_info) + data_src = self.c_dict['DATA_SRC'] if not var_list: var_list = [None] @@ -635,6 +642,7 @@ def get_accumulation(self, time_info, accum, data_src, @return True if full set of files to build accumulation is found """ search_time = time_info['valid'] + custom = time_info.get('custom', '') # last time to search is the output accumulation subtracted from the # valid time, then add back the smallest accumulation that is available # in the input. This is done because data contains an accumulation from @@ -687,7 +695,8 @@ def get_accumulation(self, time_info, accum, data_src, search_file, lead = self.find_input_file(time_info['init'], search_time, accum_dict['amount'], - data_src) + data_src, + custom) if not search_file: continue @@ -699,7 +708,8 @@ def get_accumulation(self, time_info, accum, data_src, accum_amount = self.get_template_accum(accum_dict, search_time, lead, - data_src) + data_src, + custom) if accum_amount > total_accum: self.logger.debug("Accumulation amount is bigger " "than remaining accumulation.") @@ -746,11 +756,12 @@ def get_accumulation(self, time_info, accum, data_src, return files_found - def get_lowest_fcst_file(self, valid_time, data_src): + def get_lowest_fcst_file(self, valid_time, data_src, custom): """! Find the lowest forecast hour that corresponds to the valid time @param valid_time valid time to search @param data_src data type (FCST or OBS) to get filename template + @param custom string from custom loop list to use in template sub @rtype string @return Path to file with the lowest forecast hour """ @@ -786,7 +797,7 @@ def get_lowest_fcst_file(self, valid_time, data_src): 'lead_seconds': forecast_lead } time_info = ti_calculate(input_dict) - time_info['custom'] = self.c_dict.get('CUSTOM_STRING', '') + time_info['custom'] = custom search_file = os.path.join(self.c_dict[f'{data_src}_INPUT_DIR'], self.c_dict[data_src+'_INPUT_TEMPLATE']) search_file = do_string_sub(search_file, **time_info) @@ -821,7 +832,8 @@ def get_field_string(self, time_info=None, search_accum=0, name=None, field_info = do_string_sub(field_info, **time_info) return field_info - def find_input_file(self, init_time, valid_time, search_accum, data_src): + def find_input_file(self, init_time, valid_time, search_accum, data_src, + custom): lead = 0 in_template = self.c_dict[data_src+'_INPUT_TEMPLATE'] @@ -829,7 +841,7 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src): if ('{lead?' in in_template or ('{init?' in in_template and '{valid?' in in_template)): if not self.c_dict[f'{data_src}_CONSTANT_INIT']: - return self.get_lowest_fcst_file(valid_time, data_src) + return self.get_lowest_fcst_file(valid_time, data_src, custom) # set init time and lead in time dict if init should be constant # ti_calculate cannot currently handle both init and valid @@ -842,7 +854,7 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src): input_dict = {'valid': valid_time} time_info = ti_calculate(input_dict) - time_info['custom'] = self.c_dict.get('CUSTOM_STRING', '') + time_info['custom'] = custom time_info['level'] = int(search_accum) input_path = os.path.join(self.c_dict[f'{data_src}_INPUT_DIR'], in_template) @@ -852,11 +864,12 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src): self.c_dict[f'{data_src}_INPUT_DATATYPE'], self.config), lead - def get_template_accum(self, accum_dict, search_time, lead, data_src): + def get_template_accum(self, accum_dict, search_time, lead, data_src, + custom): # apply string substitution to accum amount search_time_dict = {'valid': search_time, 'lead_seconds': lead} search_time_info = ti_calculate(search_time_dict) - search_time_info['custom'] = self.c_dict.get('CUSTOM_STRING', '') + search_time_info['custom'] = custom amount = do_string_sub(accum_dict['template'], **search_time_info) amount = get_seconds_from_string(amount, default_unit='S', diff --git a/metplus/wrappers/plot_data_plane_wrapper.py b/metplus/wrappers/plot_data_plane_wrapper.py index 26f844772..b0efc8798 100755 --- a/metplus/wrappers/plot_data_plane_wrapper.py +++ b/metplus/wrappers/plot_data_plane_wrapper.py @@ -13,8 +13,8 @@ import os from ..util import time_util -from . import CommandBuilder from ..util import do_string_sub, remove_quotes, skip_time, get_lead_sequence +from . import LoopTimesWrapper '''!@namespace PlotDataPlaneWrapper @brief Wraps the PlotDataPlane tool to plot data @@ -22,7 +22,10 @@ ''' -class PlotDataPlaneWrapper(CommandBuilder): +class PlotDataPlaneWrapper(LoopTimesWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + def __init__(self, config, instance=None): self.app_name = "plot_data_plane" self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -107,33 +110,6 @@ def get_command(self): cmd += f" -v {self.c_dict['VERBOSITY']}" return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - """ - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = time_util.ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info("Processing custom string: " - f"{custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_once(time_info) - def run_at_time_once(self, time_info): """! Process runtime and try to build command to run ascii2nc Args: diff --git a/metplus/wrappers/plot_point_obs_wrapper.py b/metplus/wrappers/plot_point_obs_wrapper.py index 071d71310..bf6f2798f 100755 --- a/metplus/wrappers/plot_point_obs_wrapper.py +++ b/metplus/wrappers/plot_point_obs_wrapper.py @@ -19,6 +19,8 @@ class PlotPointObsWrapper(LoopTimesWrapper): """! Wrapper used to build commands to call plot_point_obs """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = 'ALL' WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_GRID_DATA_DICT', @@ -180,30 +182,6 @@ def get_command(self): f' "{self.infiles[0]}" {self.get_output_path()}' f" {' '.join(self.args)}") - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run plot_point_obs. - - @param time_info dictionary containing timing information - @returns True if command was built/run successfully or - False if something went wrong - """ - # get input files - if not self.find_input_files(time_info): - return False - - # get output path - if not self.find_and_check_output_file(time_info): - return False - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - return self.build() - def find_input_files(self, time_info): """! Get all input files for plot_point_obs. Sets self.infiles list. diff --git a/metplus/wrappers/point2grid_wrapper.py b/metplus/wrappers/point2grid_wrapper.py index e502554e7..e4cb356cd 100755 --- a/metplus/wrappers/point2grid_wrapper.py +++ b/metplus/wrappers/point2grid_wrapper.py @@ -16,7 +16,7 @@ from ..util import ti_calculate from ..util import do_string_sub from ..util import remove_quotes -from . import CommandBuilder +from . import LoopTimesWrapper '''!@namespace Point2GridWrapper @brief Wraps the Point2Grid tool to reformat ascii format to NetCDF @@ -24,7 +24,9 @@ ''' -class Point2GridWrapper(CommandBuilder): +class Point2GridWrapper(LoopTimesWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] def __init__(self, config, instance=None): self.app_name = "point2grid" @@ -145,27 +147,6 @@ def get_command(self): cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the application for - each. - Args: - @param input_dict dictionary containing timing information - """ - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - self.clear() - input_dict['lead'] = lead - - time_info = ti_calculate(input_dict) - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - time_info['custom'] = custom_string - - self.run_at_time_once(time_info) - def run_at_time_once(self, time_info): """! Process runtime and try to build command to run point2grid Args: diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index a5848b361..1836097d5 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -17,8 +17,11 @@ from ..util import do_string_sub from . import CompareGriddedWrapper + class PointStatWrapper(CompareGriddedWrapper): """! Wrapper to the MET tool, Point-Stat.""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', diff --git a/metplus/wrappers/py_embed_ingest_wrapper.py b/metplus/wrappers/py_embed_ingest_wrapper.py index 22732f473..0f6d06917 100755 --- a/metplus/wrappers/py_embed_ingest_wrapper.py +++ b/metplus/wrappers/py_embed_ingest_wrapper.py @@ -14,15 +14,19 @@ import re from ..util import time_util -from . import CommandBuilder -from . import RegridDataPlaneWrapper from ..util import do_string_sub, get_lead_sequence +from . import LoopTimesWrapper +from . import RegridDataPlaneWrapper VALID_PYTHON_EMBED_TYPES = ['NUMPY', 'XARRAY', 'PANDAS'] -class PyEmbedIngestWrapper(CommandBuilder): + +class PyEmbedIngestWrapper(LoopTimesWrapper): """!Wrapper to utilize Python Embedding in the MET tools to read in data using a python script""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + def __init__(self, config, instance=None): self.app_name = 'py_embed_ingest' super().__init__(config, instance=instance) @@ -123,34 +127,7 @@ def get_ingest_items(self, item_type, index, ingest_script_addons): return ingest_items - def run_at_time(self, input_dict): - """! Do some processing for the current run time (init or valid) - Args: - @param input_dict dictionary containing time information of current run - generally contains 'now' (current) time and 'init' or 'valid' time - """ - # get forecast leads to loop over - lead_seq = get_lead_sequence(self.config, input_dict) - for lead in lead_seq: - - # set forecast lead time in hours - input_dict['lead'] = lead - - # recalculate time info items - time_info = time_util.ti_calculate(input_dict) - - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing loop string: {custom_string}") - - time_info['custom'] = custom_string - - if not self.run_at_time_lead(time_info): - return False - - return True - - def run_at_time_lead(self, time_info): + def run_at_time_once(self, time_info): rdp = self.c_dict['regrid_data_plane'] # run each ingester specified diff --git a/metplus/wrappers/reformat_gridded_wrapper.py b/metplus/wrappers/reformat_gridded_wrapper.py index fc0dd5078..92aa3ce16 100755 --- a/metplus/wrappers/reformat_gridded_wrapper.py +++ b/metplus/wrappers/reformat_gridded_wrapper.py @@ -12,9 +12,9 @@ import os -from ..util import get_lead_sequence, sub_var_list +from ..util import get_lead_sequence from ..util import time_util, skip_time -from . import CommandBuilder +from . import LoopTimesWrapper # pylint:disable=pointless-string-statement '''!@namespace ReformatGriddedWrapper @@ -27,20 +27,13 @@ ''' -class ReformatGriddedWrapper(CommandBuilder): +class ReformatGriddedWrapper(LoopTimesWrapper): """! Common functionality to wrap similar MET applications that reformat gridded data """ def __init__(self, config, instance=None): super().__init__(config, instance=instance) - # this class should not be called directly - # pylint:disable=unused-argument - def run_at_time_once(self, time_info, var_list, data_type): - """!To be implemented by child class""" - self.log_error('ReformatGridded wrapper cannot be called directly.' - ' Please use child wrapper') - def run_at_time(self, input_dict): """! Runs the MET application for a given run time. Processing forecast or observation data is determined by conf variables. @@ -50,9 +43,6 @@ def run_at_time(self, input_dict): @param input_dict dictionary containing init or valid time info """ app_name_caps = self.app_name.upper() - class_name = self.__class__.__name__[0: -7] - lead_seq = get_lead_sequence(self.config, input_dict) - run_list = [] if self.config.getbool('config', 'FCST_'+app_name_caps+'_RUN', False): run_list.append("FCST") @@ -60,6 +50,7 @@ def run_at_time(self, input_dict): run_list.append("OBS") if not run_list: + class_name = self.__class__.__name__[0: -7] self.log_error(f"{class_name} specified in process_list, but " f"FCST_{app_name_caps}_RUN and " f"OBS_{app_name_caps}_RUN are both False. " @@ -69,33 +60,6 @@ def run_at_time(self, input_dict): for to_run in run_list: self.logger.info("Processing {} data".format(to_run)) - for lead in lead_seq: - input_dict['lead'] = lead - - time_info = time_util.ti_calculate(input_dict) - - self.logger.info("Processing forecast lead " - f"{time_info['lead_string']}") - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): - self.logger.debug('Skipping run time') - continue - - # loop over custom string list and set - # custom in the time_info dictionary - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info("Processing custom string: " - f"{custom_string}") - - time_info['custom'] = custom_string - self.c_dict['CUSTOM_STRING'] = custom_string - var_list_name = f'VAR_LIST_{to_run}' - var_list = ( - sub_var_list(self.c_dict.get(var_list_name, ''), - time_info) - ) - if not var_list: - var_list = None - - self.run_at_time_once(time_info, var_list, to_run) + self.c_dict['VAR_LIST'] = self.c_dict.get(f'VAR_LIST_{to_run}') + self.c_dict['DATA_SRC'] = to_run + super().run_at_time(input_dict) diff --git a/metplus/wrappers/regrid_data_plane_wrapper.py b/metplus/wrappers/regrid_data_plane_wrapper.py index 0211af427..7761d600e 100755 --- a/metplus/wrappers/regrid_data_plane_wrapper.py +++ b/metplus/wrappers/regrid_data_plane_wrapper.py @@ -14,7 +14,7 @@ from ..util import get_seconds_from_string, do_string_sub from ..util import parse_var_list, get_process_list -from ..util import add_field_info_to_time_info +from ..util import add_field_info_to_time_info, sub_var_list from ..util import remove_quotes, split_level, format_level from . import ReformatGriddedWrapper @@ -23,9 +23,13 @@ @brief Wraps the MET tool regrid_data_plane to reformat gridded datasets @endcode ''' + + class RegridDataPlaneWrapper(ReformatGriddedWrapper): - '''! Wraps the MET tool regrid_data_plane to reformat gridded datasets - ''' + """! Wraps the MET tool regrid_data_plane to reformat gridded datasets""" + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_FOR_EACH' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_FOR_EACH'] + def __init__(self, config, instance=None): self.app_name = 'regrid_data_plane' self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -111,7 +115,6 @@ def create_c_dict(self): met_tool=self.app_name ) - if self.config.getbool('config', 'OBS_REGRID_DATA_PLANE_RUN', False): window_types.append('OBS') c_dict['OBS_INPUT_DIR'] = \ @@ -292,14 +295,14 @@ def run_once_for_all_fields(self, time_info, var_list, data_type): # build and run commands return self.build() - def run_at_time_once(self, time_info, var_list, data_type): + def run_at_time_once(self, time_info): """!Build command or commands to run at the given run time - Args: - @param time_info time dictionary used for string substitution - @param var_list list of field dictionaries to process - @param data_type type of data to process, i.e. FCST or OBS + + @param time_info time dictionary used for string substitution """ self.clear() + var_list = sub_var_list(self.c_dict['VAR_LIST'], time_info) + data_type = self.c_dict['DATA_SRC'] # set output dir and template to current data type's values self.c_dict['OUTPUT_DIR'] = self.c_dict.get(f'{data_type}_OUTPUT_DIR') diff --git a/metplus/wrappers/runtime_freq_wrapper.py b/metplus/wrappers/runtime_freq_wrapper.py index 15de1d1ed..127fd9a20 100755 --- a/metplus/wrappers/runtime_freq_wrapper.py +++ b/metplus/wrappers/runtime_freq_wrapper.py @@ -17,7 +17,7 @@ from . import CommandBuilder from ..util import do_string_sub from ..util import log_runtime_banner, get_lead_sequence, is_loop_by_init -from ..util import skip_time, getlist +from ..util import skip_time, getlist, get_start_and_end_times, get_time_prefix from ..util import time_generator, add_to_time_input '''!@namespace RuntimeFreqWrapper @@ -57,9 +57,52 @@ def create_c_dict(self): f'{app_name_upper}_RUNTIME_FREQ', '').upper() ) + self.validate_runtime_freq(c_dict) return c_dict + def validate_runtime_freq(self, c_dict): + """!Check and update RUNTIME_FREQ. If RUNTIME_FREQ is unset and a + default value is set by the wrapper, use that value. If + """ + if not c_dict['RUNTIME_FREQ']: + # use default if there is one + if (hasattr(self, 'RUNTIME_FREQ_DEFAULT') and + self.RUNTIME_FREQ_DEFAULT is not None): + c_dict['RUNTIME_FREQ'] = self.RUNTIME_FREQ_DEFAULT + return + + # otherwise error + self.log_error(f'Must set {self.app_name.upper()}_RUNTIME_FREQ') + return + + # error if invalid value is set + if c_dict['RUNTIME_FREQ'] not in self.FREQ_OPTIONS: + self.log_error(f"Invalid value for " + f"{self.app_name.upper()}_RUNTIME_FREQ: " + f"({c_dict['RUNTIME_FREQ']}) " + f"Valid options include:" + f" {', '.join(self.FREQ_OPTIONS)}") + return + + # if list of supported frequencies are set by wrapper, + # warn and use default if frequency is not supported + if hasattr(self, 'RUNTIME_FREQ_SUPPORTED'): + if self.RUNTIME_FREQ_SUPPORTED == 'ALL': + return + + if c_dict['RUNTIME_FREQ'] not in self.RUNTIME_FREQ_SUPPORTED: + err_msg = (f"{self.app_name.upper()}_RUNTIME_FREQ=" + f"{c_dict['RUNTIME_FREQ']} not supported.") + if hasattr(self, 'RUNTIME_FREQ_DEFAULT'): + self.logger.warning( + f"{err_msg} Using {self.RUNTIME_FREQ_DEFAULT}" + ) + c_dict['RUNTIME_FREQ'] = self.RUNTIME_FREQ_DEFAULT + else: + self.log_error(err_msg) + return + def get_input_templates(self, c_dict): app_upper = self.app_name.upper() template_dict = {} @@ -94,14 +137,6 @@ def get_input_templates(self, c_dict): c_dict['TEMPLATE_DICT'] = template_dict def run_all_times(self): - if self.c_dict['RUNTIME_FREQ'] not in self.FREQ_OPTIONS: - self.log_error(f"Invalid value for " - f"{self.app_name.upper()}_RUNTIME_FREQ: " - f"({self.c_dict['RUNTIME_FREQ']}) " - f"Valid options include:" - f" {', '.join(self.FREQ_OPTIONS)}") - return None - wrapper_instance_name = self.get_wrapper_instance_name() self.logger.info(f'Running wrapper: {wrapper_instance_name}') @@ -146,11 +181,21 @@ def run_once(self, custom): time_input['valid'] = '*' time_input['lead'] = '*' + # set init or valid to time if _BEG is equal to _END + start_dt, end_dt = get_start_and_end_times(self.config) + if start_dt and start_dt == end_dt: + loop_by = get_time_prefix(self.config) + if loop_by: + time_input[loop_by.lower()] = start_dt + + time_info = time_util.ti_calculate(time_input) + if not self.get_all_files(custom): self.log_error("A problem occurred trying to obtain input files") return None - return self.run_at_time_once(time_input) + self.clear() + return self.run_at_time_once(time_info) def run_once_per_init_or_valid(self, custom): self.logger.debug(f"Running once for each init/valid time") @@ -172,11 +217,12 @@ def run_once_per_init_or_valid(self, custom): time_input['init'] = '*' time_input['lead'] = '*' + time_info = time_util.ti_calculate(time_input) - self.c_dict['ALL_FILES'] = self.get_all_files_from_leads(time_input) + self.c_dict['ALL_FILES'] = self.get_all_files_from_leads(time_info) self.clear() - if not self.run_at_time_once(time_input): + if not self.run_at_time_once(time_info): success = False return success @@ -202,10 +248,12 @@ def run_once_per_lead(self, custom): time_input['init'] = '*' time_input['valid'] = '*' - self.c_dict['ALL_FILES'] = self.get_all_files_for_lead(time_input) + time_info = time_util.ti_calculate(time_input) + + self.c_dict['ALL_FILES'] = self.get_all_files_for_lead(time_info) self.clear() - if not self.run_at_time_once(time_input): + if not self.run_at_time_once(time_info): success = False return success @@ -232,8 +280,13 @@ def run_once_for_each(self, custom): def run_at_time(self, input_dict): success = True + # loop of forecast leads and process each - lead_seq = get_lead_sequence(self.config, input_dict) + if self.c_dict.get('SKIP_LEAD_SEQ', False): + lead_seq = [0] + else: + lead_seq = get_lead_sequence(self.config, input_dict) + for lead in lead_seq: input_dict['lead'] = lead @@ -250,14 +303,8 @@ def run_at_time(self, input_dict): # since run_all_times was not called (LOOP_BY=times) then # get files for current run time - file_dict = self.get_files_from_time(time_info) all_files = [] - if file_dict: - if isinstance(file_dict, list): - all_files = file_dict - else: - all_files = [file_dict] - + self._update_list_with_new_files(time_info, all_files) self.c_dict['ALL_FILES'] = all_files # Run for given init/valid time and forecast lead combination @@ -267,6 +314,33 @@ def run_at_time(self, input_dict): return success + def run_at_time_once(self, time_info): + """! Process runtime and try to build command to run. Most wrappers + should be able to call this function to perform all of the actions + needed to build the commands using this template. This function can + be overridden if necessary. + + @param time_info dictionary containing timing information + @returns True if command was built/run successfully or + False if something went wrong + """ + # get input files + if not self.find_input_files(time_info): + return False + + # get output path + if not self.find_and_check_output_file(time_info): + return False + + # get other configurations for command + self.set_command_line_arguments(time_info) + + # set environment variables if using config file + self.set_environment_variables(time_info) + + # build command and run + return self.build() + def get_all_files(self, custom=None): """! Get all files that can be processed with the app. @returns A dictionary where the key is the type of data that was found, @@ -303,8 +377,7 @@ def get_all_files_from_leads(self, time_input): lead_files = [] # loop over all forecast leads - wildcard_if_empty = self.c_dict.get('WILDCARD_LEAD_IF_EMPTY', - False) + wildcard_if_empty = self.c_dict.get('WILDCARD_LEAD_IF_EMPTY', False) lead_seq = get_lead_sequence(self.config, time_input, wildcard_if_empty=wildcard_if_empty) @@ -318,12 +391,7 @@ def get_all_files_from_leads(self, time_input): if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): continue - file_dict = self.get_files_from_time(time_info) - if file_dict: - if isinstance(file_dict, list): - lead_files.extend(file_dict) - else: - lead_files.append(file_dict) + self._update_list_with_new_files(time_info, lead_files) return lead_files @@ -346,12 +414,8 @@ def get_all_files_for_lead(self, time_input): time_info = time_util.ti_calculate(current_time_input) if skip_time(time_info, self.c_dict.get('SKIP_TIMES')): continue - file_dict = self.get_files_from_time(time_info) - if file_dict: - if isinstance(file_dict, list): - new_files.extend(file_dict) - else: - new_files.append(file_dict) + + self._update_list_with_new_files(time_info, new_files) return new_files @@ -360,12 +424,19 @@ def get_files_from_time(time_info): """! Create dictionary containing time information (key time_info) and any relevant files for that runtime. @param time_info dictionary containing time information - @returns dictionary containing time_info dict and any relevant + @returns list of dict containing time_info dict and any relevant files with a key representing a description of that file """ - file_dict = {} - file_dict['time_info'] = time_info.copy() - return file_dict + return {'time_info': time_info.copy()} + + def _update_list_with_new_files(self, time_info, list_to_update): + new_files = self.get_files_from_time(time_info) + if not new_files: + return + if isinstance(new_files, list): + list_to_update.extend(new_files) + else: + list_to_update.append(new_files) @staticmethod def compare_time_info(runtime, filetime): @@ -400,7 +471,7 @@ def compare_time_info(runtime, filetime): return runtime_lead == filetime_lead - def find_input_files(self, time_info, fill_missing=False): + def get_input_files(self, time_info, fill_missing=False): """! Loop over list of input templates and find files for each @param time_info time dictionary to use for string substitution @@ -414,10 +485,16 @@ def find_input_files(self, time_info, fill_missing=False): return None for label, input_template in self.c_dict['TEMPLATE_DICT'].items(): - self.c_dict['INPUT_TEMPLATE'] = input_template + data_type = '' + template_key = 'INPUT_TEMPLATE' + if label in ('FCST', 'OBS'): + data_type = label + template_key = f'{label}_{template_key}' + + self.c_dict[template_key] = input_template # if fill missing is true, data is not mandatory to find mandatory = not fill_missing - input_files = self.find_data(time_info, + input_files = self.find_data(time_info, data_type=data_type, return_list=True, mandatory=mandatory) if not input_files: diff --git a/metplus/wrappers/series_analysis_wrapper.py b/metplus/wrappers/series_analysis_wrapper.py index e1394f425..1a3d0ae9d 100755 --- a/metplus/wrappers/series_analysis_wrapper.py +++ b/metplus/wrappers/series_analysis_wrapper.py @@ -34,9 +34,12 @@ from .plot_data_plane_wrapper import PlotDataPlaneWrapper from . import RuntimeFreqWrapper + class SeriesAnalysisWrapper(RuntimeFreqWrapper): """! Performs series analysis with filtering options """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = 'ALL' WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', diff --git a/metplus/wrappers/stat_analysis_wrapper.py b/metplus/wrappers/stat_analysis_wrapper.py index 180908f54..7011ef77e 100755 --- a/metplus/wrappers/stat_analysis_wrapper.py +++ b/metplus/wrappers/stat_analysis_wrapper.py @@ -27,6 +27,9 @@ class StatAnalysisWrapper(RuntimeFreqWrapper): ensemble_stat, and wavelet_stat """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE' + RUNTIME_FREQ_SUPPORTED = 'ALL' + WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', 'METPLUS_OBTYPE', @@ -176,23 +179,6 @@ def create_c_dict(self): c_dict['DATE_BEG'] = start_dt c_dict['DATE_END'] = end_dt - if not c_dict['RUNTIME_FREQ']: - # if start and end times are not equal and - # LOOP_ORDER = times (legacy), set frequency to once per init/valid - if (start_dt != end_dt and - self.config.has_option('config', 'LOOP_ORDER') and - self.config.getraw('config', 'LOOP_ORDER') == 'times'): - self.logger.warning( - 'LOOP_ORDER has been deprecated. Please set ' - 'STAT_ANALYSIS_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID ' - 'instead.' - ) - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_PER_INIT_OR_VALID' - else: - self.logger.debug('Setting RUNTIME_FREQ to RUN_ONCE. Set ' - 'STAT_ANALYSIS_RUNTIME_FREQ to override.') - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE' - # read jobs from STAT_ANALYSIS_JOB or legacy JOB_NAME/ARGS if unset c_dict['JOBS'] = self._read_jobs_from_config() @@ -219,6 +205,27 @@ def create_c_dict(self): return self._c_dict_error_check(c_dict, all_field_lists_empty) + def validate_runtime_freq(self, c_dict): + """!Check and update RUNTIME_FREQ. Performs additional checks for + deprecated LOOP_ORDER=times setting before calling parent class + version of function. This function will eventually be removed. + """ + if not c_dict['RUNTIME_FREQ']: + # if start and end times are not equal and + # LOOP_ORDER = times (legacy), set frequency to once per init/valid + start_dt, end_dt = get_start_and_end_times(self.config) + if (start_dt != end_dt and + self.config.has_option('config', 'LOOP_ORDER') and + self.config.getraw('config', 'LOOP_ORDER') == 'times'): + self.logger.warning( + 'LOOP_ORDER has been deprecated. Please set ' + 'STAT_ANALYSIS_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID ' + 'instead.' + ) + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_PER_INIT_OR_VALID' + + super().validate_runtime_freq(c_dict) + def run_at_time_once(self, time_input): """! Function called when processing all times. diff --git a/metplus/wrappers/tc_diag_wrapper.py b/metplus/wrappers/tc_diag_wrapper.py index 1b8ebd8a7..56bf83bcc 100755 --- a/metplus/wrappers/tc_diag_wrapper.py +++ b/metplus/wrappers/tc_diag_wrapper.py @@ -26,6 +26,8 @@ class TCDiagWrapper(RuntimeFreqWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_PER_INIT_OR_VALID'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', @@ -80,12 +82,6 @@ def create_c_dict(self): # skip RuntimeFreq wrapper logic to find files c_dict['FIND_FILES'] = False - if not c_dict['RUNTIME_FREQ']: - c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_PER_INIT_OR_VALID' - if c_dict['RUNTIME_FREQ'] != 'RUN_ONCE_PER_INIT_OR_VALID': - self.log_error('Only RUN_ONCE_PER_INIT_OR_VALID is supported for ' - 'TC_DIAG_RUNTIME_FREQ.') - # get command line arguments domain and tech id list for -data self._read_data_inputs(c_dict) diff --git a/metplus/wrappers/tc_gen_wrapper.py b/metplus/wrappers/tc_gen_wrapper.py index db77f7cc5..6f3a9a9f7 100755 --- a/metplus/wrappers/tc_gen_wrapper.py +++ b/metplus/wrappers/tc_gen_wrapper.py @@ -17,14 +17,17 @@ from ..util import time_util from ..util import do_string_sub, skip_time, get_lead_sequence from ..util import time_generator -from . import CommandBuilder +from . import RuntimeFreqWrapper '''!@namespace TCGenWrapper @brief Wraps the TC-Gen tool @endcode ''' -class TCGenWrapper(CommandBuilder): + +class TCGenWrapper(RuntimeFreqWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_INIT_FREQ', @@ -92,7 +95,6 @@ class TCGenWrapper(CommandBuilder): 'best_fn_oy', ] - def __init__(self, config, instance=None): self.app_name = "tc_gen" self.app_path = os.path.join(config.getdir('MET_BIN_DIR'), @@ -279,11 +281,6 @@ def create_c_dict(self): ) self.add_met_config_window('genesis_match_window') - # get INPUT_TIME_DICT values since wrapper doesn't loop over time - c_dict['INPUT_TIME_DICT'] = next(time_generator(self.config)) - if not c_dict['INPUT_TIME_DICT']: - self.isOK = False - return c_dict def handle_filter(self): @@ -326,37 +323,6 @@ def get_command(self): return cmd - def run_all_times(self): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the - application for each. - - @param input_dict dictionary containing timing information - """ - # run using input time dictionary - self.run_at_time(self.c_dict['INPUT_TIME_DICT']) - return self.all_commands - - def run_at_time(self, input_dict): - """! Process runtime and try to build command to run ascii2nc - Args: - @param input_dict dictionary containing timing information - """ - input_dict['instance'] = self.instance if self.instance else '' - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - time_info = time_util.ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - self.clear() - self.run_at_time_once(time_info) - def run_at_time_once(self, time_info): """! Process runtime and try to build command to run ascii2nc Args: diff --git a/metplus/wrappers/tc_pairs_wrapper.py b/metplus/wrappers/tc_pairs_wrapper.py index 78377e431..a95bd925c 100755 --- a/metplus/wrappers/tc_pairs_wrapper.py +++ b/metplus/wrappers/tc_pairs_wrapper.py @@ -26,7 +26,7 @@ from ..util import get_tags, find_indices_in_config_section from ..util.met_config import add_met_config_dict_list from ..util import time_generator, log_runtime_banner, add_to_time_input -from . import CommandBuilder +from . import RuntimeFreqWrapper '''!@namespace TCPairsWrapper @brief Wraps the MET tool tc_pairs to parse ADeck and BDeck ATCF_by_pairs @@ -37,10 +37,13 @@ @endcode ''' -class TCPairsWrapper(CommandBuilder): + +class TCPairsWrapper(RuntimeFreqWrapper): """!Wraps the MET tool, tc_pairs to parse and match ATCF_by_pairs adeck and bdeck files. Pre-processes extra tropical cyclone data. """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE' + RUNTIME_FREQ_SUPPORTED = 'ALL' WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', @@ -206,12 +209,6 @@ def create_c_dict(self): if not c_dict['OUTPUT_DIR']: self.log_error('TC_PAIRS_OUTPUT_DIR must be set') - c_dict['READ_ALL_FILES'] = ( - self.config.getbool('config', - 'TC_PAIRS_READ_ALL_FILES', - False) - ) - # get list of models to process c_dict['MODEL_LIST'] = getlist( self.config.getraw('config', 'MODEL', '') @@ -287,92 +284,77 @@ def create_c_dict(self): self.handle_description() - c_dict['SKIP_LEAD_SEQ'] = ( - self.config.getbool('config', - 'TC_PAIRS_SKIP_LEAD_SEQ', - False) + return c_dict + + def validate_runtime_freq(self, c_dict): + """!Figure out time looping configuration based on legacy config + variables. + READ_ALL_FILES: Only pass directories to tc_pairs and + let the app handle all filtering. Force RUN_ONCE if set. + To preserve behavior when deprecated LOOP_ORDER=times and + TC_PAIRS_RUN_ONCE is not set, + + @param c_dict dictionary to populate with values from METplusConfig + """ + c_dict['READ_ALL_FILES'] = ( + self.config.getbool('config', 'TC_PAIRS_READ_ALL_FILES', False) ) + if c_dict['READ_ALL_FILES']: + if c_dict['RUNTIME_FREQ'] != 'RUN_ONCE': + self.logger.debug('TC_PAIRS_READ_ALL_FILES=True. ' + 'Forcing TC_PAIRS_RUNTIME_FREQ=RUN_ONCE') + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE' # check for settings that cause differences moving from v4.1 to v5.0 # warn and update run setting to preserve old behavior - if (self.config.has_option('config', 'LOOP_ORDER') and - self.config.getstr_nocheck('config', 'LOOP_ORDER') == 'times' and - not self.config.has_option('config', 'TC_PAIRS_RUN_ONCE')): + elif (self.config.has_option('config', 'LOOP_ORDER') and + self.config.getstr_nocheck('config', 'LOOP_ORDER') == 'times' and + not (self.config.has_option('config', 'TC_PAIRS_RUN_ONCE') or + self.config.has_option('config', 'TC_PAIRS_RUNTIME_FREQ'))): self.logger.warning( 'LOOP_ORDER has been deprecated. LOOP_ORDER has been set to ' - '"times" and TC_PAIRS_RUN_ONCE is not set. ' - 'Forcing TC_PAIRS_RUN_ONCE=False to preserve behavior prior to ' - 'v5.0.0. Please remove LOOP_ORDER and set ' - 'TC_PAIRS_RUN_ONCE=False to preserve previous behavior and ' - 'remove this warning message.' + '"times" and TC_PAIRS_RUNTIME_FREQ is not set. ' + 'Forcing TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH to ' + 'preserve behavior prior to v5.0.0. Please remove LOOP_ORDER ' + 'and set TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH to preserve ' + 'previous behavior and remove this warning message.' ) - c_dict['RUN_ONCE'] = False - return c_dict - - # only run once if True - c_dict['RUN_ONCE'] = self.config.getbool('config', - 'TC_PAIRS_RUN_ONCE', - True) - return c_dict - - def run_all_times(self): - """! Build up the command to invoke the MET tool tc_pairs. - """ - # use first run time - input_dict = next(time_generator(self.config)) - if not input_dict: - return self.all_commands - - add_to_time_input(input_dict, - instance=self.instance) - log_runtime_banner(self.config, input_dict, self) - - # if running in READ_ALL_FILES mode, call tc_pairs once and exit - if self.c_dict['READ_ALL_FILES']: - return self._read_all_files(input_dict) - - if not self.c_dict['RUN_ONCE']: - return super().run_all_times() - - self.logger.debug('Only processing first run time. Set ' - 'TC_PAIRS_RUN_ONCE=False to process all run times.') - self.run_at_time(input_dict) - return self.all_commands - - def run_at_time(self, input_dict): - """! Create the arguments to run MET tc_pairs - Args: - input_dict dictionary containing init or valid time - Returns: - """ - input_dict['instance'] = self.instance if self.instance else '' - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - - # if skipping lead sequence, only run once per init/valid time - if self.c_dict['SKIP_LEAD_SEQ']: - lead_seq = [0] + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_FOR_EACH' + + # check deprecated TC_PAIRS_RUN_ONCE, warn and handle if set + elif self.config.has_option('config', 'TC_PAIRS_RUN_ONCE'): + self.logger.warning('TC_PAIRS_RUN_ONCE is deprecated.') + run_once = self.config.getbool('config', 'TC_PAIRS_RUN_ONCE', True) + if run_once: + self.logger.warning('Setting TC_PAIRS_RUNTIME_FREQ=RUN_ONCE.' + 'Please remove TC_PAIRS_RUN_ONCE and ' + 'set TC_PAIRS_RUNTIME_FREQ=RUN_ONCE ' + 'to remove this warning') + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE' else: - lead_seq = get_lead_sequence(self.config, input_dict) - - for lead in lead_seq: - input_dict['lead'] = lead - time_info = ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue + self.logger.warning('Setting TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH.' + 'Please remove TC_PAIRS_RUN_ONCE and ' + 'set TC_PAIRS_RUNTIME_FREQ=RUN_ONCE_FOR_EACH ' + 'to remove this warning') + c_dict['RUNTIME_FREQ'] = 'RUN_ONCE_FOR_EACH' + + # if runtime frequency set to run once for each time, check skip lead + if c_dict['RUNTIME_FREQ'] == 'RUN_ONCE_FOR_EACH': + c_dict['SKIP_LEAD_SEQ'] = ( + self.config.getbool('config', 'TC_PAIRS_SKIP_LEAD_SEQ', False) + ) - self.run_at_time_loop_string(time_info) + super().validate_runtime_freq(c_dict) - def run_at_time_loop_string(self, time_info): + def run_at_time_once(self, time_info): """! Create the arguments to run MET tc_pairs @param time_info dictionary containing time information """ + # if running in READ_ALL_FILES mode, call tc_pairs once and exit + if self.c_dict['READ_ALL_FILES']: + return self._read_all_files(time_info) + # set output dir self.outdir = self.c_dict['OUTPUT_DIR'] @@ -800,6 +782,7 @@ def _get_basin_cyclone_from_bdeck(self, bdeck_file, wildcard_used, # capture wildcard values in template - must replace ? wildcard # character after substitution because ? is used in template tags + bdeck_regex = bdeck_regex.replace('(*)', '*') bdeck_regex = bdeck_regex.replace('*', '(.*)').replace('?', '(.)') self.logger.debug(f'Regex to extract basin/cyclone: {bdeck_regex}') diff --git a/metplus/wrappers/tc_stat_wrapper.py b/metplus/wrappers/tc_stat_wrapper.py index 9c7b6722b..0a133a861 100755 --- a/metplus/wrappers/tc_stat_wrapper.py +++ b/metplus/wrappers/tc_stat_wrapper.py @@ -17,7 +17,7 @@ from datetime import datetime from ..util import getlist, mkdir_p, do_string_sub, ti_calculate -from . import CommandBuilder +from . import RuntimeFreqWrapper ## @namespace TCStatWrapper # @brief Wrapper to the MET tool tc_stat, which is used for filtering tropical @@ -29,10 +29,12 @@ # attribute data. -class TCStatWrapper(CommandBuilder): +class TCStatWrapper(RuntimeFreqWrapper): """! Wrapper for the MET tool, tc_stat, which is used to filter tropical cyclone pair data. """ + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_PER_INIT_OR_VALID'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_AMODEL', @@ -245,7 +247,7 @@ def set_met_config_for_environment_variables(self): 'TC_STAT_INIT_STR_EXCLUDE_VAL', ]) - def run_at_time(self, input_dict=None): + def run_at_time_once(self, input_dict=None): """! Builds the call to the MET tool TC-STAT for all requested initialization times (init or valid). Called from run_metplus """ diff --git a/metplus/wrappers/tcrmw_wrapper.py b/metplus/wrappers/tcrmw_wrapper.py index 4784c18bf..0f021e6e0 100755 --- a/metplus/wrappers/tcrmw_wrapper.py +++ b/metplus/wrappers/tcrmw_wrapper.py @@ -12,10 +12,10 @@ import os -from ..util import time_util -from . import CommandBuilder +from ..util import ti_calculate, ti_get_hours_from_relativedelta from ..util import do_string_sub, skip_time, get_lead_sequence from ..util import parse_var_list, sub_var_list +from . import RuntimeFreqWrapper '''!@namespace TCRMWWrapper @brief Wraps the TC-RMW tool @@ -23,7 +23,9 @@ ''' -class TCRMWWrapper(CommandBuilder): +class TCRMWWrapper(RuntimeFreqWrapper): + RUNTIME_FREQ_DEFAULT = 'RUN_ONCE_PER_INIT_OR_VALID' + RUNTIME_FREQ_SUPPORTED = ['RUN_ONCE_PER_INIT_OR_VALID'] WRAPPER_ENV_VAR_KEYS = [ 'METPLUS_MODEL', @@ -162,6 +164,8 @@ def create_c_dict(self): c_dict['VAR_LIST_TEMP'] = parse_var_list(self.config, data_type='FCST', met_tool=self.app_name) + if not c_dict['VAR_LIST_TEMP']: + self.log_error("Could not get field information from config.") return c_dict @@ -196,87 +200,6 @@ def get_command(self): cmd += ' -v ' + self.c_dict['VERBOSITY'] return cmd - def run_at_time(self, input_dict): - """! Runs the MET application for a given run time. This function - loops over the list of forecast leads and runs the - application for each. - Args: - @param input_dict dictionary containing timing information - """ - for custom_string in self.c_dict['CUSTOM_LOOP_LIST']: - if custom_string: - self.logger.info(f"Processing custom string: {custom_string}") - - input_dict['custom'] = custom_string - time_info = time_util.ti_calculate(input_dict) - - if skip_time(time_info, self.c_dict.get('SKIP_TIMES', {})): - self.logger.debug('Skipping run time') - continue - - self.clear() - self.run_at_time_once(time_info) - - def run_at_time_once(self, time_info): - """! Process runtime and try to build command to run ascii2nc - Args: - @param time_info dictionary containing timing information - """ - # get input files - if self.find_input_files(time_info) is None: - return - - # get output path - if not self.find_and_check_output_file(time_info): - return - - # get field information to set in MET config - if not self.set_data_field(time_info): - return - - # get other configurations for command - self.set_command_line_arguments(time_info) - - # set environment variables if using config file - self.set_environment_variables(time_info) - - # build command and run - cmd = self.get_command() - if cmd is None: - self.log_error("Could not generate command") - return - - self.build() - - def set_data_field(self, time_info): - """!Get list of fields from config to process. Build list of field info - that are formatted to be read by the MET config file. Set DATA_FIELD - item of c_dict with the formatted list of fields. - Args: - @param time_info time dictionary to use for string substitution - @returns True if field list could be built, False if not. - """ - field_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) - if not field_list: - self.log_error("Could not get field information from config.") - return False - - all_fields = [] - for field in field_list: - field_list = self.get_field_info(d_type='FCST', - v_name=field['fcst_name'], - v_level=field['fcst_level'], - ) - if field_list is None: - return False - - all_fields.extend(field_list) - - data_field = ','.join(all_fields) - self.env_var_dict['METPLUS_DATA_FIELD'] = f'field = [{data_field}];' - - return True - def find_input_files(self, time_info): """!Get DECK file and list of input data files and set c_dict items. Args: @@ -308,7 +231,7 @@ def find_input_files(self, time_info): self.clear() time_info['lead'] = lead - time_info = time_util.ti_calculate(time_info) + time_info = ti_calculate(time_info) # get a list of the input data files, # write to an ascii file if there are more than one @@ -327,23 +250,58 @@ def find_input_files(self, time_info): self.infiles.append(list_file) - # set LEAD_LIST to list of forecast leads used - if lead_seq != [0]: - lead_list = [] - for lead in lead_seq: - lead_hours = ( - time_util.ti_get_hours_from_relativedelta(lead, - valid_time=time_info['valid']) - ) - lead_list.append(f'"{str(lead_hours).zfill(2)}"') + if not self._set_data_field(time_info): + return None - self.env_var_dict['METPLUS_LEAD_LIST'] = f"lead = [{', '.join(lead_list)}];" + self._set_lead_list(time_info, lead_seq) return self.infiles - def set_command_line_arguments(self, time_info): + def _set_data_field(self, time_info): + """!Get list of fields from config to process. Build list of field info + that are formatted to be read by the MET config file. Set DATA_FIELD + item of c_dict with the formatted list of fields. + Args: + @param time_info time dictionary to use for string substitution + @returns True if field list could be built, False if not. + """ + field_list = sub_var_list(self.c_dict['VAR_LIST_TEMP'], time_info) + if not field_list: + self.log_error("Could not get field information from config.") + return False - # add config file - passing through do_string_sub to get custom string if set + all_fields = [] + for field in field_list: + field_list = self.get_field_info(d_type='FCST', + v_name=field['fcst_name'], + v_level=field['fcst_level'], + ) + if field_list is None: + self.log_error(f'Could not get field info from {field}') + return False + + all_fields.extend(field_list) + + data_field = ','.join(all_fields) + self.env_var_dict['METPLUS_DATA_FIELD'] = f'field = [{data_field}];' + return True + + def _set_lead_list(self, time_info, lead_seq): + # set LEAD_LIST to list of forecast leads used + if lead_seq == [0]: + return + + lead_list = [] + for lead in lead_seq: + lead_hours = ( + ti_get_hours_from_relativedelta(lead, + valid_time=time_info['valid']) + ) + lead_list.append(f'"{str(lead_hours).zfill(2)}"') + + self.env_var_dict['METPLUS_LEAD_LIST'] = f"lead = [{', '.join(lead_list)}];" + + def set_command_line_arguments(self, time_info): if self.c_dict['CONFIG_FILE']: config_file = do_string_sub(self.c_dict['CONFIG_FILE'], **time_info) diff --git a/metplus/wrappers/usage_wrapper.py b/metplus/wrappers/usage_wrapper.py index 77c26b575..f37151a80 100644 --- a/metplus/wrappers/usage_wrapper.py +++ b/metplus/wrappers/usage_wrapper.py @@ -6,6 +6,7 @@ from . import CommandBuilder from ..util import LOWER_TO_WRAPPER_NAME + class UsageWrapper(CommandBuilder): """! A default process, prints out usage when nothing is defined in the PROCESS_LIST diff --git a/metplus/wrappers/user_script_wrapper.py b/metplus/wrappers/user_script_wrapper.py index 32e50ac38..dae9bdf0a 100755 --- a/metplus/wrappers/user_script_wrapper.py +++ b/metplus/wrappers/user_script_wrapper.py @@ -22,7 +22,11 @@ @endcode ''' + class UserScriptWrapper(RuntimeFreqWrapper): + RUNTIME_FREQ_DEFAULT = None + RUNTIME_FREQ_SUPPORTED = 'ALL' + def __init__(self, config, instance=None): self.app_name = "user_script" super().__init__(config, instance=instance) @@ -95,7 +99,7 @@ def get_files_from_time(self, time_info): """ file_dict = super().get_files_from_time(time_info) - input_files = self.find_input_files(time_info, fill_missing=True) + input_files = self.get_input_files(time_info, fill_missing=True) if input_files is None: return file_dict diff --git a/parm/README b/parm/README deleted file mode 100644 index efcf9eb23..000000000 --- a/parm/README +++ /dev/null @@ -1,17 +0,0 @@ -This README describes what config files reside here and their function/purpose. -The following directories contain config files specific to their responsibilities: - -1) metplus_config - Contains all the configuration files necessary for running METplus: - a) metplus_data.conf - Indicate the directories where all necessary input data resides and where output data should be saved. - b) metplus_logging.conf - Specify the format of log output. - c) metplus_runtime.conf - Indicate the information needed to run METplus, such as ... - d) metplus_system.conf - Any configuration that is specific to a user's work environment, such as the location of the MET executables and any other libraries used by METplus that - may vary from one work environment to another. - -3) use_cases - This directory contains the configuration files specific to running a particular use case. diff --git a/parm/README.md b/parm/README.md new file mode 100644 index 000000000..27b2f07e0 --- /dev/null +++ b/parm/README.md @@ -0,0 +1,6 @@ +The following directories contain configuration/parameter files: + +* **met_config** - Contains *wrapped* MET configuration files. These files reference environment variables that are set by the METplus wrappers to override the default MET settings. See [How METplus controls MET configuration variables](https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#how-metplus-controls-met-configuration-variables) for more information. +* **metplus_config** - Contains the default configuration file (_defaults.conf_) that contains settings that are read first when running METplus. See [Default Configuration File](https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#default-configuration-file) for more information. + +* **use_cases** - Contains example configuration files used to run a particular use case. See [Use Case Configuration Files](https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#use-case-configuration-files) for more information. diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf index 23c81e647..e2db24760 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_extra_tropical.conf @@ -31,7 +31,7 @@ INIT_BEG = 2014121318 INIT_END = 2014121318 INIT_INCREMENT = 21600 -TC_PAIRS_RUN_ONCE = True +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE ### @@ -59,7 +59,6 @@ TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = yes TC_PAIRS_SKIP_IF_REFORMAT_EXISTS = yes TC_PAIRS_READ_ALL_FILES = no -#TC_PAIRS_SKIP_LEAD_SEQ = False TC_PAIRS_REFORMAT_DECK = yes TC_PAIRS_REFORMAT_TYPE = SBU diff --git a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf index ab9d64900..2df99931c 100644 --- a/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf +++ b/parm/use_cases/met_tool_wrapper/TCPairs/TCPairs_tropical.conf @@ -31,9 +31,7 @@ INIT_BEG = 2018083006 INIT_END = 2018083018 INIT_INCREMENT = 21600 -#TC_PAIRS_SKIP_LEAD_SEQ = False - -TC_PAIRS_RUN_ONCE = False +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE_FOR_EACH ### diff --git a/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf b/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf index c9b54fc87..8f9186f46 100644 --- a/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf +++ b/parm/use_cases/model_applications/medium_range/MTD_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf @@ -46,13 +46,13 @@ OBS_MTD_INPUT_DIR = {FCST_MTD_INPUT_DIR} OBS_MTD_INPUT_TEMPLATE = {valid?fmt=%Y%m%d%H}/gfs.t{valid?fmt=%H}z.pgrb2.1p00.f000 MTD_OUTPUT_DIR = {OUTPUT_BASE}/mtd -MTD_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d%H} +MTD_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d%H} EXTRACT_TILES_SKIP_IF_OUTPUT_EXISTS = no -EXTRACT_TILES_MTD_INPUT_DIR = {OUTPUT_BASE}/mtd -EXTRACT_TILES_MTD_INPUT_TEMPLATE = {init?fmt=%Y%m%d%H}/mtd_{MODEL}_{FCST_VAR1_NAME}_vs_{OBTYPE}_{OBS_VAR1_NAME}_{OBS_VAR1_LEVELS}_{init?fmt=%Y%m%d_%H%M%S}V_2d.txt +EXTRACT_TILES_MTD_INPUT_DIR = {MTD_OUTPUT_DIR} +EXTRACT_TILES_MTD_INPUT_TEMPLATE = {MTD_OUTPUT_TEMPLATE}/mtd_{MODEL}_{FCST_VAR1_NAME}_vs_{OBTYPE}_{OBS_VAR1_NAME}_{OBS_VAR1_LEVELS}_{init?fmt=%Y%m%d_%H%M%S}V_2d.txt FCST_EXTRACT_TILES_INPUT_DIR = {FCST_MTD_INPUT_DIR} FCST_EXTRACT_TILES_INPUT_TEMPLATE = {FCST_MTD_INPUT_TEMPLATE} diff --git a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf index 4a11246bf..087d75938 100644 --- a/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf +++ b/parm/use_cases/model_applications/medium_range/PointStat_fcstGFS_obsGDAS_UpperAir_MultiField_PrepBufr.conf @@ -114,7 +114,6 @@ PB2NC_TIME_SUMMARY_TYPES = min, max, range, mean, stdev, median, p80 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#pointstat ### -POINT_STAT_CLIMO_MEAN_TIME_INTERP_METHOD = NEAREST POINT_STAT_INTERP_TYPE_METHOD = BILIN POINT_STAT_INTERP_TYPE_WIDTH = 2 diff --git a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf index dd0fc4827..3ac8a6f08 100644 --- a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf +++ b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead.conf @@ -29,7 +29,6 @@ LOOP_BY = INIT INIT_TIME_FMT = %Y%m%d INIT_BEG = 20141214 INIT_END = 20141214 -INIT_INCREMENT = 21600 ;; set to every 6 hours=21600 seconds LEAD_SEQ_1 = begin_end_incr(0,18,6) LEAD_SEQ_1_LABEL = Day1 @@ -37,6 +36,8 @@ LEAD_SEQ_1_LABEL = Day1 LEAD_SEQ_2 = begin_end_incr(24,42,6) LEAD_SEQ_2_LABEL = Day2 +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE + ### # File I/O @@ -109,8 +110,6 @@ BOTH_VAR1_LEVELS = Z2 # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#tcpairs ### -TC_PAIRS_SKIP_LEAD_SEQ = True - TC_PAIRS_INIT_INCLUDE = TC_PAIRS_INIT_EXCLUDE = diff --git a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf index 6cf9ba3c7..ea00e6493 100644 --- a/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf +++ b/parm/use_cases/model_applications/medium_range/TCStat_SeriesAnalysis_fcstGFS_obsGFS_FeatureRelative_SeriesByLead_PyEmbed_Multiple_Diagnostics.conf @@ -33,6 +33,7 @@ INIT_INCREMENT = 21600 LEAD_SEQ = 90, 96, 102, 108, 114 +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE SERIES_ANALYSIS_RUNTIME_FREQ = RUN_ONCE_PER_LEAD SERIES_ANALYSIS_RUN_ONCE_PER_STORM_ID = False @@ -60,7 +61,7 @@ TC_PAIRS_REFORMAT_DIR = {OUTPUT_BASE}/track_data_atcf TC_PAIRS_SKIP_IF_REFORMAT_EXISTS = no TC_PAIRS_OUTPUT_DIR = {OUTPUT_BASE}/tc_pairs -TC_PAIRS_OUTPUT_TEMPLATE = {date?fmt=%Y%m}/{basin?fmt=%s}q{date?fmt=%Y%m%d%H}.dorian +TC_PAIRS_OUTPUT_TEMPLATE = {basin?fmt=%s}q{INIT_BEG}.dorian TC_PAIRS_SKIP_IF_OUTPUT_EXISTS = no @@ -169,8 +170,6 @@ PY_EMBED_INGEST_2_OUTPUT_GRID = {MODEL_DIR}/{valid?fmt=%Y%m%d}/gfs_4_{valid?fmt= # https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#tcpairs ### -TC_PAIRS_SKIP_LEAD_SEQ = True - TC_PAIRS_INIT_INCLUDE = TC_PAIRS_INIT_EXCLUDE = diff --git a/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf b/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf index 3ac50c4d4..35cb1522c 100644 --- a/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf +++ b/parm/use_cases/model_applications/short_range/METdbLoad_fcstFV3_obsGoes_BrightnessTemp.conf @@ -28,8 +28,8 @@ PROCESS_LIST = METDbLoad LOOP_BY = VALID VALID_TIME_FMT = %Y%m%d%H -VALID_BEG = 2019052112 -VALID_END = 2019052100 +VALID_BEG = 2019052100 +VALID_END = 2019052112 VALID_INCREMENT = 12H MET_DB_LOAD_RUNTIME_FREQ = RUN_ONCE diff --git a/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect/read_ascii_storm.py b/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect/read_ascii_storm.py index 1340e3b27..9588eb2ef 100644 --- a/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect/read_ascii_storm.py +++ b/parm/use_cases/model_applications/short_range/Point2Grid_obsLSR_ObsOnly_PracticallyPerfect/read_ascii_storm.py @@ -46,6 +46,9 @@ #Allows for concatenating storm reports together temp_data = temp_data[temp_data["Time"] != "Time"] +# strip out any rows that have any null/NaN values +temp_data = temp_data[~temp_data.isnull().any(axis=1)] + #Change some columns to floats and ints temp_data[["Lat","Lon"]] = temp_data[["Lat","Lon"]].apply(pd.to_numeric) diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf index 1e3b3fc45..807619631 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_UserScript_ExtraTC.conf @@ -26,12 +26,11 @@ PROCESS_LIST = UserScript, TCPairs, CyclonePlotter ### LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%d%H -INIT_BEG = 2020100700 -INIT_END = 2020100700 -INIT_INCREMENT = 21600 +INIT_TIME_FMT = %Y%m%d +INIT_BEG = 20201007 -USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE_PER_INIT_OR_VALID +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE ### diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf index e45b93aa0..52f2e39ac 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/Plotter_fcstGFS_obsGFS_ExtraTC.conf @@ -26,12 +26,10 @@ PROCESS_LIST = TCPairs, CyclonePlotter ### LOOP_BY = init -INIT_TIME_FMT = %Y%m%d -INIT_BEG = 20150301 -INIT_END = 20150330 -INIT_INCREMENT = 21600 +INIT_TIME_FMT = %Y%m +INIT_BEG = 201503 -TC_PAIRS_RUN_ONCE = True +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE ### @@ -74,6 +72,8 @@ TC_PAIRS_REFORMAT_TYPE = SBU TC_PAIRS_MISSING_VAL_TO_REPLACE = -99 TC_PAIRS_MISSING_VAL = -9999 +TC_PAIRS_INIT_BEG = {init?fmt=%Y%m}00 +TC_PAIRS_INIT_END = {init?fmt=%Y%m}30 ### # CyclonePlotter Settings diff --git a/parm/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf b/parm/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf index 9847b29a1..854f18005 100644 --- a/parm/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf +++ b/parm/use_cases/model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf @@ -26,10 +26,10 @@ PROCESS_LIST = TCPairs, TCStat ### LOOP_BY = INIT -INIT_TIME_FMT = %Y%m%d%H -INIT_BEG = 2021082500 -INIT_END = 2021083000 -INIT_INCREMENT = 21600 +INIT_TIME_FMT = %Y%m +INIT_BEG = 202108 + +TC_PAIRS_RUNTIME_FREQ = RUN_ONCE ### # File I/O @@ -67,6 +67,8 @@ TC_PAIRS_DLAND_FILE = MET_BASE/tc_data/dland_global_tenth_degree.nc TC_PAIRS_MATCH_POINTS = TRUE +TC_PAIRS_INIT_BEG = {init?fmt=%Y%m}25_00 +TC_PAIRS_INIT_END = {init?fmt=%Y%m}30_00 ### # TCStat Settings @@ -81,3 +83,6 @@ TC_STAT_COLUMN_STRING_VAL = HU,SD,SS,TS,TD TC_STAT_WATER_ONLY = FALSE TC_STAT_JOB_ARGS = -job filter -dump_row {TC_STAT_OUTPUT_DIR}/tc_stat_summary.tcst + +TC_STAT_INIT_BEG = {init?fmt=%Y%m}25_00 +TC_STAT_INIT_END = {init?fmt=%Y%m}30_00