diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6c4450e814..6c3879a447 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -54,7 +54,7 @@ Describe the steps to reproduce the behavior: ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Bugfix Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md index 3133dec892..1e9e317569 100644 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -43,7 +43,7 @@ Consider breaking the enhancement down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Enhancement Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/new_feature_request.md b/.github/ISSUE_TEMPLATE/new_feature_request.md index b37efe8aef..f075a79cba 100644 --- a/.github/ISSUE_TEMPLATE/new_feature_request.md +++ b/.github/ISSUE_TEMPLATE/new_feature_request.md @@ -47,7 +47,7 @@ Consider breaking the new feature down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## New Feature Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/task.md b/.github/ISSUE_TEMPLATE/task.md index ce3a6fb09a..0e25bfe880 100644 --- a/.github/ISSUE_TEMPLATE/task.md +++ b/.github/ISSUE_TEMPLATE/task.md @@ -43,7 +43,7 @@ Consider breaking the task down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Task Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/dummy_for_action b/.github/dummy_for_action deleted file mode 100644 index 91bdf4d63e..0000000000 --- a/.github/dummy_for_action +++ /dev/null @@ -1,2 +0,0 @@ -#update me to add action comment - diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 14f85ddb93..03d8a2fe32 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -17,6 +17,13 @@ if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then run_diff=true + # if pull request into main_vX.Y branch, + # set truth data version to branch name and set input data version to X.Y + if [ "${GITHUB_BASE_REF:0:6}" == "main_v" ]; then + truth_data_version=${GITHUB_BASE_REF} + input_data_version=${GITHUB_BASE_REF:6} + fi + fi elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then @@ -30,6 +37,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then run_diff=true truth_data_version=${branch_name:0: -4} + # if main_vX.Y-ref branch, use X.Y input data + if [ "${branch_name:0:6}" == "main_v" ]; then + input_data_version=${branch_name:6: -4} + fi + else # if develop or main_vX.Y branch, run diff tests using branch's truth data @@ -39,6 +51,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then run_diff=true truth_data_version=${branch_name} + # if main_vX.Y branch, use X.Y input data + if [ "${branch_name:0:6}" == "main_v" ]; then + input_data_version=${branch_name:6} + fi + fi # check commit messages for skip or force keywords @@ -66,7 +83,13 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then fi fi - + +elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then + + if [ "${force_tests}" == "true" ]; then + run_diff=true + fi + fi # if updating truth or running diff, run unit tests diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d99be08c6..2f6746c378 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -30,6 +30,6 @@ See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors Select: **Reviewer(s)** Select: **Organization** level software support **Project** or **Repository** level development cycle **Project** Select: **Milestone** as the version that will include these changes -- [ ] After submitting the PR, select **Linked issues** with the original issue number. +- [ ] After submitting the PR, select **Development** with the original issue number. - [ ] After the PR is approved, merge your changes. If permissions do not allow this, request that the reviewer do the merge. - [ ] Close the linked issue and delete your feature or bugfix branch from GitHub. diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index ed97e44d80..497d3221bc 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -7,6 +7,8 @@ on: paths-ignore: - 'met/docs/**' + workflow_dispatch: + env: DOCKERHUB_REPO: dtcenter/met-dev diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 88332b4da0..550add3594 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -23,6 +23,13 @@ on: paths-ignore: - 'met/docs/**' + workflow_dispatch: + inputs: + force_tests: + description: 'Run the unit tests' + default: true + type: boolean + env: DOCKERHUB_REPO: dtcenter/met-dev @@ -39,6 +46,7 @@ jobs: run: .github/jobs/set_job_controls.sh env: commit_msg: ${{ github.event.head_commit.message }} + force_tests: ${{ github.event.inputs.force_tests }} outputs: run_compile: ${{ steps.job_status.outputs.run_compile }} diff --git a/README.md b/README.md index ee97ac3082..bb0b829264 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Model Evaluation Tools (MET) Repository ======================================= +[![Tests](https://github.com/DTCenter/MET/actions/workflows/testing.yml/badge.svg?event=push)](https://github.com/DTCenter/MET/actions/workflows/testing.yml) +[![Docs](https://img.shields.io/badge/Documentation-latest-brightgreen.svg)](https://met.readthedocs.io) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5565322.svg)](https://doi.org/10.5281/zenodo.5565322) This repository contains the source code for the Model Evaluation Tools package (met), unit test code (test), and scripts used to build and test the code (scripts). diff --git a/met/README b/met/README index 9ea770c2bf..82fb383f4d 100644 --- a/met/README +++ b/met/README @@ -188,7 +188,7 @@ sub-directory, or the MET Online Tutorial: - If all tools are enabled and the build is successful, the "/bin" directory (where is the prefix you specified on your configure - command line) will contain 36 executables: + command line) will contain the following executables: - ascii2nc - ensemble_stat - gen_ens_prod @@ -200,6 +200,7 @@ sub-directory, or the MET Online Tutorial: - grid_diag - gsid2mpr - gsidens2orank + - ioda2nc - lidar2nc - madis2nc - mode diff --git a/met/data/config/ConfigConstants b/met/data/config/ConfigConstants index 8cbbeef1e9..c28b13ec28 100644 --- a/met/data/config/ConfigConstants +++ b/met/data/config/ConfigConstants @@ -35,6 +35,13 @@ F_to_K(t) = C_to_K(F_to_C(t)); MM_to_IN(x) = x / 25.4; IN_to_MM(x) = x * 25.4; +M_to_FT(x) = x * 3.28084; +FT_to_M(x) = x / 3.28084; +M_to_KFT(x) = M_to_FT(x) / 1000.0; +KFT_to_M(x) = FT_to_M(x * 1000.0); +KM_to_KFT(x) = M_to_KFT(1000.0 * x); +KFT_to_KM(x) = KFT_to_M(x) / 1000.0; + PA_to_HPA(p) = p / 100.0; HPA_to_PA(p) = p * 100.0; PA_to_MB(p) = PA_to_HPA(p); diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index ae5ca26a40..ceb19d6b4c 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -58,6 +58,13 @@ The configuration file language supports the following data types: and 10 and "==1||==2" defines numbers exactly equal to 1 or 2. * Percentile Thresholds: + + * A threshold type (<, <=, ==, !=, >=, or >), followed by a percentile + type description (SFP, SOP, SCP, USP, CDP, or FBIAS), followed by a + numeric value, typically between 0 and 100. + + * Note that the two letter threshold type abbreviations (lt, le, eq, ne, + ge, gt) are not supported for percentile thresholds. * Thresholds may be defined as percentiles of the data being processed in several places: diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index e989913062..62293cceaa 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -332,7 +332,7 @@ The **prob_pct_thresh** entry is an array of thresholds which define the Nx2 pro .. code-block:: none - prob_cat_thresh = [ ==0.25 ]; + prob_pct_thresh = [ ==0.25 ]; __________________ diff --git a/met/docs/Users_Guide/point-stat.rst b/met/docs/Users_Guide/point-stat.rst index 8ee096b946..90b3946b55 100644 --- a/met/docs/Users_Guide/point-stat.rst +++ b/met/docs/Users_Guide/point-stat.rst @@ -851,13 +851,13 @@ The first set of header columns are common to all of the output files generated * - 113-115 - RMSFA, :raw-html:`
` RMSFA_BCL, :raw-html:`
` RMSFA_BCU - Root mean squared forecast anomaly (f-c) including bootstrap upper and lower confidence limits - * - 117-119 + * - 116-118 - RMSOA, :raw-html:`
` RMSOA_BCL, :raw-html:`
` RMSOA_BCU - Root mean squared observation anomaly (o-c) including bootstrap upper and lower confidence limits - * - 120-122 + * - 119-121 - ANOM_CORR_UNCNTR, :raw-html:`
` ANOM_CORR_UNCNTR_BCL, :raw-html:`
` ANOM_CORR_UNCNTR_BCU - The uncentered Anomaly Correlation excluding mean error including bootstrap upper and lower confidence limits - * - 123-125 + * - 122-124 - SI, :raw-html:`
` SI_BCL, :raw-html:`
` SI_BCU - Scatter Index including bootstrap upper and lower confidence limits diff --git a/met/docs/Users_Guide/release-notes.rst b/met/docs/Users_Guide/release-notes.rst index d5fa2e5ec7..51750f0390 100644 --- a/met/docs/Users_Guide/release-notes.rst +++ b/met/docs/Users_Guide/release-notes.rst @@ -5,6 +5,23 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. +MET Version 10.1.2 release notes (20220516) +------------------------------------------- + +* Bugfixes: + + * Update static arrays with hard-coded size (`METplus-Internal #14 `_). + * Fix TC-Gen to only count misses from requested initialization hours and lead times (`#2148 `_). + +MET Version 10.1.1 release notes (20220419) +------------------------------------------- + +* Bugfixes: + + * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). + * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). + * Fix support for int64 NetCDF variable types (`#2123 `_). + MET Version 10.1.0 release notes (20220314) ------------------------------------------- diff --git a/met/docs/conf.py b/met/docs/conf.py index 44bbb2f91f..94c93b66bd 100644 --- a/met/docs/conf.py +++ b/met/docs/conf.py @@ -20,11 +20,12 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Newman, K., J. Opatz, T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway' -version = '10.1.0' + +version = '10.1.2' verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-03-14' +release_date = f'{release_year}-05-16' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/met/src/basic/vx_cal/unix_to_mdyhms.cc b/met/src/basic/vx_cal/unix_to_mdyhms.cc index e7d2a11330..72fd5cfcf9 100644 --- a/met/src/basic/vx_cal/unix_to_mdyhms.cc +++ b/met/src/basic/vx_cal/unix_to_mdyhms.cc @@ -16,6 +16,7 @@ using namespace std; #include #include +#include #include #include @@ -94,6 +95,35 @@ const char * short_month_name[] = { }; +//////////////////////////////////////////////////////////////////////// +// Converts a month string to month (1 to 12). + +int month_name_to_m (const char *month_str) + +{ + +int month = -1; +int str_len = strlen(month_str); +char t_month_str[str_len + 1]; + +for (int idx=0; idxassign(c.text()); else s->clear(); - memcpy(FloatFormat, c.FloatFormat, sizeof(FloatFormat)); + int buf_size = sizeof(c.FloatFormat); + if (buf_size > concat_string_buf_size) buf_size = concat_string_buf_size; + + memcpy(FloatFormat, c.FloatFormat, buf_size); Precision = c.Precision; } @@ -337,9 +340,12 @@ if ( (k < 0) || (k > concat_string_max_precision) ) { if (Precision != k) { Precision = k; - memset(FloatFormat, 0, sizeof(FloatFormat)); + int buf_size = sizeof(FloatFormat); + if (buf_size > concat_string_buf_size) buf_size = concat_string_buf_size; + + memset(FloatFormat, 0, buf_size); - snprintf(FloatFormat, sizeof(FloatFormat), "%%.%df", Precision); + snprintf(FloatFormat, buf_size, "%%.%df", Precision); } return; diff --git a/met/src/basic/vx_log/concat_string.h b/met/src/basic/vx_log/concat_string.h index df41cbf78e..59dec5f4f6 100644 --- a/met/src/basic/vx_log/concat_string.h +++ b/met/src/basic/vx_log/concat_string.h @@ -47,6 +47,7 @@ static const int max_str_len = 512; static const int concat_string_default_precision = 5; static const int concat_string_max_precision = 12; +static const int concat_string_buf_size = concat_string_max_precision + 4; //////////////////////////////////////////////////////////////////////// @@ -73,7 +74,7 @@ class ConcatString { int Precision; - char FloatFormat[16]; + char FloatFormat[concat_string_buf_size]; std::string *s; diff --git a/met/src/basic/vx_util/ascii_table.cc b/met/src/basic/vx_util/ascii_table.cc index fccfede13b..343a5aad23 100644 --- a/met/src/basic/vx_util/ascii_table.cc +++ b/met/src/basic/vx_util/ascii_table.cc @@ -271,8 +271,12 @@ BadDataValue = a.BadDataValue; set_bad_data_str(a.BadDataStr); -memcpy(f_FloatFormat, a.f_FloatFormat, sizeof(f_FloatFormat)); -memcpy(g_FloatFormat, a.g_FloatFormat, sizeof(g_FloatFormat)); +int f_buf_size = sizeof(a.f_FloatFormat); +int g_buf_size = sizeof(a.g_FloatFormat); +if (f_buf_size > ascii_table_buf_size) f_buf_size = ascii_table_buf_size; +if (g_buf_size > ascii_table_buf_size) g_buf_size = ascii_table_buf_size; +memcpy(f_FloatFormat, a.f_FloatFormat, f_buf_size); +memcpy(g_FloatFormat, a.g_FloatFormat, g_buf_size); DoCommaString = a.DoCommaString; @@ -697,11 +701,15 @@ if ( (k < 0) || (k > ascii_table_max_precision) ) { Precision = k; -memset(f_FloatFormat, 0, sizeof(f_FloatFormat)); -memset(g_FloatFormat, 0, sizeof(g_FloatFormat)); +int f_buf_size = sizeof(f_FloatFormat); +int g_buf_size = sizeof(g_FloatFormat); +if (f_buf_size > ascii_table_buf_size) f_buf_size = ascii_table_buf_size; +if (g_buf_size > ascii_table_buf_size) g_buf_size = ascii_table_buf_size; +memset(f_FloatFormat, 0, f_buf_size); +memset(g_FloatFormat, 0, g_buf_size); -snprintf(f_FloatFormat, sizeof(f_FloatFormat), "%%.%df", Precision); -snprintf(g_FloatFormat, sizeof(g_FloatFormat), "%%.%dg", Precision); +snprintf(f_FloatFormat, f_buf_size, "%%.%df", Precision); +snprintf(g_FloatFormat, g_buf_size, "%%.%dg", Precision); return; diff --git a/met/src/basic/vx_util/ascii_table.h b/met/src/basic/vx_util/ascii_table.h index a0f32056fd..c13c67c904 100644 --- a/met/src/basic/vx_util/ascii_table.h +++ b/met/src/basic/vx_util/ascii_table.h @@ -112,6 +112,7 @@ static const bool default_fill_blank = false; static const int ascii_table_default_precision = 2; static const int ascii_table_max_precision = 12; +static const int ascii_table_buf_size = ascii_table_max_precision + 4; static const double ascii_table_default_bad_data_value = -9999.0; @@ -163,8 +164,8 @@ class AsciiTable { std::string BadDataStr; - char f_FloatFormat[16]; - char g_FloatFormat[16]; + char f_FloatFormat[ascii_table_buf_size]; + char g_FloatFormat[ascii_table_buf_size]; bool DoCommaString; // do comma string? diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 8f5feea095..d34e56a361 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -30,7 +30,6 @@ using namespace std; #include "GridTemplate.h" - //////////////////////////////////////////////////////////////////////// // // Utility functions operating on a DataPlane @@ -206,7 +205,7 @@ DataPlane smooth_field(const DataPlane &dp, //////////////////////////////////////////////////////////////////////// void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, - int width, const GridTemplateFactory::GridTemplates shape, + int width, GridTemplateFactory::GridTemplates shape, bool wrap_lon, SingleThresh t, const DataPlane *cmn, const DataPlane *csd, double vld_t) { GridPoint *gp = NULL; @@ -251,9 +250,9 @@ void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, } } -#pragma omp parallel default(none) \ - shared(mlog, dp, frac_dp, width, wrap_lon, t) \ - shared(use_climo, cmn, csd, vld_t, bad) \ +#pragma omp parallel default(none) \ + shared(mlog, dp, frac_dp, shape, width, wrap_lon, t) \ + shared(use_climo, cmn, csd, vld_t, bad) \ private(x, y, n_vld, n_thr, gp, v) { diff --git a/met/src/basic/vx_util/data_plane_util.h b/met/src/basic/vx_util/data_plane_util.h index 1b5788dddc..c4f0b07c16 100644 --- a/met/src/basic/vx_util/data_plane_util.h +++ b/met/src/basic/vx_util/data_plane_util.h @@ -51,7 +51,7 @@ extern DataPlane smooth_field(const DataPlane &dp, bool wrap_lon, double t, const GaussianInfo &gaussian); extern void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, - int width, const GridTemplateFactory::GridTemplates shape, + int width, GridTemplateFactory::GridTemplates shape, bool wrap_lon, SingleThresh t, const DataPlane *cmn, const DataPlane *csd, double vld_t); diff --git a/met/src/basic/vx_util/observation.h b/met/src/basic/vx_util/observation.h index f9230e2dfe..58d9e29285 100644 --- a/met/src/basic/vx_util/observation.h +++ b/met/src/basic/vx_util/observation.h @@ -22,6 +22,7 @@ #include #include "config.h" +#include "util_constants.h" #ifdef ENABLE_PYTHON #include "vx_python3_utils.h" @@ -223,7 +224,7 @@ class Observation { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", @@ -240,7 +241,7 @@ class Observation string start_time_string = _getTimeString(start_time); string end_time_string = _getTimeString(end_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%s-%s", diff --git a/met/src/basic/vx_util/util_constants.h b/met/src/basic/vx_util/util_constants.h index 34da0d9e91..453d7b8303 100644 --- a/met/src/basic/vx_util/util_constants.h +++ b/met/src/basic/vx_util/util_constants.h @@ -18,6 +18,9 @@ //////////////////////////////////////////////////////////////////////// // Released versions of MET + +static const char met_version_10_1_2[] = "V10.1.2"; +static const char met_version_10_1_1[] = "V10.1.1"; static const char met_version_10_1_0[] = "V10.1.0"; static const char met_version_10_0_0[] = "V10.0.0"; static const char met_version_9_1[] = "V9.1"; @@ -40,7 +43,7 @@ static const char met_version_1_1[] = "V1.1"; //////////////////////////////////////////////////////////////////////// -static const char * const met_version = met_version_10_1_0; +static const char * const met_version = met_version_10_1_2; static const char default_met_data_dir[] = "MET_BASE"; static const char txt_file_ext[] = ".txt"; static const char stat_file_ext[] = ".stat"; @@ -92,6 +95,7 @@ static const int boot_perc_flag = 1; //////////////////////////////////////////////////////////////////////// static const int max_line_len = 2048; +static const int tmp_buf_size = 512; static const double grib_earth_radius_km = 6371.20; static const int default_nc_compression = 0; static const int default_precision = 5; diff --git a/met/src/libcode/vx_afm/afm_keywords.h b/met/src/libcode/vx_afm/afm_keywords.h index 026f0ca0c5..70c141bdef 100644 --- a/met/src/libcode/vx_afm/afm_keywords.h +++ b/met/src/libcode/vx_afm/afm_keywords.h @@ -83,7 +83,13 @@ struct AfmKeywordInfo { //////////////////////////////////////////////////////////////////////// -static const AfmKeywordInfo kw_info[42] = { +static const int n_kw_infos = 42; + + +//////////////////////////////////////////////////////////////////////// + + +static const AfmKeywordInfo kw_info[n_kw_infos] = { { "StartFontMetrics", afm_keyword_StartFontMetrics }, { "EndFontMetrics", afm_keyword_EndFontMetrics }, @@ -134,12 +140,6 @@ static const AfmKeywordInfo kw_info[42] = { //////////////////////////////////////////////////////////////////////// -static const int n_kw_infos = 42; - - -//////////////////////////////////////////////////////////////////////// - - extern int is_afm_keyword (ConcatString text, AfmKeyword &); diff --git a/met/src/libcode/vx_analysis_util/stat_job.cc b/met/src/libcode/vx_analysis_util/stat_job.cc index af3c450c74..f82b0f6151 100644 --- a/met/src/libcode/vx_analysis_util/stat_job.cc +++ b/met/src/libcode/vx_analysis_util/stat_job.cc @@ -2046,9 +2046,9 @@ void STATAnalysisJob::setup_stat_file(int n_row, int n) { case stat_prc: write_prc_header_row (1, n, stat_at, 0, 0); break; case stat_eclv: write_eclv_header_row (1, n, stat_at, 0, 0); break; case stat_mpr: write_header_row (mpr_columns, n_mpr_columns, 1, stat_at, 0, 0); break; - case stat_nbrctc: write_header_row (nbrctc_columns, n_sl1l2_columns, 1, stat_at, 0, 0); break; - case stat_nbrcts: write_header_row (nbrcts_columns, n_sl1l2_columns, 1, stat_at, 0, 0); break; - case stat_nbrcnt: write_header_row (nbrcnt_columns, n_sl1l2_columns, 1, stat_at, 0, 0); break; + case stat_nbrctc: write_header_row (nbrctc_columns, n_nbrctc_columns, 1, stat_at, 0, 0); break; + case stat_nbrcts: write_header_row (nbrcts_columns, n_nbrcts_columns, 1, stat_at, 0, 0); break; + case stat_nbrcnt: write_header_row (nbrcnt_columns, n_nbrcnt_columns, 1, stat_at, 0, 0); break; case stat_grad: write_header_row (grad_columns, n_grad_columns, 1, stat_at, 0, 0); break; case stat_isc: write_header_row (isc_columns, n_isc_columns, 1, stat_at, 0, 0); break; case stat_wdir: write_header_row (job_wdir_columns, n_job_wdir_columns, 1, stat_at, 0, 0); break; diff --git a/met/src/libcode/vx_color/color.h b/met/src/libcode/vx_color/color.h index 746b3610bb..986287aa6c 100644 --- a/met/src/libcode/vx_color/color.h +++ b/met/src/libcode/vx_color/color.h @@ -200,6 +200,7 @@ extern ostream & operator<<(ostream &, const CtableEntry &); static const int ctable_alloc_inc = 30; +static const int fudge_size = 256; //////////////////////////////////////////////////////////////////////// @@ -226,7 +227,7 @@ class ColorTable { double Gamma; - unsigned char fudge[256]; + unsigned char fudge[fudge_size]; public: diff --git a/met/src/libcode/vx_color/color_table.cc b/met/src/libcode/vx_color/color_table.cc index ed6f66843b..46157de13f 100644 --- a/met/src/libcode/vx_color/color_table.cc +++ b/met/src/libcode/vx_color/color_table.cc @@ -332,7 +332,7 @@ Nalloc = 0; int j; -for (j=0; j<256; ++j) { +for (j=0; jtype == 5) { res_flag = r.gds->grid_type.stereographic.res_flag; } + // Rotated LatLon + else if(r.gds->type == 10) { + res_flag = r.gds->grid_type.rot_latlon_grid.res_flag; + } else { mlog << Error << "\nis_grid_relative() -> " << "Unsupported grid type value: " << r.gds->type diff --git a/met/src/libcode/vx_data2d_grib/grib_classes.cc b/met/src/libcode/vx_data2d_grib/grib_classes.cc index da1a5f0f76..d02445c6ef 100644 --- a/met/src/libcode/vx_data2d_grib/grib_classes.cc +++ b/met/src/libcode/vx_data2d_grib/grib_classes.cc @@ -1874,6 +1874,27 @@ else if ((h.type == 5)) file << " scan_flag: " << (int) h.grid_type.stereographic.scan_flag << "\n\n"; } +else if ((h.type == 10)) +{ + + file << " lat1: " << char3_to_int(h.grid_type.rot_latlon_grid.lat1) << "\n"; + file << " lon1: " << char3_to_int(h.grid_type.rot_latlon_grid.lon1) << "\n"; + + file << " res_flag: " << (int) h.grid_type.rot_latlon_grid.res_flag << "\n"; + + file << " lat2: " << char3_to_int(h.grid_type.rot_latlon_grid.lat2) << "\n"; + file << " lon2: " << char3_to_int(h.grid_type.rot_latlon_grid.lon2) << "\n"; + + file << " di: " << char2_to_int(h.grid_type.rot_latlon_grid.di) << "\n"; + file << " dj: " << char2_to_int(h.grid_type.rot_latlon_grid.dj) << "\n"; + + file << " scan_flag: " << (int) h.grid_type.rot_latlon_grid.scan_flag << "\n"; + + file << " lat_sp: " << char3_to_int(h.grid_type.rot_latlon_grid.lat_sp) << "\n"; + file << " lon_sp: " << char3_to_int(h.grid_type.rot_latlon_grid.lon_sp) << "\n"; + + file << " rotation: " << char4_to_dbl(h.grid_type.rot_latlon_grid.rotation) << "\n\n"; +} return ( file ); diff --git a/met/src/libcode/vx_data2d_grib/grib_classes.h b/met/src/libcode/vx_data2d_grib/grib_classes.h index 55b7046d65..3b0295cf45 100644 --- a/met/src/libcode/vx_data2d_grib/grib_classes.h +++ b/met/src/libcode/vx_data2d_grib/grib_classes.h @@ -162,7 +162,7 @@ struct Section1_Header { // PDS //////////////////////////////////////////////////////////////////////// -struct LatLon { // Latitude/Longitude Grid +struct LatLon { // Latitude/Longitude Grid unsigned char lat1[3]; // 11 - 13 unsigned char lon1[3]; // 14 - 16 @@ -179,6 +179,32 @@ struct LatLon { // Latitude/Longitude Grid unsigned char unused[14]; // 29 - 42 +}; + + // Reference: https://apps.ecmwf.int/codes/grib/format/grib1/grids/10 + +struct RotLatLon { // Rotated Latitude/Longitude Grid + + unsigned char lat1[3]; // 11 - 13 + unsigned char lon1[3]; // 14 - 16 + + unsigned char res_flag; // 17 + + unsigned char lat2[3]; // 18 - 20 + unsigned char lon2[3]; // 21 - 23 + + unsigned char di[2]; // 24 - 25 + unsigned char dj[2]; // 26 - 27 + + unsigned char scan_flag; // 28 + + unsigned char unused[4]; // 29 - 32 + + unsigned char lat_sp[3]; // 33 - 35 + unsigned char lon_sp[3]; // 36 - 38 + + unsigned char rotation[4]; // 39 - 42 + }; struct Mercator { // Mercator Grid @@ -282,7 +308,7 @@ struct Gaussian { union GridType { struct LatLon latlon_grid; // Latitude/Longitude Grid - // struct RotLatLon rot_latlon_grid; // Rotated Latitude/Longitude Grid + struct RotLatLon rot_latlon_grid; // Rotated Latitude/Longitude Grid struct Mercator mercator; // Mercator Grid struct LambertConf lambert_conf; // Lambert Conformal Secant Grid struct Stereographic stereographic; // Stereographic Grid diff --git a/met/src/libcode/vx_data2d_grib/grib_utils.cc b/met/src/libcode/vx_data2d_grib/grib_utils.cc index 6dc5b7c563..c1ef70f775 100644 --- a/met/src/libcode/vx_data2d_grib/grib_utils.cc +++ b/met/src/libcode/vx_data2d_grib/grib_utils.cc @@ -29,11 +29,13 @@ using namespace std; // grid types from the GDS section // -static const int latlon_type = 0; -static const int mercator_type = 1; -static const int lambert_type = 3; -static const int stereographic_type = 5; -static const int gaussian_type = 4; +static const int latlon_type = 0; +static const int mercator_type = 1; +static const int lambert_type = 3; +static const int gaussian_type = 4; +static const int stereographic_type = 5; +static const int rotated_latlon_type = 10; + //////////////////////////////////////////////////////////////////////// @@ -62,7 +64,7 @@ void gds_to_grid(const Section2_Header & gds, Grid & gr) LambertData lc_data; StereographicData st_data; LatLonData ll_data; -//RotatedLatLonData rll_data; +RotatedLatLonData rll_data; MercatorData mc_data; GaussianData g_data; @@ -84,11 +86,11 @@ if ( gds.type == latlon_type ) { gr.set(ll_data); -// else if ( gds.type == rotated_latlon_type ) { -// -// gds_to_rotated_latlon(gds, rll_data); -// -// gr.set(rll_data); +} else if ( gds.type == rotated_latlon_type ) { + + gds_to_rotated_latlon(gds, rll_data); + + gr.set(rll_data); } else if ( gds.type == mercator_type ) { @@ -143,6 +145,7 @@ void gds_to_order(const Section2_Header & gds, int & xdir, int & ydir, int & ord // Check GDS for the grid type. // The following Projection types are supported: // - Lat/Lon + // - Rotated Lat/Lon // - Mercator // - Lambert Conformal // - Polar Stereographic @@ -154,6 +157,10 @@ if ( gds.type == latlon_type ) { scan_flag_to_order(gds.grid_type.latlon_grid.scan_flag, xdir, ydir, order); +} else if ( gds.type == rotated_latlon_type ) { + + scan_flag_to_order(gds.grid_type.rot_latlon_grid.scan_flag, xdir, ydir, order); + } else if (gds.type == mercator_type ) { scan_flag_to_order(gds.grid_type.mercator.scan_flag, xdir, ydir, order); @@ -278,10 +285,88 @@ void gds_to_rotated_latlon(const Section2_Header & gds, RotatedLatLonData & data { -mlog << Error << "\ngds_to_rotated_latlon() -> " - << "Rotated Lat/Lon grids are not supported in GRIB version 1.\n\n"; +int xdir, ydir, order; +double d; + +scan_flag_to_order(gds.grid_type.rot_latlon_grid.scan_flag, xdir, ydir, order); + + // Store the grid name +data.name = rotated_latlon_proj_type; + + // + // Multiply longitude values by -1 since the NCAR code considers + // degrees west to be positive. + // + + // Latitude of the bottom left corner +data.rot_lat_ll = min(decode_lat_lon(gds.grid_type.rot_latlon_grid.lat1, 3), + decode_lat_lon(gds.grid_type.rot_latlon_grid.lat2, 3)); + + // Longitude of the bottom left corner +if ( xdir == 0 ) data.rot_lon_ll = -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon1, 3)); +else data.rot_lon_ll = -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon2, 3)); + + // Number of points in the Latitudinal (y) direction +data.Nlat = char2_to_int(gds.ny); + + // Number of points in the Longitudinal (x) direction +data.Nlon = char2_to_int(gds.nx); + + // Check for thinned lat/lon grids +if ( data.Nlon == 65535 ) { -exit ( 1 ); + mlog << Error << "\ngds_to_rotated_latlon() -> " + << "Thinned Lat/Lon grids are not supported for GRIB version 1.\n\n"; + + exit ( 1 ); + +} + + // Compute latitudinal increment from lat1 and lat2 +d = fabs(decode_lat_lon(gds.grid_type.rot_latlon_grid.lat1, 3) + - decode_lat_lon(gds.grid_type.rot_latlon_grid.lat2, 3)); + +data.delta_rot_lat = d/(data.Nlat); + + // If explicitly specified and non-zero, use it +if ( ! all_bits_set(gds.grid_type.rot_latlon_grid.dj, 2) ) { + + d = decode_lat_lon(gds.grid_type.rot_latlon_grid.dj, 2); + + if ( d > 0 ) data.delta_rot_lat = d; + +} + + // Compute longitudinal increment from lon1 and lon2 +d = fabs(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon1, 3) + - decode_lat_lon(gds.grid_type.rot_latlon_grid.lon2, 3)); + +data.delta_rot_lon = d/(data.Nlon); + + // If explicitly specified and non-zero, use it +if ( ! all_bits_set(gds.grid_type.rot_latlon_grid.di, 2) ) { + + d = decode_lat_lon(gds.grid_type.rot_latlon_grid.di, 2); + + if ( d > 0 ) data.delta_rot_lon = d; + +} + + // Location of (rotated) south pole +data.true_lat_south_pole = + decode_lat_lon(gds.grid_type.rot_latlon_grid.lat_sp, 3); + +data.true_lon_south_pole = + -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon_sp, 3)); + + // Auxiliary rotation +data.aux_rotation = char4_to_dbl(gds.grid_type.rot_latlon_grid.rotation); + +data.dump(); + + // + // done + // return; diff --git a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 0061d2fd00..65df4c7ef7 100644 --- a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -1100,7 +1100,7 @@ void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { data.true_lon_south_pole = s_lon; // - // auxilliary rotation around the rotated polar axis + // auxiliary rotation around the rotated polar axis // data.aux_rotation = (t[21])*angle_factor; diff --git a/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc b/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc index 8e9e68cf4d..3512c6aaed 100644 --- a/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc +++ b/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc @@ -309,7 +309,7 @@ data.rot_lon_ll *= -1.0; get_global_att(ncfile, string("true_lon_south_pole"), data.true_lon_south_pole); if ( !west_longitude_positive ) data.true_lon_south_pole *= -1.0; - // auxilliary rotation + // auxiliary rotation get_global_att(ncfile, string("aux_rotation"), data.aux_rotation); diff --git a/met/src/libcode/vx_data2d_nc_met/met_file.cc b/met/src/libcode/vx_data2d_nc_met/met_file.cc index 14a41b64d1..26aae78398 100644 --- a/met/src/libcode/vx_data2d_nc_met/met_file.cc +++ b/met/src/libcode/vx_data2d_nc_met/met_file.cc @@ -32,16 +32,16 @@ using namespace std; //////////////////////////////////////////////////////////////////////// -static const char x_dim_name [] = "lon"; -static const char y_dim_name [] = "lat"; +static const char x_dim_name [] = "lon"; +static const char y_dim_name [] = "lat"; -static const string valid_time_att_name = "valid_time"; -static const string init_time_att_name = "init_time"; -static const string valid_time_ut_att_name = "valid_time_ut"; -static const string init_time_ut_att_name = "init_time_ut"; -static const string accum_time_att_name = "accum_time_sec"; +static const string valid_time_att_name = "valid_time"; +static const string init_time_att_name = "init_time"; +static const string valid_time_ut_att_name = "valid_time_ut"; +static const string init_time_ut_att_name = "init_time_ut"; +static const string accum_time_att_name = "accum_time_sec"; -static const int max_met_args = 30; +static const int max_met_args = 30; //////////////////////////////////////////////////////////////////////// @@ -446,18 +446,10 @@ short s; float f; double d = bad_data_double; bool status; -float add_offset = 0.f; -float scale_factor = 1.f; double missing_value = get_var_missing_value(var); double fill_value = get_var_fill_value(var); -NcVarAtt *att_add_offset = get_nc_att(var, (string)"add_offset"); -NcVarAtt *att_scale_factor = get_nc_att(var, (string)"scale_factor"); -if (!IS_INVALID_NC_P(att_add_offset) && !IS_INVALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); -} -if (att_add_offset) delete att_add_offset; -if (att_scale_factor) delete att_scale_factor; +float add_offset = get_var_add_offset_value(var); +float scale_factor = get_var_scale_factor_value(var); status = false; switch ( GET_NC_TYPE_ID_P(var) ) { @@ -641,11 +633,13 @@ const int x_slot = x_slot_tmp; const int y_slot = y_slot_tmp; // - // get the bad data value + // get the attributes // double missing_value = get_var_missing_value(v); double fill_value = get_var_fill_value(v); +float add_offset = get_var_add_offset_value(v); +float scale_factor = get_var_scale_factor_value(v); // // set up the DataPlane object @@ -659,16 +653,6 @@ plane.set_size(Nx, Ny); // clock_t clock_time; double nc_time; - float add_offset = 0.f; - float scale_factor = 1.f; - NcVarAtt *att_add_offset = get_nc_att(v, (string)"add_offset"); - NcVarAtt *att_scale_factor = get_nc_att(v, (string)"scale_factor"); - if (!IS_INVALID_NC_P(att_add_offset) && !IS_INVALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); - } - if (att_add_offset) delete att_add_offset; - if (att_scale_factor) delete att_scale_factor; int type_id = GET_NC_TYPE_ID_P(v); long dim[dimCount], cur[dimCount]; diff --git a/met/src/libcode/vx_data2d_nc_pinterp/pinterp_file.cc b/met/src/libcode/vx_data2d_nc_pinterp/pinterp_file.cc index d4f8451006..878cacf4be 100644 --- a/met/src/libcode/vx_data2d_nc_pinterp/pinterp_file.cc +++ b/met/src/libcode/vx_data2d_nc_pinterp/pinterp_file.cc @@ -567,18 +567,10 @@ int i; short s; float f; double d = bad_data_double; -float add_offset = 0.f; -float scale_factor = 1.f; double missing_value = get_var_missing_value(var); double fill_value = get_var_fill_value(var); -NcVarAtt *att_add_offset = get_nc_att(var, (string)"add_offset"); -NcVarAtt *att_scale_factor = get_nc_att(var, (string)"scale_factor"); -if (!IS_INVALID_NC_P(att_add_offset) && !IS_INVALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); -} -if (att_add_offset) delete att_add_offset; -if (att_scale_factor) delete att_scale_factor; +float add_offset = get_var_add_offset_value(var); +float scale_factor = get_var_scale_factor_value(var); switch ( GET_NC_TYPE_ID_P(var) ) { @@ -790,16 +782,8 @@ float f[Ny]; long offsets[dim_count]; long lengths[dim_count]; -float add_offset = 0.f; -float scale_factor = 1.f; -NcVarAtt *att_add_offset = get_nc_att(v, (string)"add_offset"); -NcVarAtt *att_scale_factor = get_nc_att(v, (string)"scale_factor"); -if (!IS_INVALID_NC_P(att_add_offset) && !IS_INVALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); -} -if (att_add_offset) delete att_add_offset; -if (att_scale_factor) delete att_scale_factor; +float add_offset = get_var_add_offset_value(var->var); +float scale_factor = get_var_scale_factor_value(var->var); for (int k=0; k 1 ) { @@ -375,6 +369,7 @@ bool NcCfFile::open(const char * filepath) } } } + else ValidTime.add(0); //Initialize delete [] time_values; } @@ -853,19 +848,10 @@ double NcCfFile::getData(NcVar * var, const LongArray & a) const bool status = false; double d = bad_data_double; - float add_offset = 0.f; - float scale_factor = 1.f; - NcVarAtt *att_add_offset = get_nc_att(var, (string)"add_offset"); - NcVarAtt *att_scale_factor = get_nc_att(var, (string)"scale_factor"); - if (IS_VALID_NC_P(att_add_offset) && IS_VALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); - } - if (att_add_offset) delete att_add_offset; - if (att_scale_factor) delete att_scale_factor; - double missing_value = get_var_missing_value(var); double fill_value = get_var_fill_value(var); + float add_offset = get_var_add_offset_value(var); + float scale_factor = get_var_scale_factor_value(var); switch (GET_NC_TYPE_ID_P(var)) { @@ -1030,11 +1016,13 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const const int y_slot = y_slot_tmp; // - // get the bad data values + // get the attributes // double missing_value = get_var_missing_value(v); double fill_value = get_var_fill_value(v); + float add_offset = get_var_add_offset_value(v); + float scale_factor = get_var_scale_factor_value(v); // set up the DataPlane object @@ -1078,17 +1066,6 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const offsets[y_slot] = 0; lengths[y_slot] = ny; - float add_offset = 0.f; - float scale_factor = 1.f; - NcVarAtt *att_add_offset = get_nc_att(v, (string)"add_offset"); - NcVarAtt *att_scale_factor = get_nc_att(v, (string)"scale_factor"); - if (IS_VALID_NC_P(att_add_offset) && IS_VALID_NC_P(att_scale_factor)) { - add_offset = get_att_value_float(att_add_offset); - scale_factor = get_att_value_float(att_scale_factor); - } - if (att_add_offset) delete att_add_offset; - if (att_scale_factor) delete att_scale_factor; - int type_id = GET_NC_TYPE_ID_P(v); bool do_scale_factor = add_offset != 0.0 || scale_factor != 1.0; @@ -2430,10 +2407,12 @@ void NcCfFile::get_grid_mapping_rotated_latitude_longitude(const NcVar *grid_map data.name = rotated_latlon_proj_type; - // Derive south pole location from the north pole + // Derive south pole location from the north pole: + // - Reverse the sign of the latitude + // - Add 180 to the longitude and switch from degrees east to west data.true_lat_south_pole = -1.0 * get_att_value_double(grid_np_lat_att); double np_lon = rescale_lon(get_att_value_double(grid_np_lon_att)); - data.true_lon_south_pole = rescale_lon(-1.0 * (180.0 - fabs(np_lon))); + data.true_lon_south_pole = rescale_lon(-1.0 * (np_lon + 180.0)); // Copied from the LatLon data structure data.rot_lat_ll = ll_data.lat_ll; diff --git a/met/src/libcode/vx_nc_util/nc_utils.cc b/met/src/libcode/vx_nc_util/nc_utils.cc index 8d8e5b1087..85f624401d 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.cc +++ b/met/src/libcode/vx_nc_util/nc_utils.cc @@ -25,8 +25,6 @@ using namespace netCDF::exceptions; //////////////////////////////////////////////////////////////////////// -//////////////////////////////////////////////////////////////////////// - void patch_nc_name(string *var_name) { size_t offset; @@ -1008,6 +1006,30 @@ double get_var_fill_value(const NcVar *var) { //////////////////////////////////////////////////////////////////////// +float get_var_add_offset_value(const NcVar *var) { + float v; + + if(!get_var_att_float(var, add_offset_att_name, v)) { + v = 0.f; + } + + return(v); +} + +//////////////////////////////////////////////////////////////////////// + +float get_var_scale_factor_value(const NcVar *var) { + float v; + + if(!get_var_att_float(var, scale_factor_att_name, v)) { + v = 1.f; + } + + return(v); +} + +//////////////////////////////////////////////////////////////////////// + char get_char_val(NcFile * nc, const char * var_name, const int index) { NcVar var = get_var(nc, var_name); return (get_char_val(&var, index)); @@ -1533,6 +1555,7 @@ bool get_nc_data(NcVar *var, float *data) { // Note: missing data was checked here // int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if (NcType::nc_FLOAT == type_id) { var->getVar(data); } @@ -1554,20 +1577,33 @@ bool get_nc_data(NcVar *var, float *data) { cell_count *= get_dim_size(var, idx); } - float add_offset = 0.; - float scale_factor = 1.; + float add_offset = get_var_add_offset_value(var); + float scale_factor = get_var_scale_factor_value(var); int unpacked_count = 0; bool unsigned_value = has_unsigned_attribute(var); - NcVarAtt *att_add_offset = get_nc_att(var, string("add_offset")); - NcVarAtt *att_scale_factor = get_nc_att(var, string("scale_factor")); - NcVarAtt *att_fill_value = get_nc_att(var, string("_FillValue")); - if (IS_VALID_NC_P(att_add_offset)) add_offset = get_att_value_float(att_add_offset); - if (IS_VALID_NC_P(att_scale_factor)) scale_factor = get_att_value_float(att_scale_factor); + NcVarAtt *att_fill_value = get_nc_att(var, string("_FillValue")); mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int64", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -1720,15 +1756,13 @@ bool get_nc_data(NcVar *var, float *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; } - if(att_add_offset) delete att_add_offset; - if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } mlog << Debug(6) << method_name << "took " @@ -1821,6 +1855,7 @@ bool get_nc_data(NcVar *var, double *data) { // int unpacked_count = 0; int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if ((NcType::nc_DOUBLE == type_id) || (NcType::nc_FLOAT == type_id)){ var->getVar(data); } @@ -1830,23 +1865,32 @@ bool get_nc_data(NcVar *var, double *data) { cell_count *= get_dim_size(var, idx); } - double add_offset = 0.; - double scale_factor = 1.; + double add_offset = get_var_add_offset_value(var); + double scale_factor = get_var_scale_factor_value(var); bool unsigned_value = has_unsigned_attribute(var); - NcVarAtt *att_add_offset = get_nc_att(var, (string)"add_offset"); - NcVarAtt *att_scale_factor = get_nc_att(var, (string)"scale_factor"); NcVarAtt *att_fill_value = get_nc_att(var, (string)"_FillValue"); - if (IS_VALID_NC_P(att_add_offset)) { - add_offset = get_att_value_double(att_add_offset); - } - if (IS_VALID_NC_P(att_scale_factor)) { - scale_factor = get_att_value_double(att_scale_factor); - } mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int64", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -2001,15 +2045,14 @@ bool get_nc_data(NcVar *var, double *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; + } - if(att_add_offset) delete att_add_offset; - if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } return(return_status); } diff --git a/met/src/libcode/vx_nc_util/nc_utils.h b/met/src/libcode/vx_nc_util/nc_utils.h index 560ae06309..a9cb921aab 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.h +++ b/met/src/libcode/vx_nc_util/nc_utils.h @@ -131,6 +131,8 @@ static const char nc_att_met_point_nccf[] = "MET_point_NCCF"; static const string description_att_name = "description"; static const string fill_value_att_name = "_FillValue"; +static const string add_offset_att_name = "add_offset"; +static const string scale_factor_att_name = "scale_factor"; static const string level_att_name = "level"; static const string long_name_att_name = "long_name"; static const string missing_value_att_name = "missing_value"; @@ -218,6 +220,9 @@ extern bool get_var_level(const NcVar *, ConcatString &); extern double get_var_missing_value(const NcVar *); extern double get_var_fill_value(const NcVar *); +extern float get_var_add_offset_value(const NcVar *); +extern float get_var_scale_factor_value(const NcVar *); + extern bool args_ok(const LongArray &); extern char get_char_val(NcFile *, const char * var_name, const int index); diff --git a/met/src/libcode/vx_pxm/ppm.cc b/met/src/libcode/vx_pxm/ppm.cc index 0d69531877..3bdaeeb84b 100644 --- a/met/src/libcode/vx_pxm/ppm.cc +++ b/met/src/libcode/vx_pxm/ppm.cc @@ -752,7 +752,7 @@ if ( fabs(g - 1.0) < 1.0e-3 ) return; int j, k; int nxy; -unsigned char fudge[256]; +unsigned char fudge[fudge_size]; double t; double exponent; Color color; @@ -761,7 +761,7 @@ Color color; exponent = 1.0/g; -for (j=0; j<256; ++j) { +for (j=0; j max_decimal_places) ) { DecimalPlaces = k; -snprintf(double_format, sizeof(double_format), "%%.%df", DecimalPlaces); // example: "%.5f" +int buf_size = sizeof(double_format); +if (buf_size > filter_buf_size) buf_size = filter_buf_size; +snprintf(double_format, buf_size, "%%.%df", DecimalPlaces); // example: "%.5f" return; diff --git a/met/src/libcode/vx_render/ps_filter.h b/met/src/libcode/vx_render/ps_filter.h index 23b3100de4..00bacc0e25 100644 --- a/met/src/libcode/vx_render/ps_filter.h +++ b/met/src/libcode/vx_render/ps_filter.h @@ -44,6 +44,7 @@ static const int FlateEncode = 4; static const int max_filters = 10; +static const int filter_buf_size = 32; //////////////////////////////////////////////////////////////////////// @@ -60,7 +61,7 @@ class PSFilter { int DecimalPlaces; - char double_format[32]; + char double_format[filter_buf_size]; virtual void eat(unsigned char); diff --git a/met/src/libcode/vx_statistics/ens_stats.cc b/met/src/libcode/vx_statistics/ens_stats.cc index d0bcfde0bd..d7c0e2c088 100644 --- a/met/src/libcode/vx_statistics/ens_stats.cc +++ b/met/src/libcode/vx_statistics/ens_stats.cc @@ -257,9 +257,17 @@ void ECNTInfo::set(const PairDataEnsemble &pd) { } } - // Compute ensemble mean based statistics, if possible + // Check if the ensemble mean based statistics were + // already computed by Stat-Analysis + if(!is_bad_data(pd.me)) { + me = pd.me; + rmse = pd.rmse; + me_oerr = pd.me_oerr; + rmse_oerr = pd.rmse_oerr; + } + // If not, compute them from the pairs, if possible // HiRA stores the ensemble mean as bad data - if(!pd.mn_na.is_const(bad_data_double)) { + else if(!pd.mn_na.is_const(bad_data_double)) { // Compute ME and RMSE values fbar = obar = ffbar = oobar = fobar = 0.0; diff --git a/met/src/libcode/vx_statistics/pair_data_ensemble.cc b/met/src/libcode/vx_statistics/pair_data_ensemble.cc index 51b43d0c37..3c16d13033 100644 --- a/met/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/met/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -371,7 +371,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { for(j=0, n_vld=0, n_bel=0, n_tie=0; j i && !is_bad_data(e_na[j][i])) { // Increment the valid count n_vld++; diff --git a/met/src/libcode/vx_summary/summary_obs.h b/met/src/libcode/vx_summary/summary_obs.h index 67b2aa353f..284b67b388 100644 --- a/met/src/libcode/vx_summary/summary_obs.h +++ b/met/src/libcode/vx_summary/summary_obs.h @@ -19,6 +19,7 @@ #include "observation.h" #include "summary_calc.h" #include "time_summary_interval.h" +#include "util_constants.h" //////////////////////////////////////////////////////////////////////// @@ -135,7 +136,7 @@ class SummaryObs // Create the string - char string_buffer[20]; + char string_buffer[tmp_buf_size]; snprintf(string_buffer, sizeof(string_buffer), "%02d%02d%02d", hour, minute, second); @@ -222,7 +223,7 @@ class SummaryObs { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", @@ -239,11 +240,11 @@ class SummaryObs memset(&time_struct, 0, sizeof(time_struct)); time_struct.tm_year = atoi(time_string.substr(0, 4).c_str()) - 1900; - time_struct.tm_mon = atoi(time_string.substr(4, 2).c_str()) - 1; + time_struct.tm_mon = atoi(time_string.substr(4, 2).c_str()) - 1; time_struct.tm_mday = atoi(time_string.substr(6, 2).c_str()); time_struct.tm_hour = atoi(time_string.substr(9, 2).c_str()); - time_struct.tm_min = atoi(time_string.substr(11, 2).c_str()); - time_struct.tm_sec = atoi(time_string.substr(13, 2).c_str()); + time_struct.tm_min = atoi(time_string.substr(11, 2).c_str()); + time_struct.tm_sec = atoi(time_string.substr(13, 2).c_str()); return timegm(&time_struct); } diff --git a/met/src/libcode/vx_tc_util/pair_data_genesis.cc b/met/src/libcode/vx_tc_util/pair_data_genesis.cc index 2b3e5f07b7..b07185e4c1 100644 --- a/met/src/libcode/vx_tc_util/pair_data_genesis.cc +++ b/met/src/libcode/vx_tc_util/pair_data_genesis.cc @@ -260,7 +260,8 @@ void PairDataGenesis::add_fcst_gen(const GenesisInfo *fgi) { void PairDataGenesis::add_best_gen(const GenesisInfo *bgi, const int fcst_beg, const int fcst_end, const int init_add, const unixtime init_beg, const unixtime init_end, - const TimeArray &init_inc, const TimeArray &init_exc) { + const TimeArray &init_inc, const TimeArray &init_exc, + const NumArray &init_hour, const NumArray &lead) { if(!bgi) return; @@ -282,6 +283,11 @@ void PairDataGenesis::add_best_gen(const GenesisInfo *bgi, (init_exc.n() > 0 && init_exc.has(init_ut))) continue; + // Check if this initialization hour and lead time should be used + if((init_hour.n() > 0 && !init_hour.has(unix_to_sec_of_day(init_ut))) || + (lead.n() > 0 && !lead.has(nint(bgi->genesis_time() - init_ut)))) + continue; + // Check if this case already exists if(!has_case(bgi->storm_id(), init_ut, i_case)) { diff --git a/met/src/libcode/vx_tc_util/pair_data_genesis.h b/met/src/libcode/vx_tc_util/pair_data_genesis.h index 12949ec790..1725273751 100644 --- a/met/src/libcode/vx_tc_util/pair_data_genesis.h +++ b/met/src/libcode/vx_tc_util/pair_data_genesis.h @@ -129,7 +129,8 @@ class PairDataGenesis { void add_best_gen(const GenesisInfo *, const int, const int, const int, const unixtime, const unixtime, - const TimeArray &, const TimeArray &); + const TimeArray &, const TimeArray &, + const NumArray &, const NumArray &); void add_gen_pair(const GenesisInfo *, const GenesisInfo *); void set_gen_diff(int, const GenesisPairDiff &); diff --git a/met/src/libcode/vx_tc_util/track_pair_info.cc b/met/src/libcode/vx_tc_util/track_pair_info.cc index bc013b6821..b9fbfc19ce 100644 --- a/met/src/libcode/vx_tc_util/track_pair_info.cc +++ b/met/src/libcode/vx_tc_util/track_pair_info.cc @@ -404,7 +404,7 @@ void TrackPairInfo::add(const TCStatLine &l) { cs << cs_erase << deck[i] << "SPEED"; tp->set_speed(atof(l.get_item(cs.c_str()))); cs << cs_erase << deck[i] << "DEPTH"; - tp->set_eye(string_to_systemsdepth(l.get_item(cs.c_str()))); + tp->set_depth(string_to_systemsdepth(l.get_item(cs.c_str()))); tp->set_watch_warn(string_to_watchwarntype(l.get_item("WATCH_WARN"))); // Loop over the winds diff --git a/met/src/libcode/vx_time_series/compute_swinging_door.h b/met/src/libcode/vx_time_series/compute_swinging_door.h index 9c990fe231..052ce0a789 100644 --- a/met/src/libcode/vx_time_series/compute_swinging_door.h +++ b/met/src/libcode/vx_time_series/compute_swinging_door.h @@ -118,7 +118,7 @@ class SDObservation { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", diff --git a/met/src/tools/other/ascii2nc/aeronet_handler.cc b/met/src/tools/other/ascii2nc/aeronet_handler.cc index 0c9bf2ad7a..d11e3e60f1 100644 --- a/met/src/tools/other/ascii2nc/aeronet_handler.cc +++ b/met/src/tools/other/ascii2nc/aeronet_handler.cc @@ -29,7 +29,9 @@ using namespace std; #include "aeronet_handler.h" +static bool test_AOD_550 = false; static const char *AERONET_NA_STR = "N/A"; +static const char *AERONET_V3_STR = "AERONET Version 3"; const int AeronetHandler::NUM_HDR_COLS = 7; const int AeronetHandler::NUM_OBS_COLS = 45; @@ -46,6 +48,8 @@ const string elv_col1 = "Site_Elevation"; // "Site_Elevation(m)"; const string lat_col2 = "Latitude"; // "Latitude(degrees)" const string lon_col2 = "Longitude"; // "Longitude(degrees)" const string elv_col2 = "Elevation"; // "Elevation(meters)" +const string date_col = "Date"; // "Date(dd:mm:yyyy)" +const string month_col = "Month"; // "Month" const string AeronetHandler::HEADER_TYPE = ""; ///// @@ -67,6 +71,7 @@ const string WAVELENGTHS_PW_NAME = "Exact_Wavelengths_of_PW"; // Exact_Wavelen const string WAVELENGTHS_INPUT_AOD_NAME = "Exact_Wavelengths_for_Input_AOD"; // Exact_Wavelengths_for_Input_AOD(um) static int format_version; +const string SITE_MISSING = "site_missing"; const float AERONET_MISSING_VALUE = -999.; @@ -121,7 +126,7 @@ bool AeronetHandler::isFileType(LineDataFile &ascii_file) const string line = dl.get_line(); if (line.length() > 17) { line = line.substr(0, 17); - if (strcmp(line.c_str(), "AERONET Version 3") == 0) { + if (strcmp(line.c_str(), AERONET_V3_STR) == 0) { is_file_type = true; format_version = 3; return is_file_type; @@ -178,8 +183,9 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) if (format_version == 3) { use_var_id = true; - // Get the field information from the 7-th header line - ascii_file >> data_line; + // Skip lines to get the field information from 7-th header line + // (with site name at line 2) or the 6-th header line (without site name) + if(_stationId != SITE_MISSING) ascii_file >> data_line; ascii_file >> data_line; ascii_file >> data_line; } @@ -197,7 +203,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) int flag; int aod_var_id = bad_data_int; - int var_idx, sid_idx, elv_idx, lat_idx, lon_idx; + int var_idx, sid_idx, elv_idx, lat_idx, lon_idx, date_idx, month_idx; double height_from_header; string aot = "AOT"; //string angstrom = "Angstrom"; @@ -207,7 +213,8 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) IntArray header_var_index; StringArray header_var_names; - sid_idx = elv_idx = lat_idx = lon_idx = -1; + date_idx = 0; + sid_idx = elv_idx = lat_idx = lon_idx = month_idx = -1; for (int j = 0; j < hdr_tokens.n_elements(); j++) { @@ -249,6 +256,8 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) else if (0 == hdr_field.find(lat_col1) || 0 == hdr_field.find(lat_col2)) lat_idx = j; else if (0 == hdr_field.find(lon_col1) || 0 == hdr_field.find(lon_col2)) lon_idx = j; else if (0 == hdr_field.find(elv_col1) || 0 == hdr_field.find(elv_col2)) elv_idx = j; + else if (0 == strcmp(hdr_field.c_str(), month_col.c_str())) month_idx = j; + else if (0 == hdr_field.find(date_col)) date_idx = j; // Collect variable names and index var_name = make_var_name_from_header(hdr_field); @@ -263,7 +272,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) header_var_index.add(var_idx); header_var_names.add(var_name.c_str()); header_heights.add(height_from_header); - mlog << Debug(5) << method_name << "header_idx: " << j + mlog << Debug(7) << method_name << "header_idx: " << j << ", var_idx: " << var_idx << ", var: " << var_name << " from " << hdr_field << ", flag: " << flag << ", height: " << height_from_header << "\n"; } @@ -336,7 +345,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) << site_name_col << "\" from the input \"" << ascii_file.filename() << "\"\n\n"; } - else if (_stationId != data_line[sid_idx]) { + else if (_stationId != data_line[sid_idx] && _stationId != SITE_MISSING) { mlog << Error << "\nAeronetHandler::_readObservations() The header and data columns don't match." << " The station ID from data column (" << data_line[sid_idx] << ") at " << sid_idx << " is different from " << _stationId @@ -354,7 +363,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) if (elv_idx >= 0) _stationAlt = atof(data_line[elv_idx]); else _stationAlt = bad_data_float; - mlog << Debug(5) << "AeronetHandler::_readObservations() stationID: " + mlog << Debug(7) << "AeronetHandler::_readObservations() stationID: " << ((sid_idx < 0) ? _stationId : data_line[sid_idx]) << " from index " << sid_idx << " lat: " << _stationLat << " lon: " << _stationLon @@ -366,7 +375,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) // Pull the valid time from the data line // - time_t valid_time = _getValidTime(data_line); + time_t valid_time = _getValidTime(data_line, (0 <= month_idx ? month_idx : date_idx)); if (valid_time == 0) return false; @@ -385,7 +394,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) for (int k = 0; k < process_flag.n_elements(); k++) { if (process_flag[k] != 1) continue; - string hdr_field = hdr_tokens[k]; size_t found_aot = hdr_field.find(aot); //int found_angstrom = hdr_field.find(angstrom); @@ -396,15 +404,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) height = hdr_field.substr((found_aot + 4), hdr_field.size() - 1); } - //if (found_angstrom != string::npos) - //{ - // size_t found_dash = hdr_field.find("-"); - // if (found_dash != string::npos) - // { - // height = hdr_field.substr(0, found_dash); - // } - //} - if(strcmp(data_line[k], AERONET_NA_STR) == 0) continue; var_name = AOT_NAME; @@ -424,7 +423,7 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) } } - _addObservations(Observation(header_type, _stationId, + _addObservations(Observation(header_type, (sid_idx<0 ? _stationId : data_line[sid_idx]), valid_time, _stationLat, _stationLon, _stationAlt, @@ -435,20 +434,22 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) var_name)); } if (format_version == 3) { - if (!has_aod_at_550 && !is_eq(aod_at_440, bad_data_float) && !is_eq(aod_at_675, bad_data_float)) { +// if (!has_aod_at_550 && aod_at_440 >= 0 && aod_at_675 >= 0) { var_id = aod_var_id; var_name = AOD_NAME; double dheight = 550; double aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,dheight); - _addObservations(Observation(header_type, _stationId, valid_time, - _stationLat, _stationLon, _stationAlt, - na_str, var_id, bad_data_double, dheight, - aod_at_550, - var_name)); - mlog << Debug(7) << "AeronetHandler::_readObservations() AOD at 550: " - << aod_at_550 << "\t440: " << aod_at_440 - << "\t675: " << aod_at_675 << "\n"; - } + if (!is_eq(aod_at_550, bad_data_double)) { + _addObservations(Observation(header_type, (sid_idx<0 ? _stationId : data_line[sid_idx]), valid_time, + _stationLat, _stationLon, _stationAlt, + na_str, var_id, bad_data_double, dheight, + aod_at_550, + var_name)); + mlog << Debug(7) << "AeronetHandler::_readObservations() AOD at 550: " + << aod_at_550 << "\t440: " << aod_at_440 + << "\t675: " << aod_at_675 << "\n"; + } +// } } } // end while if (bad_line_count > 0) { @@ -458,15 +459,13 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) << ascii_file.filename() << "\".\n\n"; } - if (format_version == 3) { + if (format_version == 3 && test_AOD_550) { double aod_at_675, aod_at_440; double aod_at_550_expected, aod_at_550; - //double angstrom_675_440_expected, angstrom_675_440; aod_at_675 = 0.645283; aod_at_440 = 0.794593; aod_at_550_expected = 0.71286864; - //angstrom_675_440_expected = 0.486381371; aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550); if (! is_eq(aod_at_550, aod_at_550_expected)) mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: " @@ -484,7 +483,6 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) aod_at_675 = 0.669274; aod_at_440 = 0.83858; aod_at_550_expected = 0.745546058; - //angstrom_675_440_expected = 0.526983959; aod_at_550 = angstrom_power_interplation(aod_at_675,aod_at_440,675.,440.,550); if (! is_eq(aod_at_550, aod_at_550_expected)) mlog << Warning << "AeronetHandler::_readObservations() Check AOD at 550: " @@ -504,29 +502,44 @@ bool AeronetHandler::_readObservations(LineDataFile &ascii_file) //////////////////////////////////////////////////////////////////////// -time_t AeronetHandler::_getValidTime(const DataLine &data_line) const +time_t AeronetHandler::_getValidTime(const DataLine &data_line, int date_offset) const { // // Pull out the date information // - ConcatString date_string(data_line[0]); + string mday; + string mon; + string year; + int month = -1; + bool date_yyyymmm = false; + ConcatString date_string(data_line[date_offset]); StringArray dateTokens = date_string.split(":"); if (1 == dateTokens.n_elements()) { - mlog << Error << "\nAeronetHandler::_getValidTime -> " - << "Not supported date: \"" << date_string << "\".\n\n"; - return 0; + // Support "yyyy-MMM" + StringArray ymTokens = date_string.split("-"); + if (2 == ymTokens.n_elements()) { + year = ymTokens[0]; + month = month_name_to_m(ymTokens[1].c_str()); + if (0 < month && month <= 12) date_yyyymmm = true; + } + if (!date_yyyymmm) { + mlog << Error << "\nAeronetHandler::_getValidTime -> " + << "Not supported date: \"" << date_string << "\".\n\n"; + return 0; + } + } + else { + mday = dateTokens[0]; + mon = dateTokens[1]; + year = dateTokens[2]; } - - string mday = dateTokens[0]; - string mon = dateTokens[1]; - string year = dateTokens[2]; // // Pull out the time information // - ConcatString time_string(data_line[1]); + ConcatString time_string(data_line[date_offset+1]); StringArray timeTokens = time_string.split(":"); // @@ -537,9 +550,9 @@ time_t AeronetHandler::_getValidTime(const DataLine &data_line) const memset(&time_struct, 0, sizeof(time_struct)); time_struct.tm_year = atoi(year.c_str()) - 1900; - time_struct.tm_mon = atoi(mon.c_str()) - 1; - time_struct.tm_mday = atoi(mday.c_str()); - if (3 <= timeTokens.n_elements()) { + time_struct.tm_mon = ((0 < month) ? month : atoi(mon.c_str())) - 1; + time_struct.tm_mday = (0 < month) ? 1 : atoi(mday.c_str()); + if (3 <= timeTokens.n_elements() && !date_yyyymmm) { string hour = timeTokens[0]; string min = timeTokens[1]; string sec = timeTokens[2]; @@ -573,7 +586,8 @@ bool AeronetHandler::_readHeaderInfo(LineDataFile &ascii_file) if (format_version == 3) { _stationId = data_line[0]; - if (' ' == _stationId[0]) _stationId = _stationId.substr(1); + if (string::npos != _stationId.find(AERONET_V3_STR)) _stationId = SITE_MISSING; + else if (' ' == _stationId[0]) _stationId = _stationId.substr(1); mlog << Debug(5) << " _stationId: [" << _stationId << "]\n"; // read lat/lon from https://aeronet.gsfc.nasa.gov/aeronet_locations_v3.txt return true; @@ -770,7 +784,10 @@ string AeronetHandler::make_var_name_from_header(string hdr_field) { double angstrom_power_interplation(double value_1, double value_2, double level_1, double level_2, double target_level) { - double angstrom_log = -log10(value_1/value_2)/log10(level_1/level_2); - double angstrom_value = value_2 * pow((target_level/level_2),-angstrom_log); + double angstrom_value = bad_data_double; + if ((value_1*value_2) >=0 && (level_1*level_2) >=0) { + double angstrom_log = -log10(value_1/value_2)/log10(level_1/level_2); + angstrom_value = value_2 * pow((target_level/level_2),-angstrom_log); + } return angstrom_value; } diff --git a/met/src/tools/other/ascii2nc/aeronet_handler.h b/met/src/tools/other/ascii2nc/aeronet_handler.h index 9a04de1dc6..34a47029ce 100644 --- a/met/src/tools/other/ascii2nc/aeronet_handler.h +++ b/met/src/tools/other/ascii2nc/aeronet_handler.h @@ -103,7 +103,7 @@ class AeronetHandler : public FileHandler // Get the observation valid time from the given observation line - time_t _getValidTime(const DataLine &data_line) const; + time_t _getValidTime(const DataLine &data_line, int date_offset) const; // Read the observations from the given file and add them to the // _observations vector. diff --git a/met/src/tools/other/gsi_tools/rad_record.cc b/met/src/tools/other/gsi_tools/rad_record.cc index 4ce91f1202..a87b453853 100644 --- a/met/src/tools/other/gsi_tools/rad_record.cc +++ b/met/src/tools/other/gsi_tools/rad_record.cc @@ -404,23 +404,23 @@ if ( n_read < 72 ) { unsigned char * b = buf; -my_memcpy( &(R_params.isis), b, 20); -my_memcpy( &(R_params.dplat), b, 10); -my_memcpy( &(R_params.obstype), b, 10); +my_memcpy( &(R_params.isis), b, rad_isis_len); +my_memcpy( &(R_params.dplat), b, rad_dplat_len); +my_memcpy( &(R_params.obstype), b, rad_obstype_len); -my_memcpy( &(R_params.jiter), b, 4); -my_memcpy( &(R_params.nchanl), b, 4); -my_memcpy( &(R_params.npred), b, 4); -my_memcpy( &(R_params.idate), b, 4); -my_memcpy( &(R_params.ireal), b, 4); -my_memcpy( &(R_params.ipchan), b, 4); -my_memcpy( &(R_params.iextra), b, 4); -my_memcpy( &(R_params.jextra), b, 4); +my_memcpy( &(R_params.jiter), b, rad_int_len); +my_memcpy( &(R_params.nchanl), b, rad_int_len); +my_memcpy( &(R_params.npred), b, rad_int_len); +my_memcpy( &(R_params.idate), b, rad_int_len); +my_memcpy( &(R_params.ireal), b, rad_int_len); +my_memcpy( &(R_params.ipchan), b, rad_int_len); +my_memcpy( &(R_params.iextra), b, rad_int_len); +my_memcpy( &(R_params.jextra), b, rad_int_len); -my_memcpy( &(R_params.idiag), b, 4); -my_memcpy( &(R_params.angord), b, 4); -my_memcpy( &(R_params.iversion), b, 4); -my_memcpy( &(R_params.inewpc), b, 4); +my_memcpy( &(R_params.idiag), b, rad_int_len); +my_memcpy( &(R_params.angord), b, rad_int_len); +my_memcpy( &(R_params.iversion), b, rad_int_len); +my_memcpy( &(R_params.inewpc), b, rad_int_len); if ( SwapEndian ) { @@ -483,14 +483,14 @@ if ( n_read != 32 ) { unsigned char * b = buf; -my_memcpy( &(cp.freq), b, 4); -my_memcpy( &(cp.plo), b, 4); -my_memcpy( &(cp.wave), b, 4); -my_memcpy( &(cp.varch), b, 4); -my_memcpy( &(cp.tlap), b, 4); -my_memcpy( &(cp.iuse), b, 4); -my_memcpy( &(cp.nuchan), b, 4); -my_memcpy( &(cp.ich), b, 4); +my_memcpy( &(cp.freq), b, rad_int_len); +my_memcpy( &(cp.plo), b, rad_int_len); +my_memcpy( &(cp.wave), b, rad_int_len); +my_memcpy( &(cp.varch), b, rad_int_len); +my_memcpy( &(cp.tlap), b, rad_int_len); +my_memcpy( &(cp.iuse), b, rad_int_len); +my_memcpy( &(cp.nuchan), b, rad_int_len); +my_memcpy( &(cp.ich), b, rad_int_len); if ( SwapEndian ) { diff --git a/met/src/tools/other/gsi_tools/rad_record.h b/met/src/tools/other/gsi_tools/rad_record.h index c26d768514..f13ac496c8 100644 --- a/met/src/tools/other/gsi_tools/rad_record.h +++ b/met/src/tools/other/gsi_tools/rad_record.h @@ -22,14 +22,21 @@ #include "vx_cal.h" +//////////////////////////////////////////////////////////////////////// + +static const int rad_obstype_len = 10; +static const int rad_dplat_len = 10; +static const int rad_isis_len = 20; +static const int rad_int_len = 4; + //////////////////////////////////////////////////////////////////////// struct RadParams { - char obstype [11]; - char dplat [11]; - char isis [21]; + char obstype [rad_obstype_len+1]; + char dplat [rad_dplat_len + 1]; + char isis [rad_isis_len + 1]; int jiter; int nchanl; diff --git a/met/src/tools/other/ioda2nc/ioda2nc.cc b/met/src/tools/other/ioda2nc/ioda2nc.cc index cc36eed5fe..00244d20fb 100644 --- a/met/src/tools/other/ioda2nc/ioda2nc.cc +++ b/met/src/tools/other/ioda2nc/ioda2nc.cc @@ -379,7 +379,7 @@ void process_ioda_file(int i_pb) { bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0); hdr_typ[0] = 0; - + file_ut = beg_ut = end_ut = hdr_vld_ut = (unixtime) 0; // List the IODA file being processed @@ -403,11 +403,11 @@ void process_ioda_file(int i_pb) { min_msg_ut = max_msg_ut = (unixtime) 0; min_time_str[0] = 0; max_time_str[0] = 0; + modified_hdr_typ[0] = 0; // Set the file name for the IODA file file_name << ioda_files[i_pb]; - int nrecs = 0; //int nstring = 0; StringArray var_names, dim_names; diff --git a/met/src/tools/other/plot_point_obs/Makefile.am b/met/src/tools/other/plot_point_obs/Makefile.am index fffb055719..690bb9ab63 100644 --- a/met/src/tools/other/plot_point_obs/Makefile.am +++ b/met/src/tools/other/plot_point_obs/Makefile.am @@ -44,7 +44,7 @@ plot_point_obs_LDADD = -lvx_statistics \ -lvx_cal \ -lvx_log \ $(PYTHON_LIBS) \ - -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas + -lm -lz -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas EXTRA_DIST = plot_point_obs.h \ plot_point_obs_conf_info.h diff --git a/met/src/tools/other/shift_data_plane/shift_data_plane.cc b/met/src/tools/other/shift_data_plane/shift_data_plane.cc index d7e85612df..bf05003341 100644 --- a/met/src/tools/other/shift_data_plane/shift_data_plane.cc +++ b/met/src/tools/other/shift_data_plane/shift_data_plane.cc @@ -20,6 +20,7 @@ // Mod# Date Name Description // ---- ---- ---- ----------- // 000 11-12-14 Halley Gotway New +// 001 06-07-22 Halley Gotway MET #2173 Fix python embedding // //////////////////////////////////////////////////////////////////////// @@ -50,6 +51,10 @@ using namespace std; #include "vx_cal.h" #include "vx_math.h" +#ifdef WITH_PYTHON +#include "data2d_python.h" +#endif + //////////////////////////////////////////////////////////////////////// static ConcatString program_name; @@ -200,17 +205,11 @@ void process_data_file() { // Populate the VarInfo object using config vinfo->set_dict(config); - // Open the input file - if(!mtddf->open(InputFilename.c_str())) { - mlog << Error << "\nprocess_data_file() -> can't open file \"" - << InputFilename << "\"\n\n"; - exit(1); - } - // Get the data plane from the file for this VarInfo object if(!mtddf->data_plane(*vinfo, dp_in)) { - mlog << Error << "\nprocess_data_file() -> trouble getting field \"" - << FieldString << "\" from file \"" << InputFilename << "\"\n\n"; + mlog << Error << "\nprocess_data_file() -> " + << "trouble getting field \"" << FieldString + << "\" from file \"" << InputFilename << "\"\n\n"; exit(1); } @@ -253,7 +252,6 @@ void process_data_file() { mlog << Debug(2) << shift_cs << "\n"; // Shift the data - dp_shift = dp_in; for(x=0; x