diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index fac8499842..11857bcd22 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -54,7 +54,7 @@ Describe the steps to reproduce the behavior: ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Bugfix Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md index f0ec5dbb89..0b82c047c5 100644 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -43,7 +43,7 @@ Consider breaking the enhancement down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Enhancement Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/new_feature_request.md b/.github/ISSUE_TEMPLATE/new_feature_request.md index de7d4f6820..4b1ae69aff 100644 --- a/.github/ISSUE_TEMPLATE/new_feature_request.md +++ b/.github/ISSUE_TEMPLATE/new_feature_request.md @@ -47,7 +47,7 @@ Consider breaking the new feature down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## New Feature Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/.github/ISSUE_TEMPLATE/task.md b/.github/ISSUE_TEMPLATE/task.md index 81c60136c9..561012a7f6 100644 --- a/.github/ISSUE_TEMPLATE/task.md +++ b/.github/ISSUE_TEMPLATE/task.md @@ -43,7 +43,7 @@ Consider breaking the task down into sub-issues. ## Define Related Issue(s) ## Consider the impact to the other METplus components. -- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdatadb](https://github.com/dtcenter/METdatadb/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) ## Task Checklist ## See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. diff --git a/data/table_files/Makefile.am b/data/table_files/Makefile.am index df5b863bd0..51e49ea11d 100644 --- a/data/table_files/Makefile.am +++ b/data/table_files/Makefile.am @@ -92,7 +92,8 @@ tablefiles_DATA = \ grib2_bom.txt \ grib2_gpp_af.txt \ grib2_mrms.txt \ - grib2_ndfd.txt + grib2_ndfd.txt \ + grib2_nbm.txt EXTRA_DIST = ${tablefiles_DATA} diff --git a/data/table_files/Makefile.in b/data/table_files/Makefile.in index ba9fcd8545..2201bc023f 100644 --- a/data/table_files/Makefile.in +++ b/data/table_files/Makefile.in @@ -368,7 +368,8 @@ tablefiles_DATA = \ grib2_bom.txt \ grib2_gpp_af.txt \ grib2_mrms.txt \ - grib2_ndfd.txt + grib2_ndfd.txt \ + grib2_nbm.txt EXTRA_DIST = ${tablefiles_DATA} MAINTAINERCLEANFILES = Makefile.in diff --git a/data/table_files/grib2_nbm.txt b/data/table_files/grib2_nbm.txt new file mode 100644 index 0000000000..48a232be6a --- /dev/null +++ b/data/table_files/grib2_nbm.txt @@ -0,0 +1,45 @@ +GRIB2 +0 1 0 255 7 1 0 21 "APTMP" "Apparent Temperature" "K" +0 1 0 255 7 1 7 6 "CAPE" "Convective Available Potential Energy" "J/kg" +0 1 0 255 7 1 19 239 "CWASP" "Craven-Wiedenfeld Aggregate Severe Parameter" "Numeric" +0 1 0 255 7 1 0 6 "DPT" "Dew Point Temperature" "K" +0 1 0 255 7 1 4 7 "DSWRF" "Downward Short-Wave Radiation Flux" "W/m^2" +0 1 0 255 7 1 19 238 "ELLINX" "Ellrod Index" "non-dim" +0 1 0 255 7 1 1 228 "FICEAC" "Flat Ice Accumulation (FRAM)" "kg/m^2" +10 1 0 255 7 1 3 204 "FRZSPR" "Freezing SprayFreezing Spray" "non-dim" +0 1 0 255 7 1 19 10 "TURB" "Turbulence" "non-dim" +2 1 0 255 7 1 4 4 "FOSINDX" "Fosberg Index" "Numeric" +2 1 0 255 7 1 4 2 "HINDEX" "Haines Index" "Numeric" +0 1 0 255 7 1 0 5 "TMIN" "Minimum Temperature" "K" +0 1 0 255 7 1 0 4 "TMAX" "Maximum Temperature" "K" +0 1 0 255 7 1 19 3 "MIXHT" "Mixed Layer Depth" "m" +0 1 0 255 7 1 0 27 "WETBT" "Wet Bulb Temperature" "K" +0 1 0 255 7 1 7 201 "BNEGELAY" "Bourgoiun Negative Energy Layer" "J/kg" +0 1 0 255 7 1 1 234 "PCPDUR" "Precipitation Duration" "hour" +0 1 0 255 7 1 1 8 "APCP" "Total Precipitation" "kg/m^2" +0 1 0 255 7 1 7 202 "BPOSELAY" "Bourgoiun Positive Energy Layer (2k ft AGL to 400 hPa)" "J/kg" +0 1 0 255 7 1 1 53 "TSRWE" "Total Snowfall Rate Water Equivalent" "prob" +0 1 0 255 7 1 1 226 "PWTHER" "Predominant Weather" "Numeric" +0 1 0 255 7 1 1 19 "PTYPE" "Precipitation Type" "prob" +0 1 0 255 7 1 1 232 "PROBCIP" "Probability Cloud Ice Present" "prob" +0 1 0 255 7 1 19 2 "TSTM" "Thunderstorm Probability" "%" +0 1 0 255 7 1 1 1 "RH" "Relative Humidity" "%" +10 1 0 255 7 1 0 3 "HTSGW" "Significant Height of Combined Wind Waves and Swell" "m" +0 1 0 255 7 1 6 1 "TCDC" "Total Cloud Cover" "%" +0 1 0 255 7 1 1 233 "SNOWLR" "Snow Liquid ratio" "kg/kg" +0 1 0 255 7 1 1 29 "ASNOW" "Total Snowfall" "kg/m^2" +0 1 0 255 7 1 19 236 "SNOWLVL" "Snow Level" "m" +0 1 0 255 7 1 0 0 "TMP" "Temperature" "K" +0 1 0 255 7 1 2 226 "TRWDIR" "Transport Wind Direction" "deg" +0 1 0 255 7 1 2 225 "TRWSPD" "Transport Wind Speed" "m/s" +0 1 0 255 7 1 2 224 "VRATE" "Ventilation Rate" "m^2/s" +0 1 0 255 7 1 2 0 "WDIR" "Wind Direction (from which blowing)" "deg" +0 1 0 255 7 1 2 22 "GUST" "Wind Speed (Gust)" "m/s" +0 1 0 255 7 1 2 1 "WIND" "Wind Speed" "m/s" +0 1 0 255 7 1 6 13 "CEIL" "Ceiling" "m" +0 1 0 255 7 1 19 237 "DRYTPROB" "Dry Thunderstorm Probability" "prob" +0 1 0 255 7 1 16 3 "RETOP" "Echo Top" "m" +0 1 0 255 7 1 3 6 "DIST" "Geometric Height" "m" +0 1 0 255 7 1 16 198 "MAXREF" "Hourly Maximum of Simulated Reflectivity at 1 km AGL" "dB" +0 1 0 255 7 1 15 3 "VIL" "Vertically-Integrated Liquid Water" "kg/m^2" +0 1 0 255 7 1 19 0 "VIS" "Visibility" "m" diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst index 5e4ae8fdbe..e93c0e18c0 100644 --- a/docs/Users_Guide/config_options.rst +++ b/docs/Users_Guide/config_options.rst @@ -974,6 +974,10 @@ File-format specific settings for the "field" entry: * The "GRIB2_stat_type" is an integer specifying the statistical processing type (Table 4.10). + + * The "GRIB2_perc_val" is an integer specifying the requested percentile + value (0 to 100) to be used. This applies only to GRIB2 product + definition templates 4.6 and 4.10. * The "GRIB2_ipdtmpl_index" and "GRIB2_ipdtmpl_val" entries are arrays of integers which specify the product description template values to diff --git a/docs/Users_Guide/plotting.rst b/docs/Users_Guide/plotting.rst index 172d99650b..09e373f0e1 100644 --- a/docs/Users_Guide/plotting.rst +++ b/docs/Users_Guide/plotting.rst @@ -178,7 +178,7 @@ ______________________ .. code-block:: none - dotsize(x) = 10; + dotsize(x) = 1.0; The **dotsize(x)** function defines the size of the circle to be plotted as a function of the observation value. The default setting shown above defines the dot size as a constant value. diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 94ad929d3a..c0b49b5e17 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -5,6 +5,26 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. +MET Version 11.0.0-beta2 release notes (20220809) +------------------------------------------------- + +* Bugfixes: + + * Fix Ensemble-Stat to work with different missing members for two or more variables (`#2208 `_). + +* Enhancements: + + * **Enhance MET's NetCDF library interface to support level strings that include coordinate variable values instead of just indexes** (`#1815 `_). + * **Enhance MTD to process time series with non-uniform time steps, such as monthly data** (`#1971 `_). + * Define the Bukovsky masking regions for use in MET (`#1940 `_). + * Update the GRIB2 MRMS table in MET (`#2081 `_). + * Add more error checking for python embedding of point observations (`#2202 `_). + * Add a sum option to the time summaries computed by the point pre-processing tools (`#2204 `_). + * Refine warning/error messages when parsing thresholds (`#2211 `_). + * Add "station_ob" to metadata_map as a message_type metadata variable for ioda2nc (`#2215 `_). + * MET: Add the user ID and the command line being executed to the log output at beginning and end of each MET tool run (`dtcenter/METplus-Internal#19 `_). + * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). + MET Version 11.0.0-beta1 release notes (20220622) ------------------------------------------------- @@ -31,3 +51,4 @@ MET Version 11.0.0-beta1 release notes (20220622) * Allow 2x2 HSS calculations to include user-defined EC values (`#2147 `_). * Enhance Gen-Vx-Mask by adding a new poly_xy masking type option (`#2152 `_). * Add M_to_KFT and KM_to_KFT functions to ConfigConstants (`#2180 `_). + * MET: Replace fixed length character arrays with strings (`dtcenter/METplus-Internal#14 `_). diff --git a/docs/conf.py b/docs/conf.py index 9ba9c47ec5..9f66d4970b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,11 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Opatz, J., T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway, K. Newman' -version = '11.0.0-beta1' +version = '11.0.0-beta2' verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-06-22' +release_date = f'{release_year}-08-09' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/internal/scripts/installation/config/install_met_env.gaea b/internal/scripts/installation/config/install_met_env.gaea new file mode 100644 index 0000000000..0e40bccd01 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.gaea @@ -0,0 +1,8 @@ +export TEST_BASE=/usw/met/10.1.2 +export COMPILER=intel_19.0.5.281 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=met-10.1.2.20220516.tar.gz +export USE_MODULES=TRUE +export MET_PYTHON=/lustre/f2/dev/esrl/Julie.Prestopnik/projects/miniconda/miniconda3/envs/metplus_v4.1_py3.6/ +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.6m +export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.6/config-3.6m-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.6m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm diff --git a/internal/scripts/installation/config/install_met_env.jet b/internal/scripts/installation/config/install_met_env.jet new file mode 100644 index 0000000000..9836dccc2d --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.jet @@ -0,0 +1,11 @@ +export TEST_BASE=/contrib/met/11.0.0 +export COMPILER=intel_18.0.5.274 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.0.tar.gz +export USE_MODULES=TRUE +export PYTHON_MODULE=intelpython_3.6.5 +export MET_PYTHON=/apps/intel/intelpython3 +export MET_PYTHON_CC=-I/apps/intel/intelpython3/include/python3.6m +export MET_PYTHON_LD=-L/apps/intel/intelpython3/lib\ -lpython3.6m\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -Xlinker\ -export-dynamic +export MET_NETCDF=/apps/netcdf/4.6.1/intel/18.0.5.274 +export MET_HDF5=/apps/hdf5/1.10.4/intel_seq/18.0.5.274 diff --git a/internal/scripts/installation/config/install_met_env.orion b/internal/scripts/installation/config/install_met_env.orion index dcad39eb26..56cbe4bfe3 100644 --- a/internal/scripts/installation/config/install_met_env.orion +++ b/internal/scripts/installation/config/install_met_env.orion @@ -1,14 +1,26 @@ module load intel/2020.2 module load intelpython3/2020.2 -export TEST_BASE=/apps/contrib/MET/10.1.0 +export TEST_BASE=/apps/contrib/MET/11.0.0 export COMPILER=intel_2020 export MET_SUBDIR=${TEST_BASE}/ -export MET_TARBALL=met-10.1.0.20220314.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE export MET_PYTHON=/apps/intel-2020.2/intel-2020.2/intelpython3 -export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm +#export EXTERNAL_LIBS=/apps/contrib/MET/11.0.0/external_libs +#export MET_GSL=${EXTERNAL_LIBS} +#export MET_BUFRLIB=${EXTERNAL_LIBS} +#export BUFRLIB_NAME=-lbufr +#export MET_HDF5=${EXTERNAL_LIBS} +#export MET_NETCDF=${EXTERNAL_LIBS} +#export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib +#export MET_GRIB2CINC=${EXTERNAL_LIBS}/include +#export GRIB2CLIB_NAME=-lgrib2c +#export LIB_JASPER=${EXTERNAL_LIBS}/lib +#export LIB_LIBPNG=${EXTERNAL_LIBS}/lib +#export LIB_Z=${EXTERNAL_LIBS}/lib export SET_D64BIT=FALSE -export CFLAGS="-Wall -g" -export CXXFLAGS="-Wall -g" +#export CFLAGS="-Wall -g" +#export CXXFLAGS="-Wall -g" diff --git a/internal/scripts/installation/config/install_met_env_all.cheyenne b/internal/scripts/installation/config/install_met_env_all.cheyenne index a7f39a0064..af9272845d 100644 --- a/internal/scripts/installation/config/install_met_env_all.cheyenne +++ b/internal/scripts/installation/config/install_met_env_all.cheyenne @@ -1,20 +1,18 @@ module load ncarenv/1.3 module load intel/2021.2 -module load python/3.7.9 module load netcdf/4.8.0 -ncar_pylib -export TEST_BASE=/glade/p/ral/jntp/MET/MET_releases/10.1.0 +export TEST_BASE=/glade/p/ral/jntp/MET/MET_releases/11.0.0 export COMPILER=intel_2021.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=met-10.1.0.20220314.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE -export MET_PYTHON=/glade/u/apps/ch/opt/python/3.7.9/gnu/9.1.0 -export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m -export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm +export MET_PYTHON=/glade/p/ral/jntp/MET/METplus/miniconda/miniconda3/envs/metplus_v5.0_py3.8 +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.8 +export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.8\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm export MET_NETCDF=/glade/u/apps/ch/opt/netcdf/4.8.0/intel/2021.2/ export MET_HDF5=/glade/u/apps/ch/opt/netcdf/4.8.0/intel/2021.2/ -#export EXTERNAL_LIBS=/glade/p/ral/jntp/MET/MET_releases/10.1.0/external_libs +#export EXTERNAL_LIBS=/glade/p/ral/jntp/MET/MET_releases/11.0.0/external_libs #export MET_GSL=${EXTERNAL_LIBS} #export MET_BUFRLIB=${EXTERNAL_LIBS} #export BUFRLIB_NAME=-lbufr @@ -25,6 +23,7 @@ export MET_HDF5=/glade/u/apps/ch/opt/netcdf/4.8.0/intel/2021.2/ #export LIB_JASPER=${EXTERNAL_LIBS}/lib #export LIB_LIBPNG=${EXTERNAL_LIBS}/lib #export LIB_Z=${EXTERNAL_LIBS}/lib -export SET_D64BIT=FALSE +#export SET_D64BIT=FALSE #export CFLAGS="-Wall -g" #export CXXFLAGS="-Wall -g" + diff --git a/internal/scripts/installation/config/install_met_env_all.hera b/internal/scripts/installation/config/install_met_env_all.hera index 7ac6267a46..b405dd1726 100644 --- a/internal/scripts/installation/config/install_met_env_all.hera +++ b/internal/scripts/installation/config/install_met_env_all.hera @@ -1,17 +1,13 @@ -module load intel/2022.1.2 impi/2022.1.2 -module load intel-tools -module load intelpython/3.6.5 - module use -a /contrib/anaconda/modulefiles module load intel/2022.1.2 -module load anaconda/latest -export TEST_BASE=/contrib/met/10.1.0 +module load anaconda/latest +export TEST_BASE=/contrib/met/11.0.0 export COMPILER=intel_2022.1.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=met-10.1.0.20220314.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE export PYTHON_MODULE=anaconda_latest -export MET_PYTHON=/contrib/anaconda/anaconda3/latest/ -export MET_PYTHON_CC=-I${MET_PYTHON}include/python3.7m +export MET_PYTHON=/contrib/anaconda/anaconda3/latest +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.7/config-3.7m-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -Xlinker\ -export-dynamic export SET_D64BIT=FALSE diff --git a/internal/scripts/installation/config/install_met_env_met_only.cheyenne b/internal/scripts/installation/config/install_met_env_met_only.cheyenne index 81de3a5406..318b3c3841 100644 --- a/internal/scripts/installation/config/install_met_env_met_only.cheyenne +++ b/internal/scripts/installation/config/install_met_env_met_only.cheyenne @@ -1,20 +1,18 @@ module load ncarenv/1.3 -module load intel/19.0.5 -module load python/3.7.9 -module load netcdf/4.7.4 -ncar_pylib +module load intel/2021.2 +module load netcdf/4.8.0 -export TEST_BASE=/glade/p/ral/jntp/MET/MET_releases/10.0.0 -export COMPILER=intel_19.0.5 +export TEST_BASE=/glade/p/ral/jntp/MET/MET_releases/11.0.0 +export COMPILER=intel_2021.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=met-10.0.0.20210510.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE -export MET_PYTHON=/glade/u/apps/ch/opt/python/3.7.9/gnu/9.1.0 -export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m -export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm -export MET_NETCDF=/glade/u/apps/ch/opt/netcdf/4.7.4/intel/19.0.5/ -export -EXTERNAL_LIBS=/glade/p/ral/jntp/MET/MET_releases/10.0.0/external_libs +export MET_PYTHON=/glade/p/ral/jntp/MET/METplus/miniconda/miniconda3/envs/metplus_v5.0_py3.8 +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.8 +export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.8\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm +export MET_NETCDF=/glade/u/apps/ch/opt/netcdf/4.8.0/intel/2021.2/ +export MET_HDF5=/glade/u/apps/ch/opt/netcdf/4.8.0/intel/2021.2/ +export EXTERNAL_LIBS=/glade/p/ral/jntp/MET/MET_releases/11.0.0/external_libs export MET_GSL=${EXTERNAL_LIBS} export MET_BUFRLIB=${EXTERNAL_LIBS} export BUFRLIB_NAME=-lbufr @@ -26,5 +24,6 @@ export LIB_JASPER=${EXTERNAL_LIBS}/lib export LIB_LIBPNG=${EXTERNAL_LIBS}/lib export LIB_Z=${EXTERNAL_LIBS}/lib export SET_D64BIT=FALSE -#export CFLAGS="-Wall -g" -#export CXXFLAGS="-Wall -g" +##export CFLAGS="-Wall -g" +##export CXXFLAGS="-Wall -g" + diff --git a/internal/scripts/installation/config/install_met_env_met_only.gaea b/internal/scripts/installation/config/install_met_env_met_only.gaea new file mode 100644 index 0000000000..478c7d0bae --- /dev/null +++ b/internal/scripts/installation/config/install_met_env_met_only.gaea @@ -0,0 +1,23 @@ +export TEST_BASE=/usw/met/10.1.2 +export COMPILER=intel_19.0.5.281 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=met-10.1.2.20220516.tar.gz +export USE_MODULES=TRUE +export MET_PYTHON=/lustre/f2/dev/esrl/Julie.Prestopnik/projects/miniconda/miniconda3/envs/metplus_v4.1_py3.6/ +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.6m +export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.6/config-3.6m-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.6m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lm +export EXTERNAL_LIBS=/usw/met/10.1.2/external_libs +export MET_NETCDF=${EXTERNAL_LIBS} +export MET_GSL=${EXTERNAL_LIBS} +export MET_BUFRLIB=${EXTERNAL_LIBS} +export BUFRLIB_NAME=-lbufr +export MET_HDF5=${EXTERNAL_LIBS} +export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib +export MET_GRIB2CINC=${EXTERNAL_LIBS}/include +export GRIB2CLIB_NAME=-lgrib2c +export LIB_JASPER=${EXTERNAL_LIBS}/lib +export LIB_LIBPNG=${EXTERNAL_LIBS}/lib +export LIB_Z=${EXTERNAL_LIBS}/lib +export SET_D64BIT=FALSE +#export CFLAGS="-Wall -g" +#export CXXFLAGS="-Wall -g -lcurl" diff --git a/internal/scripts/installation/config/install_met_env_met_only.hera b/internal/scripts/installation/config/install_met_env_met_only.hera index 591f42fa49..645ca9acab 100644 --- a/internal/scripts/installation/config/install_met_env_met_only.hera +++ b/internal/scripts/installation/config/install_met_env_met_only.hera @@ -1,20 +1,17 @@ -module load intel/2022.1.2 impi/2022.1.2 -module load intel-tools -module load intelpython/3.6.5 - module use -a /contrib/anaconda/modulefiles module load intel/2022.1.2 module load anaconda/latest -export TEST_BASE=/contrib/met/10.1.0 +export TEST_BASE=/contrib/met/11.0.0 export COMPILER=intel_2022.1.2 export MET_SUBDIR=${TEST_BASE} -export MET_TARBALL=met-10.1.0.20220314.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE export PYTHON_MODULE=anaconda_latest -export MET_PYTHON=/contrib/anaconda/anaconda3/latest/ -export MET_PYTHON_CC=-I${MET_PYTHON}include/python3.7m +export MET_PYTHON=/contrib/anaconda/anaconda3/latest +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m export MET_PYTHON_LD=-L${MET_PYTHON}/lib/python3.7/config-3.7m-x86_64-linux-gnu\ -L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -Xlinker\ -export-dynamic -export EXTERNAL_LIBS=/contrib/met/10.1.0/external_libs/ +export SET_D64BIT=FALSE +export EXTERNAL_LIBS=/contrib/met/11.0.0/external_libs/ export MET_NETCDF=${EXTERNAL_LIBS} export MET_GSL=${EXTERNAL_LIBS} export MET_BUFRLIB=${EXTERNAL_LIBS} @@ -29,4 +26,3 @@ export LIB_Z=${EXTERNAL_LIBS}/lib export SET_D64BIT=FALSE #export CFLAGS="-Wall -g" #export CXXFLAGS="-Wall -g -lcurl" - diff --git a/internal/scripts/installation/config/install_met_env_met_only.jet b/internal/scripts/installation/config/install_met_env_met_only.jet new file mode 100644 index 0000000000..cb339714f8 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env_met_only.jet @@ -0,0 +1,22 @@ +export TEST_BASE=/contrib/met/11.0.0 +export COMPILER=intel_18.0.5.274 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.0.tar.gz +export USE_MODULES=TRUE +export PYTHON_MODULE=intelpython_3.6.5 +export MET_PYTHON=/apps/intel/intelpython3 +export MET_PYTHON_CC=-I/apps/intel/intelpython3/include/python3.6m +export MET_PYTHON_LD=-L/apps/intel/intelpython3/lib\ -lpython3.6m\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm\ -Xlinker\ -export-dynamic +export MET_NETCDF=/apps/netcdf/4.6.1/intel/18.0.5.274 +export MET_HDF5=/apps/hdf5/1.10.4/intel_seq/18.0.5.274 +export EXTERNAL_LIBS=/contrib/met/11.0.0/external_libs/ +export MET_GSL=${EXTERNAL_LIBS} +export MET_BUFRLIB=${EXTERNAL_LIBS} +export BUFRLIB_NAME=-lbufr +export MET_GRIB2CLIB=${EXTERNAL_LIBS}/lib +export MET_GRIB2CINC=${EXTERNAL_LIBS}/include +export GRIB2CLIB_NAME=-lgrib2c +export LIB_JASPER=${EXTERNAL_LIBS}/lib +export LIB_LIBPNG=${EXTERNAL_LIBS}/lib +export LIB_Z=${EXTERNAL_LIBS}/lib +export SET_D64BIT=FALSE diff --git a/internal/scripts/installation/config/install_met_env_met_only.orion b/internal/scripts/installation/config/install_met_env_met_only.orion index a869ff8d27..5b9a8477d2 100644 --- a/internal/scripts/installation/config/install_met_env_met_only.orion +++ b/internal/scripts/installation/config/install_met_env_met_only.orion @@ -1,15 +1,15 @@ module load intel/2020.2 module load intelpython3/2020.2 -export TEST_BASE=/apps/contrib/MET/10.1.0 +export TEST_BASE=/apps/contrib/MET/11.0.0 export COMPILER=intel_2020 export MET_SUBDIR=${TEST_BASE}/ -export MET_TARBALL=met-10.1.0.20220314.tar.gz +export MET_TARBALL=v11.0.0.tar.gz export USE_MODULES=TRUE export MET_PYTHON=/apps/intel-2020.2/intel-2020.2/intelpython3 -export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m +export MET_PYTHON_CC=-I${MET_PYTHON}/include/python3.7m export MET_PYTHON_LD=-L${MET_PYTHON}/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm -export EXTERNAL_LIBS=/apps/contrib/MET/10.1.0/external_libs +export EXTERNAL_LIBS=/apps/contrib/MET/11.0.0/external_libs export MET_GSL=${EXTERNAL_LIBS} export MET_BUFRLIB=${EXTERNAL_LIBS} export BUFRLIB_NAME=-lbufr @@ -22,5 +22,5 @@ export LIB_JASPER=${EXTERNAL_LIBS}/lib export LIB_LIBPNG=${EXTERNAL_LIBS}/lib export LIB_Z=${EXTERNAL_LIBS}/lib export SET_D64BIT=FALSE -export CFLAGS="-Wall -g" -export CXXFLAGS="-Wall -g" +#export CFLAGS="-Wall -g" +#export CXXFLAGS="-Wall -g" diff --git a/internal/scripts/installation/modulefiles/10.1.2 b/internal/scripts/installation/modulefiles/10.1.2 new file mode 100644 index 0000000000..121150be4b --- /dev/null +++ b/internal/scripts/installation/modulefiles/10.1.2 @@ -0,0 +1,17 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v10.1.2 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +module load intel/19.0.5.281 + +set base /usw/met +set ver 10.1.2 +set share $base/$ver/share/met +set lib_base $base/10.1.2 + +prepend-path PATH $base/$ver/bin:$lib_base/external_libs/bin:/lustre/f2/dev/esrl/Julie.Prestopnik/projects/miniconda/miniconda3/envs/metplus_v4.1_py3.6/bin diff --git a/internal/scripts/installation/modulefiles/11.0.0_cheyenne b/internal/scripts/installation/modulefiles/11.0.0_cheyenne new file mode 100644 index 0000000000..ac7adc868b --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.0_cheyenne @@ -0,0 +1,24 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +# If they exist, remove ncdump and ncgen from /glade/p/ral/jntp/MET/MET_releases/11.0.0/external_libs/bin + +# The intel compiler is required to run MET +module load ncarenv/1.3 +module load intel/2021.2 +module load netcdf/4.8.0 + +set base /glade/p/ral/jntp/MET/MET_releases/11.0.0 +set ver 11.0.0 +set share $base/share/met + +prepend-path PATH $base/bin:/glade/p/ral/jntp/MET/MET_releases/11.0.0/external_libs/bin:/glade/p/ral/jntp/MET/METplus/miniconda/miniconda3/envs/metplus_v5.0_py3.8/bin + +setenv METversion V$ver + diff --git a/internal/scripts/installation/modulefiles/11.0.0_hera b/internal/scripts/installation/modulefiles/11.0.0_hera new file mode 100644 index 0000000000..905b76e82a --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.0_hera @@ -0,0 +1,22 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +# The intel compiler is required to run MET +prereq intel +prereq anaconda/latest + +set base /contrib/met +set ver 11.0.0 +set share $base/$ver/share/met +set lib_base $base/11.0.0 +setenv MET_ROOT $base/$ver/MET-11.0.0 + +prepend-path PATH $base/$ver/bin:$lib_base/external_libs/bin + + diff --git a/internal/scripts/installation/modulefiles/11.0.0_jet b/internal/scripts/installation/modulefiles/11.0.0_jet new file mode 100644 index 0000000000..356ae2bbac --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.0_jet @@ -0,0 +1,20 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +prereq intel +prereq intelpython/3.6.5 +prereq netcdf/4.6.1 +prereq hdf5/1.10.4 + +set base /contrib/met/11.0.0 +set ver 11.0.0 +set share $base/share/met +setenv MET_ROOT $base/$ver/MET-11.0.0 + +prepend-path PATH $base/bin:$base/external_libs/bin:/apps/intel/intelpython3/bin:/apps/netcdf/4.6.1/intel/18.0.5.274/bin:/apps/hdf5/1.10.4/intel_seq/18.0.5.274/bin \ No newline at end of file diff --git a/internal/scripts/installation/modulefiles/11.0.0_orion b/internal/scripts/installation/modulefiles/11.0.0_orion new file mode 100644 index 0000000000..cb2600d3f7 --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.0_orion @@ -0,0 +1,46 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.0 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +prereq intel/2020.2 +prereq intelpython3/2020.2 + +set base /apps/contrib/MET +set ver 11.0.0 +set share $base/$ver/share/met +set lib_base $base/11.0.0 +setenv MET_ROOT $base/$ver/MET-11.0.0 + +prepend-path PATH $base/$ver/bin:$lib_base/external_libs/bin + +#export CC=icc +#export CXX=icc +#export F77=ifort +#module load intel/2020.2 +#module load intelpython3/2020.2 +#export libdir=/apps/contrib/MET/11.0.0/external_libs/lib +#export incdir=/apps/contrib/MET/11.0.0/external_libs/include +#export iprefix=/apps/contrib/MET/11.0.0/external_libs +#export basedir=/apps/contrib/MET/11.0.0/MET-11.0.0 +#export MET_HDF5=$iprefix +#export MET_NETCDF=$incdir +#export MET_GRIB2CINC=$incdir +#export MET_GRIB2CLIB=$libdir +#setenv MET_GSLLIB=$libdir +#export MET_GSLINC=$incdir +#export MET_BUFR=$libdir +#export MET_HDFINC=$incdir +#export MET_HDFLIB=$libdir +#export MET_HDFEOSINC=$incdir +#export MET_HDFEOSLIB=$libdir +#export MET_PYTHON=/apps/intel-2020/intel-2020/intelpython3/ +#export MET_PYTHON_CC=-I/apps/intel-2020/intel-2020/intelpython3/include/python3.7m +#exoprt MET_PYTHON_LD=-L/apps/intel-2020/intel-2020/intelpython3/lib\ -lpython3.7m\ -lcrypt\ -lpthread\ -ldl\ -lutil\ -lrt\ -lm +#export MET_FONT_DIR=$basedir/fonts/ +#export LDFLAGS=-Wl,--disable-new-dtags -Wl,-rpath,${libdir}:${MET_PYTHON}/lib +#export CPPFLAGS=-I/apps/contrib/met/11.0.0/external_libs/include diff --git a/internal/test_unit/xml/unit_plot_data_plane.xml b/internal/test_unit/xml/unit_plot_data_plane.xml index f581160b33..3dc7c53f59 100644 --- a/internal/test_unit/xml/unit_plot_data_plane.xml +++ b/internal/test_unit/xml/unit_plot_data_plane.xml @@ -437,5 +437,75 @@ &OUTPUT_DIR;/plot_data_plane/CESM_b.e15.B1850.f09_g16.pi_control.all_ga7cpl24.66.cam.i.0083-01-01-00000.ps + + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/grib2/nbm/blend.t13z.core.f119.co.grib2 \ + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_L0.ps \ + 'name="CWASP"; level="L0"; GRIB2_pdt=2;' \ + -title "GRIB2 NBM Craven-Wiedenfeld Aggregate Severe Parameter at surface" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_L0.ps + + + + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/grib2/nbm/blend.t13z.core.f119.co.grib2 \ + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_Perc-5.ps \ + 'name="CWASP"; level="L0"; GRIB2_perc_val=5;' \ + -title "GRIB2 NBM Craven-Wiedenfeld Aggregate Severe Parameter 5% Level" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_Perc-5.ps + + + + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/grib2/nbm/blend.t13z.core.f119.co.grib2 \ + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_Prob-50.ps \ + 'name = "PROB"; level="L0"; prob = { name = "CWASP"; thresh_lo = 50; }' \ + -title "GRIB2 NBM Craven-Wiedenfeld Aggregate Severe Parameter Prob > 50" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_CWASP_Prob-50.ps + + + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/grib2/nbm/blend.t13z.core.f119.co.grib2 \ + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_WETBT_L610-40000.ps \ + 'name="WETBT"; level="L610-40000";' \ + -title "GRIB2 NBM Wet Bulb Temperature 610m above ground - 400mb" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_WETBT_L610-40000.ps + + + + + &MET_BIN;/plot_data_plane + \ + &DATA_DIR_MODEL;/grib2/nbm/blend.t13z.core.f119.co.grib2 \ + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_FICEAC_A48_Perc-10.ps \ + 'name="FICEAC"; level="A48"; GRIB2_perc_val=10;' \ + -title "GRIB2 NBM 48-hour Flat Ice Accumulation (FRAM) 10% Level" \ + -v 1 + + + &OUTPUT_DIR;/plot_data_plane/nbm_2022021513_F119_GRIB2_FICEAC_A48_Perc-10.ps + + + diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h index c51577ee4f..3cc2fe14e2 100644 --- a/src/basic/vx_config/config_constants.h +++ b/src/basic/vx_config/config_constants.h @@ -525,6 +525,7 @@ static const char conf_key_GRIB2_process[] = "GRIB2_process"; static const char conf_key_GRIB2_ens_type[] = "GRIB2_ens_type"; static const char conf_key_GRIB2_der_type[] = "GRIB2_der_type"; static const char conf_key_GRIB2_stat_type[] = "GRIB2_stat_type"; +static const char conf_key_GRIB2_perc_val[] = "GRIB2_perc_val"; static const char conf_key_GRIB2_ipdtmpl_index[] = "GRIB2_ipdtmpl_index"; static const char conf_key_GRIB2_ipdtmpl_val[] = "GRIB2_ipdtmpl_val"; static const char conf_key_level[] = "level"; diff --git a/src/basic/vx_config/my_config_scanner.cc b/src/basic/vx_config/my_config_scanner.cc index 72c7e06e72..d30d6a2058 100644 --- a/src/basic/vx_config/my_config_scanner.cc +++ b/src/basic/vx_config/my_config_scanner.cc @@ -1372,13 +1372,13 @@ if ( (pos2 = s.find('}', pos)) == string::npos ) { std::string out; std::string env; -char * env_value = 0; +char * tmp_env_value = 0; env = s.substr(pos1 + 2, pos2 - pos1 - 2); -env_value = getenv(env.c_str()); +tmp_env_value = getenv(env.c_str()); -if ( ! env_value ) { +if ( ! tmp_env_value ) { mlog << Error << "\nreplace_env() -> " << "unable to get value for environment variable \"" @@ -1390,7 +1390,7 @@ if ( ! env_value ) { out = s.substr(0, pos1); -out += env_value; +out += tmp_env_value; out += s.substr(pos2 + 1); diff --git a/src/basic/vx_log/string_array.cc b/src/basic/vx_log/string_array.cc index 95f110b7eb..56715eeca1 100644 --- a/src/basic/vx_log/string_array.cc +++ b/src/basic/vx_log/string_array.cc @@ -171,6 +171,7 @@ s = a.s; IgnoreCase = a.IgnoreCase; Sorted = a.Sorted; +MaxLength = a.MaxLength; return; diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 65df4c7ef7..42582b781e 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -432,7 +432,8 @@ void MetGrib2DataFile::find_record_matches( VarInfoGrib2* vinfo, (!is_bad_data(vinfo->process()) && vinfo->process() != (*it)->Process ) || (!is_bad_data(vinfo->ens_type()) && vinfo->ens_type() != (*it)->EnsType ) || (!is_bad_data(vinfo->der_type()) && vinfo->der_type() != (*it)->DerType ) || - (!is_bad_data(vinfo->stat_type()) && vinfo->stat_type() != (*it)->StatType ) ){ + (!is_bad_data(vinfo->stat_type()) && vinfo->stat_type() != (*it)->StatType ) || + (!is_bad_data(vinfo->perc_val()) && vinfo->perc_val() != (*it)->PercVal ) ){ continue; } @@ -523,10 +524,13 @@ void MetGrib2DataFile::find_record_matches( VarInfoGrib2* vinfo, } // END: if( level match ) // if seeking a probabilistic field, check the prob info - if( (rec_match_ex || rec_match_rn) && vinfo->p_flag() && (*it)->ProbFlag ){ + if( (rec_match_ex || rec_match_rn) && vinfo->p_flag() ) { rec_match_ex = rec_match_rn = false; - + + // no match unless the data contains probabilities + if( !(*it)->ProbFlag ) { continue; } + SingleThresh v_thr_lo = vinfo->p_thresh_lo(); SingleThresh v_thr_hi = vinfo->p_thresh_hi(); @@ -695,17 +699,24 @@ void MetGrib2DataFile::read_grib2_record_list() { 1 != gfld->ipdtnum && // individual ensemble forecast, control and perturbed, at a horizontal level or in a horizontal layer at a point in time 2 != gfld->ipdtnum && // ensemble mean 5 != gfld->ipdtnum && // probability forecast + 6 != gfld->ipdtnum && // percentile forecasts at a horizontal level or in a horizontal layer at a point in time 8 != gfld->ipdtnum && // accumulation forecast 9 != gfld->ipdtnum && // probabilistic accumulation forecast + 10 != gfld->ipdtnum && // percentile forecasts at a horizontal level or in a horizontal layer in a continuous or non-continuous time interval 11 != gfld->ipdtnum && // individual ensemble forecast, control and perturbed, at a horizontal level or in a horizontal layer, in a continuous or non-continuous time interval 12 != gfld->ipdtnum && // derived accumulation forecast (?) + 15 != gfld->ipdtnum && // Average, accumulation, extreme values or other statistically-processed values over a spatial area at a horizontal level or in a horizontal layer at a point in time 46 != gfld->ipdtnum && // average, accumulation, and/or extreme values or other statistically processed values at a horizontal level or in a horizontal layer in a continuous or non-continuous time interval for aerosol. 48 != gfld->ipdtnum ){ // analysis or forecast at a horizontal level or in a horizontal layer at a point in time for aerosol. - mlog << Error << "\nMetGrib2DataFile::data_plane() -> " + + // Print Warning, continue + mlog << Warning << "\nMetGrib2DataFile::data_plane() -> " << "PDS template number (" << gfld->ipdtnum << ") is not supported. " - << "Please create a new post with this information in the METplus GitHub Discussions forum at https://github.com/dtcenter/METplus/discussions\n\n"; - exit(1); + << "Please create a new post with this information in the METplus GitHub Discussions forum at https://github.com/dtcenter/METplus/discussions. " + << "Continuing to try and get the record information.\n\n"; + + continue; } // store the record information @@ -729,15 +740,16 @@ void MetGrib2DataFile::read_grib2_record_list() { } // store the full pdtmpl values - for(int j=0; j < gfld->ipdtlen; j++){ rec->IPDTmpl.add((int) gfld->ipdtmpl[j]); } - + for(int j=0; j < gfld->ipdtlen; j++) { + rec->IPDTmpl.add((int) gfld->ipdtmpl[j]); + } + // check for template number 46 if( gfld->ipdtnum == 46 ) { rec->LvlVal1 = scaled2dbl(gfld->ipdtmpl[16], gfld->ipdtmpl[17]); - rec->LvlVal2 = rec->LvlVal1; - - // check for special fixed level types (1 through 10 or 101) and set the level values to 0 - // Reference: https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_table4-5.shtml + rec->LvlVal2 = rec->LvlVal1; + // check for special fixed level types (1 through 10 or 101) and set the level values to 0 + // Reference: https://www.nco.ncep.noaa.gov/pmb/docs/grib2/grib2_doc/grib2_table4-5.shtml } else if( (rec->LvlTyp >= 1 && rec->LvlTyp <= 10) || rec->LvlTyp == 101 ) { rec->LvlVal1 = 0; rec->LvlVal2 = 0; @@ -790,6 +802,11 @@ void MetGrib2DataFile::read_grib2_record_list() { rec->StatType = gfld->ipdtmpl[23]; } + // percentile value for templates 6 and 10 + if( 6 == gfld->ipdtnum || 10 == gfld->ipdtnum ){ + rec->PercVal = gfld->ipdtmpl[15]; + } + // depending on the template number, determine the reference times if( 8 <= gfld->ipdtnum && 12 >= gfld->ipdtnum ){ diff --git a/src/libcode/vx_data2d_grib2/data2d_grib2.h b/src/libcode/vx_data2d_grib2/data2d_grib2.h index 0f371c758a..b4538cd11f 100644 --- a/src/libcode/vx_data2d_grib2/data2d_grib2.h +++ b/src/libcode/vx_data2d_grib2/data2d_grib2.h @@ -61,6 +61,7 @@ typedef struct { int EnsNumber; int DerType; int StatType; + int PercVal; IntArray IPDTmpl; } Grib2Record; diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.cc b/src/libcode/vx_data2d_grib2/var_info_grib2.cc index 6fc0d45759..a78eb38885 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.cc +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.cc @@ -108,6 +108,7 @@ void VarInfoGrib2::assign(const VarInfoGrib2 &v) { EnsType = v.EnsType; DerType = v.DerType; StatType = v.StatType; + PercVal = v.PercVal; IPDTmplIndex = v.IPDTmplIndex; IPDTmplVal = v.IPDTmplVal; @@ -134,6 +135,7 @@ void VarInfoGrib2::clear() { EnsType = bad_data_int; DerType = bad_data_int; StatType = bad_data_int; + PercVal = bad_data_int; IPDTmplIndex.clear(); IPDTmplVal.clear(); @@ -157,7 +159,8 @@ void VarInfoGrib2::dump(ostream &out) const { << " Process = " << Process << "\n" << " EnsType = " << EnsType << "\n" << " DerType = " << DerType << "\n" - << " StatType = " << StatType << "\n"; + << " StatType = " << StatType << "\n" + << " PercVal = " << PercVal << "\n"; out << " IPDTmplIndex:\n"; IPDTmplIndex.dump(out); out << " IPDTmplVal:\n"; @@ -245,6 +248,13 @@ void VarInfoGrib2::set_stat_type(int v) { /////////////////////////////////////////////////////////////////////////////// +void VarInfoGrib2::set_perc_val(int v) { + PercVal = v; + return; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoGrib2::set_ipdtmpl_index(const IntArray &v) { IPDTmplIndex = v; return; @@ -280,6 +290,7 @@ void VarInfoGrib2::set_dict(Dictionary & dict) { EnsType = dict.lookup_int (conf_key_GRIB2_ens_type, false); DerType = dict.lookup_int (conf_key_GRIB2_der_type, false); StatType = dict.lookup_int (conf_key_GRIB2_stat_type, false); + PercVal = dict.lookup_int (conf_key_GRIB2_perc_val, false); IPDTmplIndex = dict.lookup_int_array(conf_key_GRIB2_ipdtmpl_index, false); IPDTmplVal = dict.lookup_int_array(conf_key_GRIB2_ipdtmpl_val, false); diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.h b/src/libcode/vx_data2d_grib2/var_info_grib2.h index beca6c3b07..a6bf0fcd5e 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.h +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.h @@ -56,6 +56,7 @@ class VarInfoGrib2 : public VarInfo int EnsType; // Type of Ensemble Forecast (Table 4.6) int DerType; // Derived Forecast (Table 4.7) int StatType; // Statistical Processing Type (Table 4.10) + int PercVal; // Percentile Value (Octet 35 for Templates 4.6 and 4.10) IntArray IPDTmplIndex; // Index into the GRIB2 ipdtmpl array IntArray IPDTmplVal; // Corresponding GRIB2 ipdtmpl value @@ -88,6 +89,7 @@ class VarInfoGrib2 : public VarInfo int ens_type() const; int der_type() const; int stat_type() const; + int perc_val() const; int n_ipdtmpl() const; int ipdtmpl_index(int) const; @@ -110,6 +112,7 @@ class VarInfoGrib2 : public VarInfo void set_ens_type(int); void set_der_type(int); void set_stat_type(int); + void set_perc_val(int); void set_ipdtmpl_index(const IntArray &); void set_ipdtmpl_val(const IntArray &); @@ -143,6 +146,7 @@ inline int VarInfoGrib2::process() const { return(Process); } inline int VarInfoGrib2::ens_type() const { return(EnsType); } inline int VarInfoGrib2::der_type() const { return(DerType); } inline int VarInfoGrib2::stat_type() const { return(StatType); } +inline int VarInfoGrib2::perc_val() const { return(PercVal); } inline int VarInfoGrib2::n_ipdtmpl() const { return(IPDTmplIndex.n()); } inline int VarInfoGrib2::ipdtmpl_index(int i) const { diff --git a/src/libcode/vx_data2d_nc_met/Makefile.am b/src/libcode/vx_data2d_nc_met/Makefile.am index 3aedaf4c59..f557d252ff 100644 --- a/src/libcode/vx_data2d_nc_met/Makefile.am +++ b/src/libcode/vx_data2d_nc_met/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_data2d_nc_met.a libvx_data2d_nc_met_a_SOURCES = \ get_met_grid.cc get_met_grid.h \ - met_file.cc met_file.h \ + met_file.cc met_file.h met_file.hpp \ var_info_nc_met.cc var_info_nc_met.h \ data2d_nc_met.cc data2d_nc_met.h \ vx_data2d_nc_met.h diff --git a/src/libcode/vx_data2d_nc_met/Makefile.in b/src/libcode/vx_data2d_nc_met/Makefile.in index ebaa63426d..4c3c5965b5 100644 --- a/src/libcode/vx_data2d_nc_met/Makefile.in +++ b/src/libcode/vx_data2d_nc_met/Makefile.in @@ -341,7 +341,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_data2d_nc_met.a libvx_data2d_nc_met_a_SOURCES = \ get_met_grid.cc get_met_grid.h \ - met_file.cc met_file.h \ + met_file.cc met_file.h met_file.hpp \ var_info_nc_met.cc var_info_nc_met.h \ data2d_nc_met.cc data2d_nc_met.h \ vx_data2d_nc_met.h diff --git a/src/libcode/vx_data2d_nc_met/met_file.cc b/src/libcode/vx_data2d_nc_met/met_file.cc index 4aab77db9d..754a94b090 100644 --- a/src/libcode/vx_data2d_nc_met/met_file.cc +++ b/src/libcode/vx_data2d_nc_met/met_file.cc @@ -39,42 +39,6 @@ static const int max_met_args = 30; //////////////////////////////////////////////////////////////////////// -template - -void copy_nc_data_as_double(double *to_array, const T *from_array, - const int x_slot, const int y_slot, - const int nx, const int ny, - double missing_value, double fill_value) { - double value; - int x, y, offset, start_offset; - - offset = 0; - if (x_slot > y_slot) { - for (y=0; y + +void copy_nc_data_as_double(double *to_array, const T *from_array, + const int x_slot, const int y_slot, + const int nx, const int ny, + double missing_value, double fill_value) { + double value; + int x, y, offset, start_offset; + + offset = 0; + if (x_slot > y_slot) { + for (y=0; y= z_cnt)) { - // convert the value to index for slicing - z_offset = convert_value_to_offset(dim_offset, z_dim_name); - } if ((z_offset >= 0) && (z_offset < z_cnt)) dimension[idx] = long(z_offset); else { @@ -611,7 +607,7 @@ int MetNcCFDataFile::index(VarInfo &vinfo){ long MetNcCFDataFile::convert_time_to_offset(long time_value) { bool found = false; bool found_value = false; - long time_offset = time_value; + long time_offset = bad_data_int; int dim_size = _file->ValidTime.n(); static const string method_name = "MetNcCFDataFile::convert_time_to_offset() -> "; @@ -653,7 +649,7 @@ long MetNcCFDataFile::convert_time_to_offset(long time_value) { long MetNcCFDataFile::convert_value_to_offset(double z_value, string z_dim_name) { bool found = false; - long z_offset = (long)z_value; + long z_offset = bad_data_int; int dim_size = _file->vlevels.n(); static const string method_name = "MetNcCFDataFile::convert_value_to_offset() -> "; diff --git a/src/libcode/vx_data2d_nccf/nccf_file.cc b/src/libcode/vx_data2d_nccf/nccf_file.cc index 5235b1639d..0f45544ae3 100644 --- a/src/libcode/vx_data2d_nccf/nccf_file.cc +++ b/src/libcode/vx_data2d_nccf/nccf_file.cc @@ -893,8 +893,9 @@ double NcCfFile::getData(NcVar * var, const LongArray & a) const bool status = false; double d = bad_data_double; + double fill_value; double missing_value = get_var_missing_value(var); - double fill_value = get_var_fill_value(var); + get_var_fill_value(var, fill_value); status = get_nc_data(var, &d, (long *)a); @@ -1017,8 +1018,9 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const // get the bad data values // + double fill_value; double missing_value = get_var_missing_value(v); - double fill_value = get_var_fill_value(v); + get_var_fill_value(v, fill_value); // set up the DataPlane object diff --git a/src/libcode/vx_data2d_python/Makefile.am b/src/libcode/vx_data2d_python/Makefile.am index 185dd5d548..808e69025e 100644 --- a/src/libcode/vx_data2d_python/Makefile.am +++ b/src/libcode/vx_data2d_python/Makefile.am @@ -13,6 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_data2d_python.a libvx_data2d_python_a_SOURCES = \ dataplane_from_numpy_array.h dataplane_from_numpy_array.cc \ + dataplane_from_numpy_array.hpp \ dataplane_from_xarray.h dataplane_from_xarray.cc \ grid_from_python_dict.h grid_from_python_dict.cc \ python_dataplane.h python_dataplane.cc \ diff --git a/src/libcode/vx_data2d_python/Makefile.in b/src/libcode/vx_data2d_python/Makefile.in index ba38a293d1..f059f127f7 100644 --- a/src/libcode/vx_data2d_python/Makefile.in +++ b/src/libcode/vx_data2d_python/Makefile.in @@ -345,6 +345,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_data2d_python.a libvx_data2d_python_a_SOURCES = \ dataplane_from_numpy_array.h dataplane_from_numpy_array.cc \ + dataplane_from_numpy_array.hpp \ dataplane_from_xarray.h dataplane_from_xarray.cc \ grid_from_python_dict.h grid_from_python_dict.cc \ python_dataplane.h python_dataplane.cc \ diff --git a/src/libcode/vx_data2d_python/dataplane_from_numpy_array.cc b/src/libcode/vx_data2d_python/dataplane_from_numpy_array.cc index 8988bf2e75..f9e0bbfa06 100644 --- a/src/libcode/vx_data2d_python/dataplane_from_numpy_array.cc +++ b/src/libcode/vx_data2d_python/dataplane_from_numpy_array.cc @@ -43,49 +43,6 @@ return; //////////////////////////////////////////////////////////////////////// -template -void load_numpy (void * buf, - const int Nx, const int Ny, - const int data_endian, - void (*shuf)(void *), - DataPlane & out) - - -{ - -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j, x, y, r, c; -const int Nxy = Nx*Ny; -T * u = (T *) buf; -T value; - -for (j=0; j +void load_numpy (void * buf, + const int Nx, const int Ny, + const int data_endian, + void (*shuf)(void *), + DataPlane & out) + + +{ + +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j, x, y, r, c; +const int Nxy = Nx*Ny; +T * u = (T *) buf; +T value; + +for (j=0; j header_offset ? header_data.prpt_typ_array[header_offset] : bad_data_int); diff --git a/src/libcode/vx_nc_util/Makefile.am b/src/libcode/vx_nc_util/Makefile.am index 900ec9bd1e..a4e8499352 100644 --- a/src/libcode/vx_nc_util/Makefile.am +++ b/src/libcode/vx_nc_util/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h \ + nc_utils.cc nc_utils.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ diff --git a/src/libcode/vx_nc_util/Makefile.in b/src/libcode/vx_nc_util/Makefile.in index 086e099565..6966323260 100644 --- a/src/libcode/vx_nc_util/Makefile.in +++ b/src/libcode/vx_nc_util/Makefile.in @@ -341,7 +341,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h \ + nc_utils.cc nc_utils.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index f439407c1f..dcfa3a2ea3 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -56,32 +56,6 @@ bool get_att_value(const NcAtt *att, ConcatString &value) { //////////////////////////////////////////////////////////////////////// -template -bool get_att_num_value_(const NcAtt *att, T &att_val, int matching_type) { - bool status = false; - if (IS_VALID_NC_P(att)) { - int nc_type_id = GET_NC_TYPE_ID_P(att); - if (matching_type == nc_type_id) { - att->getValues(&att_val); - status = true; - } - else if (NC_CHAR == nc_type_id) { - string att_value; - att->getValues(att_value); - if (matching_type == NC_FLOAT) - att_val = atof(att_value.c_str()); - else if (matching_type == NC_DOUBLE) - att_val = (double)atof(att_value.c_str()); - else // if (matching_type == NC_INT) - att_val = atoi(att_value.c_str()); - status = true; - } - } - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_att_value(const NcAtt *att, ncbyte &att_val) { bool status = get_att_num_value_(att, att_val, NC_BYTE); return(status); @@ -489,35 +463,6 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// -template -bool get_nc_att_value_(const NcVar *var, const ConcatString &att_name, - T &att_val, bool exit_on_error, - T bad_data, const char *caller_name) { - bool status = false; - - // Initialize - att_val = bad_data; - - // - // Retrieve the NetCDF variable attribute. - // - NcVarAtt *att = get_nc_att(var, att_name); - status = get_att_value((NcAtt *)att, att_val); - if (!status) { - mlog << Error << "\n" << caller_name - << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); - if (exit_on_error) { - if (att) delete att; - exit(1); - } - } - if (att) delete att; - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, int &att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVar,int) -> "; @@ -567,29 +512,6 @@ bool get_nc_att_value(const NcVarAtt *att, ConcatString &att_val) { //////////////////////////////////////////////////////////////////////// -template -bool get_nc_att_value_(const NcVarAtt *att, T &att_val, bool exit_on_error, - T bad_data, const char *caller_name) { - bool status = true; - - // Initialize - att_val = bad_data; - - // - // Retrieve the NetCDF variable attribute. - // - status = get_att_value((NcAtt *)att, att_val); - if (!status) { - mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); - - if (exit_on_error) exit(1); - } - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_att_value(const NcVarAtt *att, int &att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVarAtt,int) -> "; bool status = get_nc_att_value_(att, att_val, exit_on_error, bad_data_int, method_name); @@ -767,37 +689,6 @@ bool get_global_att(const NcFile *nc, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// -template -bool get_global_att_value_(const NcFile *nc, const ConcatString& att_name, - T &att_val, T bad_data, bool error_out, const char *caller_name) { - bool status = false; - // Initialize - att_val = bad_data; - - NcGroupAtt *nc_att = get_nc_att(nc, att_name); - if (IS_VALID_NC_P(nc_att)) { - status = get_att_value((NcAtt *)nc_att, att_val); - string data_type = GET_NC_TYPE_NAME_P(nc_att); - if (error_out && !status) { - mlog << Error << caller_name - << "The data type \"" << data_type - << "\" for \"" << att_name << "\" does not match...\n\n"; - } - } - else if (error_out) { - mlog << Error << caller_name - << "can't find global NetCDF attribute \"" << att_name - << "\".\n\n"; - } - if (nc_att) delete nc_att; - // Check error_out status - if (error_out && !status) exit(1); - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_global_att(const NcFile *nc, const ConcatString& att_name, int &att_val, bool error_out) { @@ -968,27 +859,6 @@ int get_var_names(NcFile *nc, StringArray *varNames) { //////////////////////////////////////////////////////////////////////// -template -bool get_var_att_num_(const NcVar *var, const ConcatString &att_name, - T &att_val, T bad_data) { - bool status = false; - - // Initialize - att_val = bad_data; - - NcVarAtt *att = get_nc_att(var, att_name); - // Look for a match - if (IS_VALID_NC_P(att)) { - att->getValues(&att_val); - status = true; - } - if (att) delete att; - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_var_att_double(const NcVar *var, const ConcatString &att_name, double &att_val) { bool status = get_var_att_num_(var, att_name, att_val, bad_data_double); @@ -1025,36 +895,19 @@ bool get_var_axis(const NcVar *var, ConcatString &att_val) { //////////////////////////////////////////////////////////////////////// -template -bool get_var_fill_value(const NcVar *var, T &att_val) { - bool found = false; - - NcVarAtt *att = get_nc_att(var, fill_value_att_name); - if (IS_INVALID_NC_P(att)) { - if (att) delete att; - att = get_nc_att(var, missing_value_att_name); - } - if (IS_VALID_NC_P(att)) { - att->getValues(&att_val); - found = true; - } - - if (att) delete att; - - return(found); -} - -//////////////////////////////////////////////////////////////////////// - -double get_var_fill_value(const NcVar *var) { - double v; - - if(!get_var_att_double(var, fill_value_att_name, v)) { - v = bad_data_double; - } - - return(v); -} +void set_def_fill_value(ncbyte *val) { *val = bad_data_int/100; } +void set_def_fill_value(char *val) { *val = bad_data_char; } +void set_def_fill_value(double *val) { *val = bad_data_double; } +void set_def_fill_value(float *val) { *val = bad_data_float; } +void set_def_fill_value(int *val) { *val = bad_data_int; } +void set_def_fill_value(long *val) { *val = (long)bad_data_int; } +void set_def_fill_value(short *val) { *val = (short)bad_data_int; } +void set_def_fill_value(long long *val) { *val = (long long)bad_data_int; } +void set_def_fill_value(unsigned char *val) { *val = (unsigned char)-1; } +void set_def_fill_value(unsigned int *val) { *val = (unsigned int)-1; } +void set_def_fill_value(unsigned long *val) { *val = (unsigned long)-1; } +void set_def_fill_value(unsigned long long *val) { *val = (unsigned long long)-1; } +void set_def_fill_value(unsigned short *val) { *val = (unsigned short)-1; } //////////////////////////////////////////////////////////////////////// @@ -1208,7 +1061,7 @@ int get_int_var(NcVar * var, const int index) { int dim_idx = 0; int dim_size = get_dim_size(var, dim_idx); if (0 >= dim_size) { - if ((index > 0) && (0 >= dim_size)) { + if (index > 0) { mlog << Error << "\n" << method_name << "The start offset (" << index << ") should be 0 because of no dimension at the variable " << GET_NC_NAME_P(var) << ".\n\n"; @@ -1359,245 +1212,6 @@ float get_float_var(NcVar * var, const int index) { //////////////////////////////////////////////////////////////////////// -template -void apply_scale_factor_(T *data, const int cell_count, - double add_offset, double scale_factor, - const T nc_fill_value, const T met_fill_value, - bool has_fill_value, - const char *data_type, const char *var_name) { - const int debug_level = 7; - clock_t start_clock = clock(); - const char *method_name = "apply_scale_factor(T) "; - - if (cell_count > 0) { - int idx; - int positive_cnt = 0; - int unpacked_count = 0; - T min_value, max_value; - T raw_min_val, raw_max_val; - - idx = 0; - if (has_fill_value) { - for (; idx data[idx]) raw_min_val = data[idx]; - if (raw_max_val < data[idx]) raw_max_val = data[idx]; - data[idx] = (data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - //cout << typeid(nc_fill_value).name(); - mlog << Debug(debug_level) << method_name << var_name - << "(" << typeid(data[0]).name() << "): unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ", scale_factor=" << scale_factor<< " add_offset=" << add_offset - << ". FillValue(" << data_type << ")=" << nc_fill_value << "\n"; - mlog << Debug(debug_level) << method_name - << " data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - mlog << Debug(debug_level) << method_name << " took " - << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; - return; -} - -//////////////////////////////////////////////////////////////////////// -// Note: -// - template _t reads data as is (do not apply no scale_factor and add_offset) -// - template _ reads data and applies scale_factor and add_offset. - -template -bool get_nc_data_t(NcVar *var, T *data) { - bool return_status = false; - - if (IS_VALID_NC_P(var)) { - var->getVar(data); - - return_status = true; - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, const T met_missing) { - //const char *method_name = "get_nc_data_() "; - - int data_size = get_data_size(var); - for (int idx1=0; idx1", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *dims, const long *curs) { - bool return_status = false; - const char *method_name = "get_nc_data_(T, *dims, *curs) "; - - if (IS_VALID_NC_P(var)) { - std::vector start; - std::vector count; - - int data_size = 1; - int dimC = get_dim_count(var); - for (int idx = 0 ; idx < dimC; idx++) { - int dim_size = get_dim_size(var, idx); - if ((curs[idx]+dims[idx]) > dim_size) { - NcDim nc_dim = get_nc_dim(var, idx); - mlog << Error << "\n" << method_name << "The start offset and count (" - << curs[idx] << ", " << dims[idx] << ") exceeds the dimension[" - << idx << "] " << dim_size << " " - << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") - << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - - start.push_back((size_t)curs[idx]); - count.push_back((size_t)dims[idx]); - data_size *= dims[idx]; - } - - for (int idx1=0; idx1getVar(start, count, data); - return_status = true; - - //scale_factor and add_offset - if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { - T nc_missing; - double add_offset = get_var_add_offset(var); - double scale_factor = get_var_scale_factor(var); - bool has_missing_attr = get_var_fill_value(var, nc_missing); - if (!has_missing_attr) nc_missing = bad_data; - apply_scale_factor_(data, data_size, add_offset, scale_factor, - nc_missing, bad_data, has_missing_attr, - "", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, T met_missing, const long dim, const long cur) { - bool return_status = false; - const char *method_name = "get_nc_data_(T, dim, cur) "; - for (int idx=0; idx start; - std::vector count; - start.push_back((size_t)cur); - count.push_back((size_t)dim); - int dim_size = get_dim_size(var, dim_idx); - if (0 >= dim_size) { - if ((cur > 0) || (dim > 1)) { - mlog << Error << "\n" << method_name << "The start offset and count (" - << cur << ", " << dim << ") should be (0, 1) because of no dimension at the variable " - << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - } - else if (((cur+dim) > dim_size) && (0 < dim_size)) { - NcDim nc_dim = get_nc_dim(var, dim_idx); - mlog << Error << "\n" << method_name << "The start offset and count (" - << cur << " + " << dim << ") exceeds the dimension " << dim_size << " " - << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") - << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - - // - // Retrieve the variable value from the NetCDF variable. - // Note: missing data was checked here - // - var->getVar(start, count, data); - return_status = true; - - //scale_factor and add_offset - if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { - T nc_missing; - double add_offset = get_var_add_offset(var); - double scale_factor = get_var_scale_factor(var); - bool has_missing_attr = get_var_fill_value(var, nc_missing); - if (!has_missing_attr) nc_missing = met_missing; - apply_scale_factor_(data, dim, add_offset, scale_factor, - nc_missing, met_missing, has_missing_attr, - "", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// -// read a single data - -template -bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *curs) { - bool return_status = false; - const char *method_name = "get_nc_data_(*curs) "; - - if (IS_VALID_NC_P(var)) { - - int dimC = get_dim_count(var); - long dims[dimC]; - for (int idx = 0 ; idx < dimC; idx++) { - dims[idx] = 1; - } - - // Retrieve the NetCDF value from the NetCDF variable. - return_status = get_nc_data_(var, data, bad_data, dims, curs); - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, int *data, const long *curs) { bool return_status = get_nc_data_(var, data, bad_data_int, curs); @@ -1651,95 +1265,6 @@ bool get_nc_data(NcVar *var, short *data, const long *dims, const long *curs) { //////////////////////////////////////////////////////////////////////// -template -void copy_nc_data_t(NcVar *var, float *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor, bool has_missing, T missing_value) { - clock_t start_clock = clock(); - const char *method_name = "copy_nc_data_t(float) "; - - if (cell_count > 0) { - int idx; - float min_value, max_value; - bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); - - if (do_scale_factor) { - int positive_cnt = 0; - int unpacked_count = 0; - T raw_min_val, raw_max_val; - - for (idx=0; idx packed_data[idx]) raw_min_val = packed_data[idx]; - if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; - data[idx] = ((float)packed_data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - mlog << Debug(7) << method_name << GET_NC_NAME_P(var) - << " apply_scale_factor unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ". FillValue(" << data_type << ")=" << missing_value << "\n"; - mlog << Debug(7) << method_name - << "data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - else { - idx = 0; - - if (has_missing) { - for (idx=0; idx data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - } - } - mlog << Debug(7) << method_name << "data range [" << min_value - << " - " << max_value << "]\n"; - } - } - mlog << Debug(7) << method_name << "took " - << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; - return; -} - -//////////////////////////////////////////////////////////////////////// - -template -void copy_nc_data_(NcVar *var, float *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor) { - T missing_value; - bool has_missing = get_var_fill_value(var, missing_value); - copy_nc_data_t(var, data, packed_data, cell_count, data_type, - add_offset, scale_factor, has_missing, missing_value); - return; -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, float *data) { bool return_status = false; clock_t start_clock = clock(); @@ -1941,92 +1466,6 @@ bool get_nc_data(NcFile *nc, const char *var_name, double *data, //////////////////////////////////////////////////////////////////////// -template -void copy_nc_data_t(NcVar *var, double *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor, - bool has_missing, T missing_value) { - int unpacked_count = 0; - const char *method_name = "copy_nc_data_t(double) "; - - if (cell_count > 0) { - int idx; - T missing_value; - double min_value, max_value; - bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); - - if (do_scale_factor) { - int positive_cnt = 0; - T raw_min_val, raw_max_val; - - for (idx=0; idx packed_data[idx]) raw_min_val = packed_data[idx]; - if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; - data[idx] = ((double)packed_data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - mlog << Debug(7) << method_name << GET_NC_NAME_P(var) - << " apply_scale_factor unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ". FillValue(" << data_type << ")=" << missing_value - << " data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - else { - idx = 0; - - if (has_missing) { - for (idx=0; idx data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - } - } - mlog << Debug(7) << method_name << "data range [" << min_value - << " - " << max_value << "]\n"; - } - } -} - -//////////////////////////////////////////////////////////////////////// - -template -void copy_nc_data_(NcVar *var, double *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor) { - T missing_value; - bool has_missing = get_var_fill_value(var, missing_value); - copy_nc_data_t(var, data, packed_data, cell_count, data_type, - add_offset, scale_factor, has_missing, missing_value); - return; -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, double *data) { bool return_status = false; static const char *method_name = "get_nc_data(NcVar *, double *) "; @@ -2495,22 +1934,6 @@ int get_nc_string_length(NcFile *nc_file, NcVar var, const char *var_name) { //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T data, long offset0, long offset1, long offset2) { - vector offsets; - offsets.push_back((size_t)offset0); - if (0 <= offset1) { - offsets.push_back((size_t)offset1); - } - if (0 <= offset2) { - offsets.push_back((size_t)offset2); - } - var->putVar(offsets, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const int data, long offset0, long offset1, long offset2) { return put_nc_data_T(var, data, offset0, offset1, offset2); } @@ -2576,22 +1999,6 @@ bool put_nc_data(NcVar *var, const ncbyte *data ) { //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T *data, const long length, const long offset) { - vector offsets, counts; - int dim_count = get_dim_count(var); - offsets.push_back(offset); - if (dim_count >= 2) { - offsets.push_back(0); - counts.push_back(1); - } - counts.push_back(length); - var->putVar(offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const int *data, const long length, const long offset) { put_nc_data_T(var, data, length, offset); return true; @@ -2627,22 +2034,6 @@ bool put_nc_data(NcVar *var, const ncbyte *data, const long length, const long o //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T *data , const long *lengths, const long *offsets) { - int dim = get_dim_count(var); - vector nc_offsets, counts; - for (int idx = 0 ; idx < dim; idx++) { - nc_offsets.push_back(offsets[idx]); - } - for (int idx = 0 ; idx < dim; idx++) { - counts.push_back(lengths[idx]); - } - var->putVar(nc_offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const float *data , const long *lengths, const long *offsets) { put_nc_data_T(var, data , lengths, offsets); return true; @@ -2664,28 +2055,6 @@ bool put_nc_data(NcVar *var, const int *data , const long *lengths, const long * //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T_with_dims(NcVar *var, const T *data, - const long len0, const long len1, const long len2) { - vector offsets, counts; - if (0 < len0) { - offsets.push_back(0); - counts.push_back(len0); - } - if (0 < len1) { - offsets.push_back(0); - counts.push_back(len1); - } - if (0 < len2) { - offsets.push_back(0); - counts.push_back(len2); - } - var->putVar(offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data_with_dims(NcVar *var, const int *data, const int len0, const int len1, const int len2) { return put_nc_data_with_dims(var, data, (long)len0, (long)len1, (long)len2); diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 9db75b08ea..58fed1d8c3 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -226,7 +226,6 @@ extern bool get_var_att_double(const NcVar *, const ConcatString &, double &); template extern bool get_var_fill_value(const NcVar *var, T &att_val); extern bool get_var_axis(const NcVar *var, ConcatString &att_val); -extern double get_var_fill_value(const NcVar *); extern bool get_var_grid_mapping(const NcVar *var, ConcatString &att_val); extern bool get_var_grid_mapping_name(const NcVar *var, ConcatString &att_val); extern bool get_var_long_name(const NcVar *, ConcatString &); @@ -371,6 +370,10 @@ extern void parse_cf_time_string(const char *str, unixtime &ref_ut, //////////////////////////////////////////////////////////////////////// +#include "nc_utils.hpp" + +//////////////////////////////////////////////////////////////////////// + #endif /* __NC_UTILS_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp new file mode 100644 index 0000000000..f66b665a29 --- /dev/null +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -0,0 +1,710 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_UTILS_HPP__ +#define __NC_UTILS_HPP__ + +//////////////////////////////////////////////////////////////////////// + +extern bool get_att_value(const NcAtt *att, int &att_val); +extern bool get_att_value(const NcAtt *att, ConcatString &value); +extern bool get_att_value(const NcAtt *att, ncbyte &att_val); +extern bool get_att_value(const NcAtt *att, short &att_val); +extern bool get_att_value(const NcAtt *att, int &att_val); +extern bool get_att_value(const NcAtt *att, unsigned int &att_val); +extern bool get_att_value(const NcAtt *att, float &att_val); +extern bool get_att_value(const NcAtt *att, double &att_val); +extern ConcatString get_log_msg_for_att(const NcVarAtt *att); +extern ConcatString get_log_msg_for_att(const NcVarAtt *att, string var_name, + const ConcatString att_name); +extern double get_var_add_offset(const NcVar *var); +extern double get_var_scale_factor(const NcVar *var); +extern bool has_add_offset_attr(NcVar *var); +extern bool has_scale_factor_attr(NcVar *var); +extern void set_def_fill_value(ncbyte *val); +extern void set_def_fill_value(char *val); +extern void set_def_fill_value(double *val); +extern void set_def_fill_value(float *val); +extern void set_def_fill_value(int *val); +extern void set_def_fill_value(long *val); +extern void set_def_fill_value(short *val); +extern void set_def_fill_value(long long *val); +extern void set_def_fill_value(unsigned char *val); +extern void set_def_fill_value(unsigned int *val); +extern void set_def_fill_value(unsigned long *val); +extern void set_def_fill_value(unsigned long long *val); +extern void set_def_fill_value(unsigned short *val); + +//////////////////////////////////////////////////////////////////////// + +template +bool get_att_num_value_(const NcAtt *att, T &att_val, int matching_type) { + bool status = false; + if (IS_VALID_NC_P(att)) { + int nc_type_id = GET_NC_TYPE_ID_P(att); + if (matching_type == nc_type_id) { + att->getValues(&att_val); + status = true; + } + else if (NC_CHAR == nc_type_id) { + string att_value; + att->getValues(att_value); + if (matching_type == NC_FLOAT) + att_val = atof(att_value.c_str()); + else if (matching_type == NC_DOUBLE) + att_val = (double)atof(att_value.c_str()); + else // if (matching_type == NC_INT) + att_val = atoi(att_value.c_str()); + status = true; + } + } + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_value_(const NcVar *var, const ConcatString &att_name, + T &att_val, bool exit_on_error, + T bad_data, const char *caller_name) { + bool status = false; + + // Initialize + att_val = bad_data; + + // + // Retrieve the NetCDF variable attribute. + // + NcVarAtt *att = get_nc_att(var, att_name); + status = get_att_value((NcAtt *)att, att_val); + if (!status) { + mlog << Error << "\n" << caller_name + << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); + if (exit_on_error) { + if (att) delete att; + exit(1); + } + } + if (att) delete att; + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_value_(const NcVarAtt *att, T &att_val, bool exit_on_error, + T bad_data, const char *caller_name) { + bool status = true; + + // Initialize + att_val = bad_data; + + // + // Retrieve the NetCDF variable attribute. + // + status = get_att_value((NcAtt *)att, att_val); + if (!status) { + mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); + + if (exit_on_error) exit(1); + } + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_global_att_value_(const NcFile *nc, const ConcatString& att_name, + T &att_val, T bad_data, bool error_out, const char *caller_name) { + bool status = false; + // Initialize + att_val = bad_data; + + NcGroupAtt *nc_att = get_nc_att(nc, att_name); + if (IS_VALID_NC_P(nc_att)) { + status = get_att_value((NcAtt *)nc_att, att_val); + string data_type = GET_NC_TYPE_NAME_P(nc_att); + if (error_out && !status) { + mlog << Error << caller_name + << "The data type \"" << data_type + << "\" for \"" << att_name << "\" does not match...\n\n"; + } + } + else if (error_out) { + mlog << Error << caller_name + << "can't find global NetCDF attribute \"" << att_name + << "\".\n\n"; + } + if (nc_att) delete nc_att; + // Check error_out status + if (error_out && !status) exit(1); + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_var_att_num_(const NcVar *var, const ConcatString &att_name, + T &att_val, T bad_data) { + bool status = false; + + // Initialize + att_val = bad_data; + + NcVarAtt *att = get_nc_att(var, att_name); + // Look for a match + if (IS_VALID_NC_P(att)) { + att->getValues(&att_val); + status = true; + } + if (att) delete att; + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_var_fill_value(const NcVar *var, T &att_val) { + bool found = false; + + NcVarAtt *att = get_nc_att(var, fill_value_att_name); + if (IS_INVALID_NC_P(att)) { + if (att) delete att; + att = get_nc_att(var, missing_value_att_name); + } + if (IS_VALID_NC_P(att)) { + att->getValues(&att_val); + found = true; + } + else set_def_fill_value(&att_val); + + if (att) delete att; + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +template +void apply_scale_factor_(T *data, const int cell_count, + double add_offset, double scale_factor, + const T nc_fill_value, const T met_fill_value, + bool has_fill_value, + const char *data_type, const char *var_name) { + const int debug_level = 7; + clock_t start_clock = clock(); + const char *method_name = "apply_scale_factor(T) "; + + if (cell_count > 0) { + int idx; + int positive_cnt = 0; + int unpacked_count = 0; + T min_value, max_value; + T raw_min_val, raw_max_val; + + idx = 0; + if (has_fill_value) { + for (; idx data[idx]) raw_min_val = data[idx]; + if (raw_max_val < data[idx]) raw_max_val = data[idx]; + data[idx] = (data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + //cout << typeid(nc_fill_value).name(); + mlog << Debug(debug_level) << method_name << var_name + << "(" << typeid(data[0]).name() << "): unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ", scale_factor=" << scale_factor<< " add_offset=" << add_offset + << ". FillValue(" << data_type << ")=" << nc_fill_value << "\n"; + mlog << Debug(debug_level) << method_name + << " data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + mlog << Debug(debug_level) << method_name << " took " + << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; + return; +} + +//////////////////////////////////////////////////////////////////////// +// Note: +// - template _t reads data as is (do not apply no scale_factor and add_offset) +// - template _ reads data and applies scale_factor and add_offset. + +template +bool get_nc_data_t(NcVar *var, T *data) { + bool return_status = false; + + if (IS_VALID_NC_P(var)) { + var->getVar(data); + + return_status = true; + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, const T met_missing) { + //const char *method_name = "get_nc_data_() "; + + int data_size = get_data_size(var); + for (int idx1=0; idx1", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *dims, const long *curs) { + bool return_status = false; + const char *method_name = "get_nc_data_(T, *dims, *curs) "; + + if (IS_VALID_NC_P(var)) { + std::vector start; + std::vector count; + + int data_size = 1; + int dimC = get_dim_count(var); + for (int idx = 0 ; idx < dimC; idx++) { + int dim_size = get_dim_size(var, idx); + if ((curs[idx]+dims[idx]) > dim_size) { + NcDim nc_dim = get_nc_dim(var, idx); + mlog << Error << "\n" << method_name << "The start offset and count (" + << curs[idx] << ", " << dims[idx] << ") exceeds the dimension[" + << idx << "] " << dim_size << " " + << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") + << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + + start.push_back((size_t)curs[idx]); + count.push_back((size_t)dims[idx]); + data_size *= dims[idx]; + } + + for (int idx1=0; idx1getVar(start, count, data); + return_status = true; + + //scale_factor and add_offset + if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { + T nc_missing; + double add_offset = get_var_add_offset(var); + double scale_factor = get_var_scale_factor(var); + bool has_missing_attr = get_var_fill_value(var, nc_missing); + if (!has_missing_attr) nc_missing = bad_data; + apply_scale_factor_(data, data_size, add_offset, scale_factor, + nc_missing, bad_data, has_missing_attr, + "", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, T met_missing, const long dim, const long cur) { + bool return_status = false; + const char *method_name = "get_nc_data_(T, dim, cur) "; + for (int idx=0; idx start; + std::vector count; + start.push_back((size_t)cur); + count.push_back((size_t)dim); + int dim_size = get_dim_size(var, dim_idx); + if (0 >= dim_size) { + if ((cur > 0) || (dim > 1)) { + mlog << Error << "\n" << method_name << "The start offset and count (" + << cur << ", " << dim << ") should be (0, 1) because of no dimension at the variable " + << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + } + else if (((cur+dim) > dim_size) && (0 < dim_size)) { + NcDim nc_dim = get_nc_dim(var, dim_idx); + mlog << Error << "\n" << method_name << "The start offset and count (" + << cur << " + " << dim << ") exceeds the dimension " << dim_size << " " + << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") + << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + + // + // Retrieve the variable value from the NetCDF variable. + // Note: missing data was checked here + // + var->getVar(start, count, data); + return_status = true; + + //scale_factor and add_offset + if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { + T nc_missing; + double add_offset = get_var_add_offset(var); + double scale_factor = get_var_scale_factor(var); + bool has_missing_attr = get_var_fill_value(var, nc_missing); + if (!has_missing_attr) nc_missing = met_missing; + apply_scale_factor_(data, dim, add_offset, scale_factor, + nc_missing, met_missing, has_missing_attr, + "", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// +// read a single data + +template +bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *curs) { + bool return_status = false; + const char *method_name = "get_nc_data_(*curs) "; + + if (IS_VALID_NC_P(var)) { + + int dimC = get_dim_count(var); + long dims[dimC]; + for (int idx = 0 ; idx < dimC; idx++) { + dims[idx] = 1; + } + + // Retrieve the NetCDF value from the NetCDF variable. + return_status = get_nc_data_(var, data, bad_data, dims, curs); + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_t(NcVar *var, float *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor, + bool has_missing, T missing_value) { + clock_t start_clock = clock(); + const char *method_name = "copy_nc_data_t(float) "; + + if (cell_count > 0) { + int idx, first_idx; + float min_value, max_value; + bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); + + idx = first_idx = 0; + if (do_scale_factor) { + int positive_cnt = 0; + int unpacked_count = 0; + T raw_min_val, raw_max_val; + + if (has_missing) { + for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; + if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; + data[idx] = ((float)packed_data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + mlog << Debug(7) << method_name << GET_NC_NAME_P(var) + << " apply_scale_factor unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ". FillValue(" << data_type << ")=" << missing_value << "\n"; + mlog << Debug(7) << method_name + << "data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + else { + + if (has_missing) { + for (; idx data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + } + } + mlog << Debug(7) << method_name << "data range [" << min_value + << " - " << max_value << "]\n"; + } + } + mlog << Debug(7) << method_name << "took " + << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_(NcVar *var, float *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor) { + T missing_value; + bool has_missing = get_var_fill_value(var, missing_value); + copy_nc_data_t(var, data, packed_data, cell_count, data_type, + add_offset, scale_factor, has_missing, missing_value); + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_t(NcVar *var, double *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor, + bool has_missing, T missing_value) { + int unpacked_count = 0; + const char *method_name = "copy_nc_data_t(double) "; + + if (cell_count > 0) { + int idx, first_idx; + double min_value, max_value; + bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); + + idx = first_idx = 0; + if (do_scale_factor) { + int positive_cnt = 0; + T raw_min_val, raw_max_val; + + if (has_missing) { + // Skip missing values to find first min/max value + for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; + if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; + data[idx] = ((double)packed_data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + mlog << Debug(7) << method_name << GET_NC_NAME_P(var) + << " apply_scale_factor unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ". FillValue(" << data_type << ")=" << missing_value + << " data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + else { + if (has_missing) { + for (; idx data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + } + } + mlog << Debug(7) << method_name << "data range [" << min_value + << " - " << max_value << "]\n"; + } + } +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_(NcVar *var, double *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor) { + T missing_value; + bool has_missing = get_var_fill_value(var, missing_value); + copy_nc_data_t(var, data, packed_data, cell_count, data_type, + add_offset, scale_factor, has_missing, missing_value); + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T data, long offset0, long offset1, long offset2) { + vector offsets; + offsets.push_back((size_t)offset0); + if (0 <= offset1) { + offsets.push_back((size_t)offset1); + } + if (0 <= offset2) { + offsets.push_back((size_t)offset2); + } + var->putVar(offsets, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T *data, const long length, const long offset) { + vector offsets, counts; + int dim_count = get_dim_count(var); + offsets.push_back(offset); + if (dim_count >= 2) { + offsets.push_back(0); + counts.push_back(1); + } + counts.push_back(length); + var->putVar(offsets, counts, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T *data , const long *lengths, const long *offsets) { + int dim = get_dim_count(var); + vector nc_offsets, counts; + for (int idx = 0 ; idx < dim; idx++) { + nc_offsets.push_back(offsets[idx]); + } + for (int idx = 0 ; idx < dim; idx++) { + counts.push_back(lengths[idx]); + } + var->putVar(nc_offsets, counts, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T_with_dims(NcVar *var, const T *data, + const long len0, const long len1, const long len2) { + vector offsets, counts; + if (0 < len0) { + offsets.push_back(0); + counts.push_back(len0); + } + if (0 < len1) { + offsets.push_back(0); + counts.push_back(len1); + } + if (0 < len2) { + offsets.push_back(0); + counts.push_back(len2); + } + var->putVar(offsets, counts, data); + return true; +} + + +//////////////////////////////////////////////////////////////////////// + +#endif /* __NC_UTILS_HPP__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/Makefile.am b/src/libcode/vx_pointdata_python/Makefile.am index c48c07e2df..5579afc6fe 100644 --- a/src/libcode/vx_pointdata_python/Makefile.am +++ b/src/libcode/vx_pointdata_python/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_pointdata_python.a libvx_pointdata_python_a_SOURCES = \ pointdata_python.h pointdata_python.cc \ - pointdata_from_array.h pointdata_from_array.cc \ - python_pointdata.h python_pointdata.cc + pointdata_from_array.h pointdata_from_array.cc pointdata_from_array.hpp \ + python_pointdata.h python_pointdata.cc python_pointdata.hpp libvx_pointdata_python_a_CPPFLAGS = ${MET_CPPFLAGS} -I../vx_python2_utils ${MET_PYTHON_CC} $(MET_PYTHON_LD) diff --git a/src/libcode/vx_pointdata_python/Makefile.in b/src/libcode/vx_pointdata_python/Makefile.in index 8af9428f28..d45f6a82da 100644 --- a/src/libcode/vx_pointdata_python/Makefile.in +++ b/src/libcode/vx_pointdata_python/Makefile.in @@ -340,8 +340,8 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_pointdata_python.a libvx_pointdata_python_a_SOURCES = \ pointdata_python.h pointdata_python.cc \ - pointdata_from_array.h pointdata_from_array.cc \ - python_pointdata.h python_pointdata.cc + pointdata_from_array.h pointdata_from_array.cc pointdata_from_array.hpp \ + python_pointdata.h python_pointdata.cc python_pointdata.hpp libvx_pointdata_python_a_CPPFLAGS = ${MET_CPPFLAGS} -I../vx_python2_utils ${MET_PYTHON_CC} $(MET_PYTHON_LD) all: all-am diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.cc b/src/libcode/vx_pointdata_python/pointdata_from_array.cc index fc470878a2..ae196e40b1 100644 --- a/src/libcode/vx_pointdata_python/pointdata_from_array.cc +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.cc @@ -19,160 +19,6 @@ //////////////////////////////////////////////////////////////////////// -static const int api_debug_level = 11; - -//////////////////////////////////////////////////////////////////////// - - -template -void load_numpy (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - float * out) -{ - -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j, x, y, r, c; -T * u = (T *) buf; -T value; - -for (j=0; j -void load_numpy (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - int * out) -{ - -const char *method_name = "load_numpy(int *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -for (j=0; j -void load_numpy_int (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - IntArray *out) -{ - -const char *method_name = "load_numpy_int(IntArray *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -out->extend(n); - -for (j=0; jadd((int)value); - - mlog << Debug(api_debug_level) << method_name << " [" << j << "] value=" << value << "\n"; -} // for j - - - -return; - -} - - -//////////////////////////////////////////////////////////////////////// - - -template -void load_numpy_num (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - NumArray *out) -{ - -const char *method_name = "load_numpy_num(NumArray *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -out->extend(n); - -for (j=0; jadd((float)value); - - mlog << Debug(api_debug_level) << method_name << "[" << j << "] value=" << value << "\n"; -} // for j - - - -return; - -} - //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.h b/src/libcode/vx_pointdata_python/pointdata_from_array.h index 7538ef51bc..2e6bc12d21 100644 --- a/src/libcode/vx_pointdata_python/pointdata_from_array.h +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.h @@ -49,6 +49,9 @@ extern bool pointdata_from_python_list(PyObject *data_array, NumArray *data_out) //////////////////////////////////////////////////////////////////////// +#include "pointdata_from_array.hpp" + +//////////////////////////////////////////////////////////////////////// #endif /* __MET_POINTDATA_FROM_NUMPY_ARRAY_H__ */ diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.hpp b/src/libcode/vx_pointdata_python/pointdata_from_array.hpp new file mode 100644 index 0000000000..f9bf3cdb8c --- /dev/null +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.hpp @@ -0,0 +1,176 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MET_POINTDATA_FROM_ARRAY_HPP__ +#define __MET_POINTDATA_FROM_ARRAY_HPP__ + + +//////////////////////////////////////////////////////////////////////// + +static const int api_debug_level = 11; + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + float * out) +{ + +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + int * out) +{ + +const char *method_name = "load_numpy(int *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy_int (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + IntArray *out) +{ + +const char *method_name = "load_numpy_int(IntArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((int)value); + + mlog << Debug(api_debug_level) << method_name << " [" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy_num (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + NumArray *out) +{ + +const char *method_name = "load_numpy_num(NumArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((float)value); + + mlog << Debug(api_debug_level) << method_name << "[" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + +#endif /* __MET_POINTDATA_FROM_NUMPY_ARRAY_HPP__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/src/libcode/vx_pointdata_python/pointdata_python.cc b/src/libcode/vx_pointdata_python/pointdata_python.cc index 8fa505877f..a94ba3b714 100644 --- a/src/libcode/vx_pointdata_python/pointdata_python.cc +++ b/src/libcode/vx_pointdata_python/pointdata_python.cc @@ -189,8 +189,8 @@ file_name.chomp(".py"); // remove possible ".py" suffix from script filename bool status = python_point_data(file_name.c_str(), file_argc, file_argv, use_xarray, met_data); -int hdr_cnt = met_data.get_hdr_cnt(); -int obs_cnt = met_data.get_obs_cnt(); +met_data.get_hdr_cnt(); +met_data.get_obs_cnt(); MetPointHeader *hdr_data = met_data.get_header_data(); MetPointObsData *obs_data = met_data.get_point_obs_data(); diff --git a/src/libcode/vx_pointdata_python/python_pointdata.cc b/src/libcode/vx_pointdata_python/python_pointdata.cc index 52af27d297..e99cc49596 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.cc +++ b/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -26,59 +26,6 @@ static bool straight_python_point_data(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, MetPointDataPython &met_pd_out); -//////////////////////////////////////////////////////////////////////// - -template -static void set_array_from_python(PyObject *python_data, const char *python_key, T *out, bool required=true) { - const char *method_name = "set_array_from_python(T *) -> "; - PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); - if (numpy_array_obj) { - bool status = false; - ConcatString py_type_name = Py_TYPE(numpy_array_obj)->tp_name; - if ("numpy.ndarray" == py_type_name || "MaskedArray" == py_type_name ){ - Python3_Numpy np; - np.set(numpy_array_obj); - pointdata_from_np_array(np, out); - status = true; - } - else if (PyList_Check(numpy_array_obj)) { - status = pointdata_from_python_list(numpy_array_obj, out); - if(!status) { - if (PyFloat_Check(PyList_GetItem(numpy_array_obj, 0))) { - mlog << Error << "\n" << method_name - << "Only int type is supported at python list for " << python_key << "." - << " Check the data type from python and consider using numpy for python embedding\n\n"; - } - else { - mlog << Error << "\n" << method_name - << "Only int/float type is supported at python list for " << python_key << "." - << " Check the data type from python and consider using numpy for python embedding\n\n"; - } - } - } - else { - mlog << Error << "\n" << method_name - << "Not getting the point data by the key (" << python_key << ") from python object\n" - << " The python type \"" << py_type_name << "\" is not supported\n\n"; - } - if (status) { - mlog << Debug(7) << method_name - << "get the point data for " << python_key << " from python object\n"; - } - else exit (1); - } - else { - if (required) { - mlog << Error << "\n" << method_name - << "error getting the point data by the key (" << python_key << ") from python object\n\n"; - exit (1); - } - else mlog << Debug(3) << method_name - << "not exists the point data (" << python_key << ") from python object\n"; - } -} - - //////////////////////////////////////////////////////////////////////// static void set_str_array_from_python(PyObject *python_data, const char *python_key, StringArray *out) { @@ -120,12 +67,10 @@ bool straight_python_point_data(const char * script_name, int script_argc, char { int int_value; -PyObject * module_obj = 0; -PyObject * module_dict_obj = 0; -PyObject * python_key = 0; -PyObject * python_value = 0; -PyObject * numpy_array_obj = 0; -PyObject * python_met_point_data = 0; +PyObject *module_obj = 0; +PyObject *module_dict_obj = 0; +PyObject *python_value = 0; +PyObject *python_met_point_data = 0; ConcatString cs, user_dir, user_base; const char *method_name = "straight_python_point_data -> "; const char *method_name_s = "straight_python_point_data()"; diff --git a/src/libcode/vx_pointdata_python/python_pointdata.h b/src/libcode/vx_pointdata_python/python_pointdata.h index 54cd3ede9b..34005715e4 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.h +++ b/src/libcode/vx_pointdata_python/python_pointdata.h @@ -9,8 +9,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __PYTHON_POINTDATA__ -#define __PYTHON_POINTDATA__ +#ifndef __PYTHON_POINTDATA_H__ +#define __PYTHON_POINTDATA_H__ //////////////////////////////////////////////////////////////////////// @@ -72,8 +72,11 @@ extern void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_dat //////////////////////////////////////////////////////////////////////// +#include "python_pointdata.hpp" -#endif /* __PYTHON_POINTDATA__ */ +//////////////////////////////////////////////////////////////////////// + +#endif /* __PYTHON_POINTDATA_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/python_pointdata.hpp b/src/libcode/vx_pointdata_python/python_pointdata.hpp new file mode 100644 index 0000000000..c4ade37053 --- /dev/null +++ b/src/libcode/vx_pointdata_python/python_pointdata.hpp @@ -0,0 +1,79 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __PYTHON_POINTDATA_HPP__ +#define __PYTHON_POINTDATA_HPP__ + +#include "vx_python3_utils.h" + +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// + +template +static void set_array_from_python(PyObject *python_data, const char *python_key, T *out, bool required=true) { + const char *method_name = "set_array_from_python(T *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + bool status = false; + ConcatString py_type_name = Py_TYPE(numpy_array_obj)->tp_name; + if ("numpy.ndarray" == py_type_name || "MaskedArray" == py_type_name ){ + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + status = true; + } + else if (PyList_Check(numpy_array_obj)) { + status = pointdata_from_python_list(numpy_array_obj, out); + if(!status) { + if (PyFloat_Check(PyList_GetItem(numpy_array_obj, 0))) { + mlog << Error << "\n" << method_name + << "Only int type is supported at python list for " << python_key << "." + << " Check the data type from python and consider using numpy for python embedding\n\n"; + } + else { + mlog << Error << "\n" << method_name + << "Only int/float type is supported at python list for " << python_key << "." + << " Check the data type from python and consider using numpy for python embedding\n\n"; + } + } + } + else { + mlog << Error << "\n" << method_name + << "Not getting the point data by the key (" << python_key << ") from python object\n" + << " The python type \"" << py_type_name << "\" is not supported\n\n"; + } + if (status) { + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else exit (1); + } + else { + if (required) { + mlog << Error << "\n" << method_name + << "error getting the point data by the key (" << python_key << ") from python object\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; + } +} + + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __PYTHON_POINTDATA_HPP___ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index e0fc299baa..5007a7384f 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -596,7 +596,7 @@ void process_ioda_file(int i_pb) { // Initialize diff_file_time_count = 0; - for(int idx=0; idx valid_times_fcst; +vector valid_times_obs; + // // read the data files // -mtd_read_data(config, *(config.fcst_info), fcst_filenames, fcst_raw); +valid_times_fcst = mtd_read_data(config, *(config.fcst_info), fcst_filenames, fcst_raw); -mtd_read_data(config, *(config.obs_info), obs_filenames, obs_raw); +valid_times_obs = mtd_read_data(config, *(config.obs_info), obs_filenames, obs_raw); if ( fcst_raw.nt() != obs_raw.nt() ) { @@ -436,13 +444,21 @@ for (j=0; j<(fcst_obj.n_objects()); ++j) { mask_2d = fcst_obj.const_t_mask(t, j + 1); // 1-based + if (t < 0 || t >= (int)valid_times_fcst.size()) { + mlog << Error + << "\n " << program_name + << ": index " << t << " out of forecast valid times range 0 to " + << valid_times_fcst.size()-1 << "\n\n"; + exit ( 1 ); + } + fcst_raw.get_data_plane(t, raw_2d); att_2 = calc_2d_single_atts(mask_2d, raw_2d, j + 1, config.inten_perc_value); att_2.set_fcst(); - att_2.set_valid_time(fcst_obj.start_valid_time() + t*(fcst_obj.delta_t())); + att_2.set_valid_time(valid_times_fcst[t]); att_2.set_lead_time(fcst_obj.lead_time(t)); @@ -470,13 +486,21 @@ for (j=0; j<(obs_obj.n_objects()); ++j) { mask_2d = obs_obj.const_t_mask(t, j + 1); // 1-based + if (t < 0 || t >= (int)valid_times_obs.size()) { + mlog << Error + << "\n " << program_name << ": index " << t + << " out of obs valid times range 0 to " + << valid_times_obs.size()-1 << "\n\n"; + exit ( 1 ); + } + obs_raw.get_data_plane(t, raw_2d); att_2 = calc_2d_single_atts(mask_2d, raw_2d, j + 1, config.inten_perc_value); att_2.set_obs(); - att_2.set_valid_time(obs_obj.start_valid_time() + t*(obs_obj.delta_t())); + att_2.set_valid_time(valid_times_obs[t]); att_2.set_lead_time(obs_obj.lead_time(t)); @@ -680,10 +704,15 @@ if ( have_pairs ) { for (t=(att_3.tmin()); t<=(att_3.tmax()); ++t) { - // mask_2d = mask.const_t_mask(t, j + 1); // 1-based mask_2d = mask.const_t_mask(t, 1); // 1-based - // cout << "j = " << j << ", vol = " << mask_2d.object_volume(0) << '\n'; + if (t < 0 || t >= (int)valid_times_fcst.size()) { + mlog << Error + << "\n " << program_name << ": index " << t + << " out of forecast valid times range 0 to " + << valid_times_fcst.size()-1 << "\n\n"; + exit ( 1 ); + } fcst_raw.get_data_plane(t, raw_2d); @@ -691,7 +720,7 @@ if ( have_pairs ) { att_2.set_fcst(); - att_2.set_valid_time(fcst_obj.start_valid_time() + t*(fcst_obj.delta_t())); + att_2.set_valid_time(valid_times_fcst[t]); att_2.set_lead_time(fcst_obj.lead_time(t)); @@ -723,16 +752,23 @@ if ( have_pairs ) { for (t=(att_3.tmin()); t<=(att_3.tmax()); ++t) { - // mask_2d = mask.const_t_mask(t, j + 1); // 1-based mask_2d = mask.const_t_mask(t, 1); // 1-based + if (t < 0 || t >= (int)valid_times_obs.size()) { + mlog << Error + << "\n " << program_name << ": index " << t + << " out of obs valid times range 0 to " + << valid_times_obs.size()-1 << "\n\n"; + exit ( 1 ); + } + obs_raw.get_data_plane(t, raw_2d); att_2 = calc_2d_single_atts(mask_2d, raw_2d, j + 1, config.inten_perc_value); att_2.set_obs(); - att_2.set_valid_time(obs_obj.start_valid_time() + t*(obs_obj.delta_t())); + att_2.set_valid_time(valid_times_obs[t]); att_2.set_lead_time(obs_obj.lead_time(t)); @@ -785,9 +821,8 @@ mlog << Debug(2) << "Creating 2D constant-time slice attributes file: \"" << path << "\"\n"; -do_2d_txt_output(fcst_raw, obs_raw, - fcst_simple_att_2d, obs_simple_att_2d, - fcst_cluster_att_2d, obs_cluster_att_2d, config, path.c_str()); + do_2d_txt_output(fcst_raw, obs_raw, fcst_simple_att_2d, obs_simple_att_2d, + fcst_cluster_att_2d, obs_cluster_att_2d, config, path.c_str()); // // write simple single attributes @@ -872,7 +907,6 @@ mlog << Debug(2) do_mtd_nc_output(config.nc_info, engine, fcst_raw, obs_raw, fcst_obj, obs_obj, config, path.c_str()); - // // done // @@ -1058,11 +1092,17 @@ ConcatString prefix; ConcatString path; + // + // storage for valid times + // + +vector valid_times; + // // read the data files // -mtd_read_data(config, *(config.fcst_info), single_filenames, raw); +valid_times = mtd_read_data(config, *(config.fcst_info), single_filenames, raw); // // copy forecast name/units/level to observation @@ -1166,13 +1206,21 @@ for (j=0; j<(obj.n_objects()); ++j) { mask_2d = obj.const_t_mask(t, j + 1); // 1-based + if (t < 0 || t >= (int)valid_times.size()) { + mlog << Error + << "\n " << program_name << ": index " << t + << " out of valid times range 0 to " + << valid_times.size()-1 << "\n\n"; + exit ( 1 ); + } + raw.get_data_plane(t, raw_2d); att_2 = calc_2d_single_atts(mask_2d, raw_2d, j + 1, config.inten_perc_value); att_2.set_fcst(); - att_2.set_valid_time(obj.start_valid_time() + t*(obj.delta_t())); + att_2.set_valid_time(valid_times[t]); att_2.set_lead_time(obj.lead_time(t)); @@ -1232,7 +1280,6 @@ mlog << Debug(2) do_mtd_nc_output(config.nc_info, raw, obj, config, path.c_str()); - // // done // diff --git a/src/tools/other/mode_time_domain/mtd_file_base.cc b/src/tools/other/mode_time_domain/mtd_file_base.cc index 4295a2ba85..f1e3571d17 100644 --- a/src/tools/other/mode_time_domain/mtd_file_base.cc +++ b/src/tools/other/mode_time_domain/mtd_file_base.cc @@ -91,6 +91,8 @@ Nx = Ny = Nt = 0; StartValidTime = (unixtime) 0; +ActualValidTimes.clear(); + DeltaT = 0; Filename.clear(); @@ -292,7 +294,6 @@ return; } - //////////////////////////////////////////////////////////////////////// @@ -306,6 +307,19 @@ return; } +//////////////////////////////////////////////////////////////////////// + + +void MtdFileBase::init_actual_valid_times(const vector &validTimes) + +{ + +ActualValidTimes = validTimes; + +return; + +} + //////////////////////////////////////////////////////////////////////// @@ -349,6 +363,25 @@ return ( StartValidTime + t*DeltaT ); } +//////////////////////////////////////////////////////////////////////// + + +unixtime MtdFileBase::actual_valid_time(int t) const + +{ + +if ( (t < 0) || ( t >= (int)ActualValidTimes.size()) ) { + + mlog << Error << "\n\n MtdFileBase::valid_time(int t) -> range check error\n\n"; + + exit ( 1 ); + +} + +return ( ActualValidTimes[t] ); + +} + //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/mode_time_domain/mtd_file_base.h b/src/tools/other/mode_time_domain/mtd_file_base.h index 205b5feb92..98d30f10a1 100644 --- a/src/tools/other/mode_time_domain/mtd_file_base.h +++ b/src/tools/other/mode_time_domain/mtd_file_base.h @@ -85,9 +85,11 @@ class MtdFileBase { int Nx, Ny, Nt; - unixtime StartValidTime; + unixtime StartValidTime; // useful for constant time increments - int DeltaT; // seconds + int DeltaT; // seconds, useful for constant time increments + + vector ActualValidTimes; // useful for uneven time increments IntArray Lead_Times; @@ -113,8 +115,11 @@ class MtdFileBase { void set_grid(const Grid &); void set_start_valid_time (unixtime); + void set_delta_t (int); // seconds + void init_actual_valid_times(const vector &validTimes); + void set_lead_time(int index, int value); void set_filetype(MtdFileType); @@ -143,6 +148,8 @@ class MtdFileBase { unixtime valid_time (int) const; + unixtime actual_valid_time (int) const; + int lead_time (int index) const; // diff --git a/src/tools/other/mode_time_domain/mtd_read_data.cc b/src/tools/other/mode_time_domain/mtd_read_data.cc index 2b4eb852d0..d40335594b 100644 --- a/src/tools/other/mode_time_domain/mtd_read_data.cc +++ b/src/tools/other/mode_time_domain/mtd_read_data.cc @@ -22,12 +22,13 @@ using namespace std; #include "mtd_read_data.h" +#include "num_array.h" //////////////////////////////////////////////////////////////////////// -void mtd_read_data(MtdConfigInfo & config, VarInfo & varinfo, - const StringArray & filenames, MtdFloatFile & raw) +vector mtd_read_data(MtdConfigInfo & config, VarInfo & varinfo, + const StringArray & filenames, MtdFloatFile & raw) { @@ -39,16 +40,11 @@ if ( filenames.n() < 2 ) { } -int j; +int j, k; Met2dDataFile * data_2d_file = 0; Met2dDataFileFactory factory; DataPlane plane; -unixtime * valid_times = 0; - - - -valid_times = new unixtime [filenames.n()]; - +vector valid_times; // // read the files // @@ -76,7 +72,7 @@ for (j=0; j<(filenames.n()); ++j) { } - valid_times[j] = plane.valid(); + valid_times.push_back(plane.valid()); if ( j == 0 ) { @@ -96,30 +92,76 @@ for (j=0; j<(filenames.n()); ++j) { } // for j -varinfo.set_valid(valid_times[0]); + varinfo.set_valid(valid_times[0]); + + // store the valid times vector into the raw data for later use in do_2d_txt_output() + + raw.init_actual_valid_times(valid_times); // - // check the time intervals + // check the time intervals for consistency + // Store the time differences between succesive valid times in an array + // See if differences are constant or not, and if not see if all diffs are months // -unixtime dt_start, dt; +unixtime dt_start; +vector dtVector; dt_start = valid_times[1] - valid_times[0]; +dtVector.push_back(dt_start); -for (j=2; j<(filenames.n()); ++j) { - - dt = valid_times[j] - valid_times[j - 1]; - - if ( dt != dt_start ) { - - mlog << Error << "\n\n mtd_read_data() -> file time increments are not constant!\n\n"; - - exit ( 1 ); +for (size_t k=2; k File time increments are not constant, could be problematic\n"; + mlog << Warning << " mtd_read_data() -> Using MODE of the increments, mode=" << dt_start << "\n"; + mlog << Warning << " mtd_read_data() -> Time increment properties: mean=" << umean << " variance=" << uvar << " sqrt(var)=" << suvar << "\n\n"; + } } - + // // load up the rest of the MtdFloatFile class members // @@ -138,9 +180,7 @@ raw.calc_data_minmax(); // done // -if ( valid_times ) { delete [] valid_times; valid_times = 0; } - -return; +return valid_times; } diff --git a/src/tools/other/mode_time_domain/mtd_read_data.h b/src/tools/other/mode_time_domain/mtd_read_data.h index 26d0a9ba18..0da67cc2a9 100644 --- a/src/tools/other/mode_time_domain/mtd_read_data.h +++ b/src/tools/other/mode_time_domain/mtd_read_data.h @@ -26,9 +26,13 @@ //////////////////////////////////////////////////////////////////////// +// +// returns the actual valid times +// -extern void mtd_read_data(MtdConfigInfo &, VarInfo &, - const StringArray & filenames, MtdFloatFile &); +extern vector mtd_read_data(MtdConfigInfo &, VarInfo &, + const StringArray & filenames, + MtdFloatFile &); //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/mode_time_domain/mtd_txt_output.cc b/src/tools/other/mode_time_domain/mtd_txt_output.cc index a0bf061117..2372823d25 100644 --- a/src/tools/other/mode_time_domain/mtd_txt_output.cc +++ b/src/tools/other/mode_time_domain/mtd_txt_output.cc @@ -496,7 +496,7 @@ for (j=0; j<(fcst_simple_att.n()); ++j) { table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_simple_att.valid_time(j))); - table.set_entry(r, obs_valid_column, unix_to_yyyymmdd_hhmmss(obs_raw.valid_time(t))); + table.set_entry(r, obs_valid_column, unix_to_yyyymmdd_hhmmss(obs_raw.actual_valid_time(t))); table.set_entry(r, obs_lead_column, sec_to_hhmmss(obs_raw.lead_time(t))); @@ -508,7 +508,7 @@ for (j=0; j<(obs_simple_att.n()); ++j) { t = obs_simple_att.time_index(j); - table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_raw.valid_time(t))); + table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_raw.actual_valid_time(t))); table.set_entry(r, fcst_lead_column, sec_to_hhmmss(fcst_raw.lead_time(t))); @@ -528,7 +528,7 @@ for (j=0; j<(fcst_cluster_att.n()); ++j) { table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_cluster_att.valid_time(j))); - table.set_entry(r, obs_valid_column, unix_to_yyyymmdd_hhmmss(obs_raw.valid_time(t))); + table.set_entry(r, obs_valid_column, unix_to_yyyymmdd_hhmmss(obs_raw.actual_valid_time(t))); table.set_entry(r, obs_lead_column, sec_to_hhmmss(obs_raw.lead_time(t))); @@ -540,7 +540,7 @@ for (j=0; j<(obs_cluster_att.n()); ++j) { t = obs_cluster_att.time_index(j); - table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_raw.valid_time(t))); + table.set_entry(r, fcst_valid_column, unix_to_yyyymmdd_hhmmss(fcst_raw.actual_valid_time(t))); table.set_entry(r, fcst_lead_column, sec_to_hhmmss(fcst_raw.lead_time(t))); diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index fd125500cf..388e90b22d 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -277,11 +277,10 @@ void process_command_line(int argc, char **argv) { OutputFilename = cline[2]; // Check if the input file - bool use_python = false; #ifdef WITH_PYTHON string python_command = InputFilename; bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); - use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); + bool use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); if (use_python) { int offset = python_command.find("="); if (offset == std::string::npos) { @@ -376,12 +375,11 @@ void process_data_file() { mlog << Debug(1) << "Reading data file: " << InputFilename << "\n"; bool goes_data = false; bool use_python = false; - int obs_type = TYPE_UNKNOWN; + int obs_type; Met2dDataFileFactory m_factory; Met2dDataFile *fr_mtddf = (Met2dDataFile *) 0; #ifdef WITH_PYTHON string python_command = InputFilename; - MetPointData *met_point_obs = 0; bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); if (use_python) { @@ -918,7 +916,7 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI } if (cellMapping) { - for (int idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); + for (idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); delete [] cellMapping; } cellMapping = new IntArray[nx * ny]; @@ -1135,11 +1133,6 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, const Grid to_grid) { - int nhdr, nobs; - int nx, ny, var_count, to_count, var_count2; - int idx, hdr_idx; - int var_idx_or_gc; - int filtered_by_time, filtered_by_msg_type, filtered_by_qc; ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; @@ -1147,7 +1140,6 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, NcVar var_obs_gc, var_obs_var; clock_t start_clock = clock(); - bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_file() -> "; @@ -1182,11 +1174,7 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, const Grid to_grid, bool use_xarray) { - int nhdr, nobs; - int nx, ny, var_count, to_count, var_count2; int idx, hdr_idx; - int var_idx_or_gc; - int filtered_by_time, filtered_by_msg_type, filtered_by_qc; ConcatString vname, vname_cnt, vname_mask; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp;