From 45588810b13b811bde761a0e25d74c0e0ddbd127 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Sun, 26 Feb 2023 18:55:10 -0700 Subject: [PATCH] Update develop-ref after #2462 (#2470) Co-authored-by: johnhg Co-authored-by: Lisa Goodrich Co-authored-by: jprestop Co-authored-by: Seth Linden Co-authored-by: Howard Soh Co-authored-by: John Halley Gotway Co-authored-by: MET Tools Test Account Co-authored-by: Dave Albo Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Jonathan Vigh Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Seth Linden Co-authored-by: hsoh-u Co-authored-by: davidalbo Co-authored-by: Daniel Adriaansen fix 2238 link error (#2255) fix #2271 develop nbrctc (#2272) fix #2309 develop tcmpr (#2310) fix #2306 ascii2nc airnow hourly (#2314) fix_spread_md (#2335) fix #2389 develop flowchart (#2392) Fix Python environment issue (#2407) fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2406) fix #2380 develop override (#2382) fix #2408 develop empty config (#2410) fix #2390 develop compile zlib (#2404) fix #2412 develop climo (#2422) fix #2437 develop convert (#2439) fix for develop, for #2437, forgot one reference to the search_parent for a dictionary lookup. fix #2452 develop airnow (#2454) --- data/wrappers/Makefile.am | 2 + data/wrappers/Makefile.in | 2 + data/wrappers/read_tmp_point_nc.py | 26 ++ data/wrappers/write_tmp_point_nc.py | 55 +++ docs/Users_Guide/appendixF.rst | 2 +- docs/Users_Guide/release-notes.rst | 200 ++++---- docs/conf.py | 2 +- docs/requirements.txt | 1 + internal/test_unit/xml/unit_python.xml | 19 + scripts/python/Makefile.am | 1 + scripts/python/Makefile.in | 1 + scripts/python/met_point_obs.py | 262 +++++++---- scripts/python/met_point_obs_nc.py | 281 +++++++++++ scripts/python/read_met_point_obs.py | 142 +++--- .../vx_data2d_python/python_dataplane.cc | 6 +- .../vx_pointdata_python/python_pointdata.cc | 435 +++++++++++++----- src/libcode/vx_python3_utils/global_python.h | 2 +- src/libcode/vx_python3_utils/python3_util.cc | 5 + src/libcode/vx_python3_utils/python3_util.h | 7 +- 19 files changed, 1045 insertions(+), 406 deletions(-) create mode 100644 data/wrappers/read_tmp_point_nc.py create mode 100644 data/wrappers/write_tmp_point_nc.py create mode 100644 scripts/python/met_point_obs_nc.py diff --git a/data/wrappers/Makefile.am b/data/wrappers/Makefile.am index d2f83ff125..deb919438e 100644 --- a/data/wrappers/Makefile.am +++ b/data/wrappers/Makefile.am @@ -24,8 +24,10 @@ wrappers_DATA = \ set_python_env.py \ read_tmp_dataplane.py \ read_tmp_ascii.py \ + read_tmp_point_nc.py \ write_tmp_dataplane.py \ write_tmp_point.py \ + write_tmp_point_nc.py \ write_tmp_mpr.py EXTRA_DIST = ${wrappers_DATA} diff --git a/data/wrappers/Makefile.in b/data/wrappers/Makefile.in index 7d69426340..da04b2b2a0 100644 --- a/data/wrappers/Makefile.in +++ b/data/wrappers/Makefile.in @@ -360,8 +360,10 @@ wrappers_DATA = \ set_python_env.py \ read_tmp_dataplane.py \ read_tmp_ascii.py \ + read_tmp_point_nc.py \ write_tmp_dataplane.py \ write_tmp_point.py \ + write_tmp_point_nc.py \ write_tmp_mpr.py EXTRA_DIST = ${wrappers_DATA} diff --git a/data/wrappers/read_tmp_point_nc.py b/data/wrappers/read_tmp_point_nc.py new file mode 100644 index 0000000000..0ef8eefc3a --- /dev/null +++ b/data/wrappers/read_tmp_point_nc.py @@ -0,0 +1,26 @@ +######################################################################## +# +# Reads temporary point obs. file into memory. +# +# usage: /path/to/python read_tmp_point_nc.py tmp_output_filename +# +######################################################################## + +import os +import sys + +# add share/met/python directory to system path to find met_point_obs +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'python'))) +from met_point_obs import met_point_obs +from met_point_obs_nc import nc_point_obs + +netcdf_filename = sys.argv[1] + +# read NetCDF file +print('{p} reading{f}'.format(p=met_point_obs.get_prompt(), f=netcdf_filename)) +point_obs_data = nc_point_obs() +point_obs_data.read_data(netcdf_filename) + +met_point_data = point_obs_data.get_point_data() +met_point_data['met_point_data'] = point_obs_data diff --git a/data/wrappers/write_tmp_point_nc.py b/data/wrappers/write_tmp_point_nc.py new file mode 100644 index 0000000000..063a2e98cc --- /dev/null +++ b/data/wrappers/write_tmp_point_nc.py @@ -0,0 +1,55 @@ +######################################################################## +# +# Adapted from a script provided by George McCabe +# Adapted by Howard Soh +# +# usage: /path/to/python write_tmp_point_nc.py \ +# tmp_output_filename .py +# +######################################################################## + +import os +import sys +import importlib.util + +# add share/met/python directory to system path to find met_point_obs +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'python'))) + +from met_point_obs import met_point_obs +from met_point_obs_nc import nc_point_obs + +PROMPT = met_point_obs.get_prompt() +print("{p} Python Script:\t".format(p=PROMPT) + repr(sys.argv[0])) +print("{p} User Command:\t".format(p=PROMPT) + repr(' '.join(sys.argv[2:]))) +print("{p} Temporary File:\t".format(p=PROMPT) + repr(sys.argv[1])) + +tmp_filename = sys.argv[1] +pyembed_module_name = sys.argv[2] +sys.argv = sys.argv[2:] + +# append user script dir to system path +pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) +if pyembed_dir: + sys.path.insert(0, pyembed_dir) + +if not pyembed_module_name.endswith('.py'): + pyembed_module_name += '.py' + +user_base = os.path.basename(pyembed_module_name).replace('.py','') + +spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) +met_in = importlib.util.module_from_spec(spec) +spec.loader.exec_module(met_in) + +if hasattr(met_in, 'point_obs_data'): + met_in.point_obs_data.save_ncfile(tmp_filename) +else: + if hasattr(met_in.met_point_data, 'point_obs_data'): + met_in.met_point_data['point_obs_data'].save_ncfile(tmp_filename) + else: + tmp_point_obs = nc_point_obs() + tmp_point_obs.put_data(met_in.met_point_data) + tmp_point_obs.save_ncfile(tmp_filename) + +#print('{p} writing {f}'.format(p=PROMPT, f=tmp_filename)) diff --git a/docs/Users_Guide/appendixF.rst b/docs/Users_Guide/appendixF.rst index 3a32cc425e..371773d519 100644 --- a/docs/Users_Guide/appendixF.rst +++ b/docs/Users_Guide/appendixF.rst @@ -260,7 +260,7 @@ The ASCII2NC tool supports the "-format python" option. With this option, point "MET_BASE/python/read_ascii_point.py sample_ascii_obs.txt" \ sample_ascii_obs_python.nc -The Point2Grid, Plot-Point-Obs, Ensemble-Stat, and Point-Stat tools also process point observations. They support POython embedding of point observations directly on the command line by replacing the input MET NetCDF point observation file name with the Python command to be run. The Python command must begin with the prefix 'PYTHON_NUMPY=' and be followed by the path to the User's Python script and any arguments. The full command should be enclosed in single quotes to prevent embedded whitespace from causing parsing errors. An example of this is shown below: +The Point2Grid, Plot-Point-Obs, Ensemble-Stat, and Point-Stat tools also process point observations. They support Python embedding of point observations directly on the command line by replacing the input MET NetCDF point observation file name with the Python command to be run. The Python command must begin with the prefix 'PYTHON_NUMPY=' and be followed by the path to the User's Python script and any arguments. The full command should be enclosed in single quotes to prevent embedded whitespace from causing parsing errors. An example of this is shown below: .. code-block:: none diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 4dfd494c4a..bcad89b962 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -12,131 +12,133 @@ Important issues are listed **in bold** for emphasis. MET Version 11.0.0 release notes (20221209) ------------------------------------------- -* Repository, build, and test: - - * **Restructure the contents of the MET repository so that it matches the existing release tarfiles** (`#1920 `_). - * **Add initial files to create the MET compilation environment in the dtcenter/met-base Docker image** (`dtcenter/METbaseimage#1 `_). - * Restructure the MET Dockerfiles to create images based on the new METbaseimage (`#2196 `_). - * Enhance METbaseimage to support NetCDF files using groups in the enhanced data model (`dtcenter/METbaseimage#6 `_). - * Add .zenodo.json file to add metadata about releases (`#2198 `_). - * Update the SonarQube version used for routine software scans (`#2270 `_). - * Fix OpenMP compilation error for GCC 9.3.0/9.4.0 (`#2106 `_). - * Fix oom() compile time linker error (`#2238 `_). - * Fix MET-11.0.0-beta3 linker errors (`#2281 `_). - * Fix GHA documentation workflow (`#2282 `_). - * Fix GHA warnings and update the version of actions (i.e. actions/checkout@v3) (`#2297 `_). - -* Documentation: - - * Create outline for the MET Contributor's Guide (`#1774 `_). - * Document PB2NC's handling of quality markers (`#2278 `_). - * Move release notes into its own chapter in the User's Guide (`#2298 `_). - -* Bugfixes: - - * Fix regression test differences in pb2nc and ioda2nc output (`#2102 `_). - * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). - * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). - * Fix support for int64 NetCDF variable types (`#2123 `_). - * Fix Stat-Analysis to aggregate the ECNT ME and RMSE values correctly (`#2170 `_). - * Fix NetCDF library code to process scale_factor and add_offset attributes independently (`#2187 `_). - * Fix Ensemble-Stat to work with different missing members for two or more variables (`#2208 `_). - * Fix truncated station_id name in the output from IODA2NC (`#2216 `_). - * Fix Stat-Analysis aggregation of the neighborhood statistics line types (`#2271 `_). - * Fix Point-Stat and Ensemble-Stat GRIB table lookup logic for python embedding of point observations (`#2286 `_). - * Fix ascii2nc_airnow_hourly test in unit_ascii2nc.xml (`#2306 `_). - * Fix TC-Stat parsing of TCMPR lines (`#2309 `_). - * Fix ASCII2NC logic for reading AERONET v3 data (`#2370 `_). - -* Enhancements: - - * NetCDF: + .. dropdown:: Repository, build, and test + + * **Restructure the contents of the MET repository so that it matches the existing release tarfiles** (`#1920 `_). + * **Add initial files to create the MET compilation environment in the dtcenter/met-base Docker image** (`dtcenter/METbaseimage#1 `_). + * Restructure the MET Dockerfiles to create images based on the new METbaseimage (`#2196 `_). + * Enhance METbaseimage to support NetCDF files using groups in the enhanced data model (`dtcenter/METbaseimage#6 `_). + * Add .zenodo.json file to add metadata about releases (`#2198 `_). + * Update the SonarQube version used for routine software scans (`#2270 `_). + * Fix OpenMP compilation error for GCC 9.3.0/9.4.0 (`#2106 `_). + * Fix oom() compile time linker error (`#2238 `_). + * Fix MET-11.0.0-beta3 linker errors (`#2281 `_). + * Fix GHA documentation workflow (`#2282 `_). + * Fix GHA warnings and update the version of actions (i.e. actions/checkout@v3) (`#2297 `_). + + + + .. dropdown:: Documentation + + * Create outline for the MET Contributor's Guide (`#1774 `_). + * Document PB2NC's handling of quality markers (`#2278 `_). + * Move release notes into its own chapter in the User's Guide (`#2298 `_). + + .. dropdown:: Bugfixes + + * Fix regression test differences in pb2nc and ioda2nc output (`#2102 `_). + * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). + * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). + * Fix support for int64 NetCDF variable types (`#2123 `_). + * Fix Stat-Analysis to aggregate the ECNT ME and RMSE values correctly (`#2170 `_). + * Fix NetCDF library code to process scale_factor and add_offset attributes independently (`#2187 `_). + * Fix Ensemble-Stat to work with different missing members for two or more variables (`#2208 `_). + * Fix truncated station_id name in the output from IODA2NC (`#2216 `_). + * Fix Stat-Analysis aggregation of the neighborhood statistics line types (`#2271 `_). + * Fix Point-Stat and Ensemble-Stat GRIB table lookup logic for python embedding of point observations (`#2286 `_). + * Fix ascii2nc_airnow_hourly test in unit_ascii2nc.xml (`#2306 `_). + * Fix TC-Stat parsing of TCMPR lines (`#2309 `_). + * Fix ASCII2NC logic for reading AERONET v3 data (`#2370 `_). + + .. dropdown:: Enhancements + + .. dropdown:: NetCDF * **Enhance MET's NetCDF library interface to support level strings that include coordinate variable values instead of just indexes** (`#1815 `_). * Enhance MET to handle NC strings when processing CF-Compliant NetCDF files (`#2042 `_). * Enhance MET to handle CF-compliant time strings with an offset defined in months or years (`#2155 `_). * Refine NetCDF level string handling logic to always interpret @ strings as values (`#2225 `_). - * GRIB: + .. dropdown:: GRIB + + * Add support for reading National Blend Model GRIB2 data (`#2055 `_). + * Update the GRIB2 MRMS table in MET (`#2081 `_). - * Add support for reading National Blend Model GRIB2 data (`#2055 `_). - * Update the GRIB2 MRMS table in MET (`#2081 `_). + .. dropdown::Python - * Python: + * Reimplement the pntnc2ascii.R utility Rscript in Python (`#2085 `_). + * Add more error checking for python embedding of point observations (`#2202 `_). + * **Add a Python helper script/function to transform point_data objects to met_point_data objects for Python Embedding** (`#2302 `_). - * Reimplement the pntnc2ascii.R utility Rscript in Python (`#2085 `_). - * Add more error checking for python embedding of point observations (`#2202 `_). - * **Add a Python helper script/function to transform point_data objects to met_point_data objects for Python Embedding** (`#2302 `_). + .. dropdown:: METplus-Internal - * METplus-Internal: + * MET: Replace fixed length character arrays with strings (`dtcenter/METplus-Internal#14 `_). + * MET: Add a timestamp to the log output at the beginning and end of each MET tool run (`dtcenter/METplus-Internal#18 `_). + * MET: Add the user ID and the command line being executed to the log output at beginning and end of each MET tool run (`dtcenter/METplus-Internal#19 `_). + * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). - * MET: Replace fixed length character arrays with strings (`dtcenter/METplus-Internal#14 `_). - * MET: Add a timestamp to the log output at the beginning and end of each MET tool run (`dtcenter/METplus-Internal#18 `_). - * MET: Add the user ID and the command line being executed to the log output at beginning and end of each MET tool run (`dtcenter/METplus-Internal#19 `_). - * MET: Enhance MET to have better signal handling for shutdown events (`dtcenter/METplus-Internal#21 `_). + .. dropdown:: Common Libraries - * Common Libraries: + * **Define new grid class to store semi-structured grid information (e.g. lat or lon vs level or time)** (`#1954 `_). + * Refine warning/error messages when parsing thresholds (`#2211 `_). + * Remove namespace specification from header files (`#2227 `_). + * Update MET version number to 11.0.0 (`#2132 `_). + * Store unspecified accumulation interval as 0 rather than bad data (`#2250 `_). + * Add sanity check to error out when both is_u_wind and is_v_wind are set to true (`#2357 `_). - * **Define new grid class to store semi-structured grid information (e.g. lat or lon vs level or time)** (`#1954 `_). - * Refine warning/error messages when parsing thresholds (`#2211 `_). - * Remove namespace specification from header files (`#2227 `_). - * Update MET version number to 11.0.0 (`#2132 `_). - * Store unspecified accumulation interval as 0 rather than bad data (`#2250 `_). - * Add sanity check to error out when both is_u_wind and is_v_wind are set to true (`#2357 `_). + .. dropdown:: Statistics - * Statistics: + * **Add Anomaly Correlation Coefficient to VCNT Line Type** (`#2022 `_). + * **Allow 2x2 HSS calculations to include user-defined EC values** (`#2147 `_). + * **Add the fair CRPS statistic to the ECNT line type in a new CRPS_EMP_FAIR column** (`#2206 `_). + * **Add MAE to the ECNT line type from Ensemble-Stat and for HiRA** (`#2325 `_). + * **Add the Mean Absolute Difference (SPREAD_MD) to the ECNT line type** (`#2332 `_). + * **Add new bias ratio statistic to the ECNT line type from Ensemble-Stat and for HiRA** (`#2058 `_). - * **Add Anomaly Correlation Coefficient to VCNT Line Type** (`#2022 `_). - * **Allow 2x2 HSS calculations to include user-defined EC values** (`#2147 `_). - * **Add the fair CRPS statistic to the ECNT line type in a new CRPS_EMP_FAIR column** (`#2206 `_). - * **Add MAE to the ECNT line type from Ensemble-Stat and for HiRA** (`#2325 `_). - * **Add the Mean Absolute Difference (SPREAD_MD) to the ECNT line type** (`#2332 `_). - * **Add new bias ratio statistic to the ECNT line type from Ensemble-Stat and for HiRA** (`#2058 `_). + .. dropdown:: Configuration and masking - * Configuration and masking: + * Define the Bukovsky masking regions for use in MET (`#1940 `_). + * **Enhance Gen-Vx-Mask by adding a new poly_xy masking type option** (`#2152 `_). + * Add M_to_KFT and KM_to_KFT functions to ConfigConstants (`#2180 `_). + * Update map data with more recent NaturalEarth definitions (`#2207 `_). - * Define the Bukovsky masking regions for use in MET (`#1940 `_). - * **Enhance Gen-Vx-Mask by adding a new poly_xy masking type option** (`#2152 `_). - * Add M_to_KFT and KM_to_KFT functions to ConfigConstants (`#2180 `_). - * Update map data with more recent NaturalEarth definitions (`#2207 `_). + .. dropdown:: Point Pre-Processing Tools - * Point Pre-Processing Tools: + * **Enhance IODA2NC to support IODA v2.0 format** (`#2068 `_). + * **Add support for EPA AirNow ASCII data in ASCII2NC** (`#2142 `_). + * Add a sum option to the time summaries computed by the point pre-processing tools (`#2204 `_). + * Add "station_ob" to metadata_map as a message_type metadata variable for ioda2nc (`#2215 `_). + * **Enhance ASCII2NC to read NDBC buoy data** (`#2276 `_). + * Print ASCII2NC warning message about python embedding support not being compiled (`#2277 `_). - * **Enhance IODA2NC to support IODA v2.0 format** (`#2068 `_). - * **Add support for EPA AirNow ASCII data in ASCII2NC** (`#2142 `_). - * Add a sum option to the time summaries computed by the point pre-processing tools (`#2204 `_). - * Add "station_ob" to metadata_map as a message_type metadata variable for ioda2nc (`#2215 `_). - * **Enhance ASCII2NC to read NDBC buoy data** (`#2276 `_). - * Print ASCII2NC warning message about python embedding support not being compiled (`#2277 `_). + .. dropdown:: Point-Stat, Grid-Stat, Stat-Analysis - * Point-Stat, Grid-Stat, Stat-Analysis: + * Add support for point-based climatologies for use in SEEPS (`#1941 `_). + * **Enhance Point-Stat to compute SEEPS for point observations and write new SEEPS and SEEPS_MPR STAT line types** (`#1942 `_). + * **Enhance Grid-Stat to compute SEEPS for gridded observations and write the SEEPS STAT line type** (`#1943 `_). + * Sort mask.sid station lists to check their contents more efficiently (`#1950 `_). + * **Enhance Stat-Analysis to aggregate SEEPS_MPR and SEEPS line types** (`#2339 `_). + * Relax Point-Stat and Ensemble-Stat logic for the configuration of message_type_group_map (`#2362 `_). + * Fix Point-Stat and Grid-Stat logic for processing U/V winds with python embedding (`#2366 `_). - * Add support for point-based climatologies for use in SEEPS (`#1941 `_). - * **Enhance Point-Stat to compute SEEPS for point observations and write new SEEPS and SEEPS_MPR STAT line types** (`#1942 `_). - * **Enhance Grid-Stat to compute SEEPS for gridded observations and write the SEEPS STAT line type** (`#1943 `_). - * Sort mask.sid station lists to check their contents more efficiently (`#1950 `_). - * **Enhance Stat-Analysis to aggregate SEEPS_MPR and SEEPS line types** (`#2339 `_). - * Relax Point-Stat and Ensemble-Stat logic for the configuration of message_type_group_map (`#2362 `_). - * Fix Point-Stat and Grid-Stat logic for processing U/V winds with python embedding (`#2366 `_). + .. dropdown:: Ensemble Tools - * Ensemble Tools: + * **Remove ensemble post-processing from the Ensemble-Stat tool** (`#1908 `_). + * Eliminate Gen-Ens-Prod warning when parsing the nbhrd_prob dictionary (`#2224 `_). - * **Remove ensemble post-processing from the Ensemble-Stat tool** (`#1908 `_). - * Eliminate Gen-Ens-Prod warning when parsing the nbhrd_prob dictionary (`#2224 `_). + .. dropdown:: Tropical Cyclone Tools - * Tropical Cyclone Tools: + * **Enhance TC-Pairs to read hurricane model diagnostic files (e.g. SHIPS) and TC-Stat to filter the new data** (`#392 `_). + * **Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure** (`#2036 `_). + * Enhance TC-RMW to compute tangential and radial winds (`#2072 `_). + * Refine TCDIAG output from TC-Pairs as needed (`#2321 `_). + * Rename the TCDIAG SOURCE column as DIAG_SOURCE (`#2337 `_). - * **Enhance TC-Pairs to read hurricane model diagnostic files (e.g. SHIPS) and TC-Stat to filter the new data** (`#392 `_). - * **Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure** (`#2036 `_). - * Enhance TC-RMW to compute tangential and radial winds (`#2072 `_). - * Refine TCDIAG output from TC-Pairs as needed (`#2321 `_). - * Rename the TCDIAG SOURCE column as DIAG_SOURCE (`#2337 `_). + .. dropdown:: Miscellaneous - * Miscellaneous: - - * Enhance MTD to process time series with non-uniform time steps, such as monthly data (`#1971 `_). - * Refine Grid-Diag output variable names when specifying two input data sources (`#2232 `_). - * Add tmp_dir configuration option to the Plot-Point-Obs tool (`#2237 `_). + * Enhance MTD to process time series with non-uniform time steps, such as monthly data (`#1971 `_). + * Refine Grid-Diag output variable names when specifying two input data sources (`#2232 `_). + * Add tmp_dir configuration option to the Plot-Point-Obs tool (`#2237 `_). MET Upgrade Instructions ======================== diff --git a/docs/conf.py b/docs/conf.py index 055e2c9651..ac12a9bf43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,7 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. # Adding 'sphinx_panels' to use drop-down menus in appendixA. -extensions = ['sphinx.ext.autodoc','sphinx.ext.intersphinx','sphinx_panels',] +extensions = ['sphinx.ext.autodoc','sphinx.ext.intersphinx','sphinx_panels','sphinx_design',] # settings for ReadTheDocs PDF creation latex_engine = 'pdflatex' diff --git a/docs/requirements.txt b/docs/requirements.txt index 59572ff5a8..80ac5cb61d 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ sphinx-gallery sphinxcontrib-bibtex sphinx-panels +sphinx-design diff --git a/internal/test_unit/xml/unit_python.xml b/internal/test_unit/xml/unit_python.xml index 05f9d2650d..5a519d9212 100644 --- a/internal/test_unit/xml/unit_python.xml +++ b/internal/test_unit/xml/unit_python.xml @@ -495,6 +495,25 @@ + + + &MET_BIN;/point2grid + + MET_PYTHON_EXE &MET_PYTHON_EXE; + + \ + 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \ + G212 \ + &OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc \ + -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ <0 ]; censor_val=[0];' \ + -name TEMP \ + -v 1 + + + &OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc + + + &MET_BIN;/plot_point_obs diff --git a/scripts/python/Makefile.am b/scripts/python/Makefile.am index 76aa04cdf1..689708e4c3 100644 --- a/scripts/python/Makefile.am +++ b/scripts/python/Makefile.am @@ -27,6 +27,7 @@ pythonscriptsdir = $(pkgdatadir)/python pythonscripts_DATA = \ met_point_obs.py \ + met_point_obs_nc.py \ read_ascii_numpy.py \ read_ascii_numpy_grid.py \ read_ascii_xarray.py \ diff --git a/scripts/python/Makefile.in b/scripts/python/Makefile.in index bfcc683427..6d85ed81f9 100644 --- a/scripts/python/Makefile.in +++ b/scripts/python/Makefile.in @@ -299,6 +299,7 @@ top_srcdir = @top_srcdir@ pythonscriptsdir = $(pkgdatadir)/python pythonscripts_DATA = \ met_point_obs.py \ + met_point_obs_nc.py \ read_ascii_numpy.py \ read_ascii_numpy_grid.py \ read_ascii_xarray.py \ diff --git a/scripts/python/met_point_obs.py b/scripts/python/met_point_obs.py index 5cfc5f7141..fb108705fd 100755 --- a/scripts/python/met_point_obs.py +++ b/scripts/python/met_point_obs.py @@ -1,21 +1,25 @@ - +#!/usr/bin/env python3 ''' Created on Nov 10, 2021 @author: hsoh - This is the base class and the customized script should extend the met_point_obs. -- The customized script (for example "custom_reader") must implement "def read_data(self, args)" - which fills the array (python list or numpy array) variables at __init__(). -- The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments. -- The variable "met_point_data" must be set for MET tools +- The customized script (for example "custom_reader") must implement + "def read_data(self, args)" which fills the array variables at __init__(). +- The args can be 1) single string argument, 2) the list of arguments, + or 3) the dictionary of arguments. +- A python objects, met_point_data, must set: + + "point_obs_data" is an optional to use custom python EXE. + It's a python instance which processes the point observation data - The customized script is expected to include following codes: # prepare arguments for the customized script args = {'input', sys.argv[1]} # or args = [] point_obs_data = custom_reader() - point_obs_data.read_data() + point_obs_data.read_data(args) met_point_data = point_obs_data.get_point_data() + ''' import os @@ -24,22 +28,29 @@ COUNT_SHOW = 30 -MET_PYTHON_OBS_ARGS = "MET_POINT_PYTHON_ARGS" +def get_prompt(): + return " python:" + +def met_is_python_prefix(user_cmd): + return user_cmd.startswith(base_met_point_obs.python_prefix) class base_met_point_obs(object): ''' classdocs ''' - ERROR_P = " ==ERROR_PYTHON==" - INFO_P = " ==INFO_PYTHON==" + ERROR_P = " ==PYTHON_ERROR==" + INFO_P = " ==PYTHON_INFO==" + + python_prefix = 'PYTHON_POINT_USER' - python_prefix = 'PYTHON_POINT_RAW' + FILL_VALUE = -9999. def __init__(self, use_var_id=True): ''' Constructor ''' + self.count_info = "" self.input_name = None self.ignore_input_file = False self.use_var_id = use_var_id # True if variable index, False if GRIB code @@ -74,6 +85,8 @@ def __init__(self, use_var_id=True): self.obs_val = [] # (nobs) float self.obs_qty_table = [] # (nobs_qty, mxstr) string self.obs_var_table = [] # (nobs_var, mxstr2) string, required if self.use_var_id is True + self.obs_var_unit = [] # (nobs_var, mxstr2) string, optional if self.use_var_id is True + self.obs_var_desc = [] # (nobs_var, mxstr3) string, optional if self.use_var_id is True # Optional variables for PREPBUFR, not supported yet self.hdr_prpt_typ = [] # optional @@ -98,17 +111,12 @@ def check_data_member_float(self, local_var, var_name): self.add_error_msg("{v} is empty (float)".format(v=var_name)) elif isinstance(local_var, list): if isinstance(local_var[0], str) and not self.is_number(local_var[0]): - self.add_error_msg("Not supported data type ({v} string) for {n}[0] (int or float only)".format( + self.add_error_msg("Not supported data type: {n}[0]={v}, string type, not a number (int or float only)".format( n=var_name, v=local_var[0])) - elif 0 <= str(type(local_var[0])).find('numpy'): - self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format( - v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var[0], (int, float)): - # self.add_error_msg("Not supported data type ({t}) for {v}[0] (int or float only)".format( - # v=var_name, t=local_var[0].type)) + elif 0 > str(type(local_var[0])).find('numpy') and not isinstance(local_var[0], (int, float)): self.add_error_msg("Not supported data type ({t}) for {v}[0] (int or float only)".format( v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var, np.ndarray): + elif not self.is_numpy_array(local_var): self.add_error_msg("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format( v=var_name, t=type(local_var))) @@ -117,15 +125,12 @@ def check_data_member_int(self, local_var, var_name): self.add_error_msg("{v} is empty (int)".format(v=var_name)) elif isinstance(local_var, list): if isinstance(local_var[0], str) and not self.is_number(local_var[0]): - self.add_error_msg("Not supported data type ({v} string) for {n}[0] (int or float only)".format( + self.add_error_msg("Not supported data type: {n}[0]={v}, string type, not a number (int only)".format( n=var_name, v=local_var[0])) - elif 0 <= str(type(local_var[0])).find('numpy'): - self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format( - v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var[0], int): + elif 0 > str(type(local_var[0])).find('numpy') and not isinstance(local_var[0], int): self.add_error_msg("Not supported data type ({t}) for {v}[0] (int only)".format( v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var, np.ndarray): + elif not self.is_numpy_array(local_var): self.add_error_msg("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format( v=var_name, t=type(local_var))) @@ -160,6 +165,15 @@ def check_point_data(self): if self.use_var_id: self.check_data_member_string(self.obs_var_table,'obs_var_table') + def convert_to_numpy(self, value_list): + return np.array(value_list) + + def dump(self): + base_met_point_obs.print_point_data(self.get_point_data()) + + def get_count_string(self): + return f' nobs={self.nobs} nhdr={self.nhdr} ntyp={self.nhdr_typ} nsid={self.nhdr_sid} nvld={self.nhdr_vld} nqty={self.nobs_qty} nvar={self.nobs_var}' + def get_point_data(self): if self.nhdr <= 0: self.nhdr = len(self.hdr_lat) @@ -178,34 +192,109 @@ def get_point_data(self): if self.nobs_var <= 0: self.nobs_var = len(self.obs_var_table) self.check_point_data() - return self.__dict__ - def get_type(self, value): - return 'string' if isinstance('str') else type(value) + if not self.is_numpy_array(self.hdr_typ): + self.hdr_typ = self.convert_to_numpy(self.hdr_typ) + if not self.is_numpy_array(self.hdr_sid): + self.hdr_sid = self.convert_to_numpy(self.hdr_sid) + if not self.is_numpy_array(self.hdr_vld): + self.hdr_vld = self.convert_to_numpy(self.hdr_vld) + if not self.is_numpy_array(self.hdr_lat): + self.hdr_lat = self.convert_to_numpy(self.hdr_lat) + if not self.is_numpy_array(self.hdr_lon): + self.hdr_lon = self.convert_to_numpy(self.hdr_lon) + if not self.is_numpy_array(self.hdr_elv): + self.hdr_elv = self.convert_to_numpy(self.hdr_elv) + + if not self.is_numpy_array(self.obs_qty): + self.obs_qty = self.convert_to_numpy(self.obs_qty) + if not self.is_numpy_array(self.obs_hid): + self.obs_hid = self.convert_to_numpy(self.obs_hid) + if not self.is_numpy_array(self.obs_vid): + self.obs_vid = self.convert_to_numpy(self.obs_vid) + if not self.is_numpy_array(self.obs_lvl): + self.obs_lvl = self.convert_to_numpy(self.obs_lvl) + if not self.is_numpy_array(self.obs_hgt): + self.obs_hgt = self.convert_to_numpy(self.obs_hgt) + if not self.is_numpy_array(self.obs_val): + self.obs_val = self.convert_to_numpy(self.obs_val) + + self.count_info = self.get_count_string() + self.met_point_data = self + return self.__dict__ def is_number(self, num_str): return num_str.replace('-','1').replace('+','2').replace('.','3').isdigit() + def is_numpy_array(self, var): + return isinstance(var, np.ndarray) + def log_error_msg(self, err_msg): - print('{p} {m}'.format(p=self.ERROR_P, m=err_msg)) + base_met_point_obs.error_msg(err_msg) def log_error(self, err_msgs): print(self.ERROR_P) for err_line in err_msgs.split('\n'): - print('{p} {m}'.format(p=self.ERROR_P, m=err_line)) + self.log_error_msg(err_line) print(self.ERROR_P) def log_info(self, info_msg): - print('{p} {m}'.format(p=self.INFO_P, m=info_msg)) + base_met_point_obs.info_msg(info_msg) + + def put_data(self, point_obs_dict): + self.use_var_id = point_obs_dict['use_var_id'] + self.hdr_typ = point_obs_dict['hdr_typ'] + self.hdr_sid = point_obs_dict['hdr_sid'] + self.hdr_vld = point_obs_dict['hdr_vld'] + self.hdr_lat = point_obs_dict['hdr_lat'] + self.hdr_lon = point_obs_dict['hdr_lon'] + self.hdr_elv = point_obs_dict['hdr_elv'] + self.hdr_typ_table = point_obs_dict['hdr_typ_table'] + self.hdr_sid_table = point_obs_dict['hdr_sid_table'] + self.hdr_vld_table = point_obs_dict['hdr_vld_table'] + + #Observation data + self.obs_qty = point_obs_dict['obs_qty'] + self.obs_hid = point_obs_dict['obs_hid'] + self.obs_lvl = point_obs_dict['obs_lvl'] + self.obs_hgt = point_obs_dict['obs_hgt'] + self.obs_val = point_obs_dict['obs_val'] + self.obs_vid = point_obs_dict['obs_vid'] + self.obs_var_table = point_obs_dict['obs_var_table'] + self.obs_qty_table = point_obs_dict['obs_qty_table'] + po_array = point_obs_dict.get('obs_unit', None) + if po_array is not None: + self.obs_var_unit = po_array + po_array = point_obs_dict.get('obs_desc', None) + if po_array is not None: + self.obs_var_desc = po_array + + po_array = point_obs_dict.get('hdr_prpt_typ', None) + if po_array is not None: + self.hdr_prpt_typ = po_array + po_array = point_obs_dict.get('hdr_irpt_typ', None) + if po_array is not None: + self.hdr_irpt_typ = po_array + po_array = point_obs_dict.get('hdr_inst_typ', None) + if po_array is not None: + self.hdr_inst_typ = po_array @staticmethod - def is_python_script(arg_value): - return arg_value.startswith(met_point_obs.python_prefix) + def error_msg(msg): + print(f'{get_prompt()} {base_met_point_obs.ERROR_P} {msg}') + + @staticmethod + def info_msg(msg): + print(f'{get_prompt()} {base_met_point_obs.INFO_P} {msg}') @staticmethod def get_python_script(arg_value): return arg_value[len(met_point_obs.python_prefix)+1:] + @staticmethod + def is_python_script(arg_value): + return arg_value.startswith(met_point_obs.python_prefix) + @staticmethod def print_data(key, data_array, show_count=COUNT_SHOW): if isinstance(data_array, list): @@ -274,22 +363,18 @@ def print_point_data(met_point_data, print_subset=True): class csv_point_obs(ABC, base_met_point_obs): def __init__(self, point_data): - super(csv_point_obs, self).__init__() - - hdr_cnt = obs_cnt = len(point_data) self.point_data = point_data - self.nhdr = self.nobs = obs_cnt - self.nhdr_typ = self.nhdr_sid = self.nhdr_vld = hdr_cnt - self.nobs_qty = self.nobs_var = obs_cnt - self.input_file = None - self.ignore_input_file = True + super(csv_point_obs, self).__init__() + self.obs_cnt = obs_cnt = len(point_data) self.obs_qty = [ 0 for _ in range(0, obs_cnt) ] # (nobs_qty) integer, index of self.obs_qty_table self.obs_hid = [ 0 for _ in range(0, obs_cnt) ] # (nobs) integer self.obs_vid = [ 0 for _ in range(0, obs_cnt) ] # (nobs) integer, veriable index from self.obs_var_table or GRIB code - self.obs_lvl = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float - self.obs_hgt = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float - self.obs_val = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_lvl = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_hgt = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_val = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + + self.convert_point_data() def check_csv_record(self, csv_point_data, index): error_msgs = [] @@ -362,35 +447,36 @@ def convert_point_data(self): hdr_vld_map = {} obs_var_map = {} obs_qty_map = {} + self.use_var_id = not self.is_grib_code() index = 0 - # Build map - # names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] - for csv_point_data in self.point_data: - hdr_typ_str = csv_point_data[0] + #names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] + for csv_point_record in self.point_data: + # Build header map. + hdr_typ_str = csv_point_record[0] hdr_typ_idx = hdr_typ_map.get(hdr_typ_str,-1) if hdr_typ_idx < 0: hdr_typ_idx = hdr_typ_cnt hdr_typ_map[hdr_typ_str] = hdr_typ_idx hdr_typ_cnt += 1 - hdr_sid_str = csv_point_data[1] + hdr_sid_str = csv_point_record[1] hdr_sid_idx = hdr_sid_map.get(hdr_sid_str,-1) if hdr_sid_idx < 0: hdr_sid_idx = hdr_sid_cnt hdr_sid_map[hdr_sid_str] = hdr_sid_idx hdr_sid_cnt += 1 - hdr_vld_str = csv_point_data[2] + hdr_vld_str = csv_point_record[2] hdr_vld_idx = hdr_vld_map.get(hdr_vld_str,-1) if hdr_vld_idx < 0: hdr_vld_idx = hdr_vld_cnt hdr_vld_map[hdr_vld_str] = hdr_vld_idx hdr_vld_cnt += 1 - lat = csv_point_data[3] - lon = csv_point_data[4] - elv = self.get_num_value(csv_point_data[5] ) + lat = csv_point_record[3] + lon = csv_point_record[4] + elv = self.get_num_value(csv_point_record[5] ) hdr_key = (hdr_typ_idx,hdr_sid_idx,hdr_vld_idx,lat,lon,elv) hdr_idx = hdr_map.get(hdr_key,-1) if hdr_idx < 0: @@ -398,7 +484,7 @@ def convert_point_data(self): hdr_map[hdr_key] = hdr_idx hdr_cnt += 1 - var_id_str = csv_point_data[6] + var_id_str = csv_point_record[6] if self.use_var_id: var_id = obs_var_map.get(var_id_str,-1) if var_id < 0: @@ -408,7 +494,7 @@ def convert_point_data(self): else: var_id = int(var_id_str) - qc_str = csv_point_data[9] + qc_str = csv_point_record[9] qc_id = obs_qty_map.get(qc_str,-1) if qc_id < 0: qc_id = qc_cnt @@ -418,9 +504,9 @@ def convert_point_data(self): # names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] self.obs_vid[index] = var_id self.obs_hid[index] = hdr_idx - self.obs_lvl[index] = self.get_num_value(csv_point_data[7]) - self.obs_hgt[index] = self.get_num_value(csv_point_data[8]) - self.obs_val[index] = self.get_num_value(csv_point_data[10]) + self.obs_lvl[index] = self.get_num_value(csv_point_record[7]) + self.obs_hgt[index] = self.get_num_value(csv_point_record[8]) + self.obs_val[index] = self.get_num_value(csv_point_record[10]) self.obs_qty[index] = qc_id index += 1 @@ -436,9 +522,9 @@ def convert_point_data(self): self.hdr_typ = [ 0 for _ in range(0, hdr_cnt) ] self.hdr_sid = [ 0 for _ in range(0, hdr_cnt) ] self.hdr_vld = [ 0 for _ in range(0, hdr_cnt) ] - self.hdr_lat = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] - self.hdr_lon = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] - self.hdr_elv = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_lat = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_lon = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_elv = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] for key, idx in hdr_map.items(): self.hdr_typ[idx] = key[0] self.hdr_sid[idx] = key[1] @@ -463,16 +549,15 @@ def convert_point_data(self): for key, idx in obs_var_map.items(): self.obs_var_table[idx] = key - return self.get_point_data() - def get_num_value(self, column_value): num_value = column_value if isinstance(column_value, str): - if column_value.lower() == 'na': - num_value = nc_tools.FILL_VALUE - elif self.is_number(column_value): + if self.is_number(column_value): num_value = float(column_value) - num_value = nc_tools.FILL_VALUE + else: + num_value = self.FILL_VALUE + if column_value.lower() != 'na' and column_value.lower() != 'n/a': + self.log_info(f'{column_value} is not a number, converted to the missing value') return num_value def is_grib_code(self): @@ -488,7 +573,7 @@ def is_grib_code(self): def is_num_string(self, column_value): is_string = isinstance(column_value, str) if is_string: - is_num = True if self.is_number(column_value) or column_value.lower() == 'na' else False + is_num = True if self.is_number(column_value) or column_value.lower() == 'na' or column_value.lower() == 'n/a' else False else: is_num = True return is_string, is_num @@ -496,6 +581,8 @@ def is_num_string(self, column_value): class met_point_obs(ABC, base_met_point_obs): + MET_ENV_RUN = 'MET_FORCE_TO_RUN' + @abstractmethod def read_data(self, args): # args can be input_file_name, list, or dictionary @@ -514,31 +601,13 @@ def read_data(self, args): ''' pass - -# Note: caller should import netCDF4 -# the argements nc_group(dataset) and nc_var should not be None -class nc_tools(): - - FILL_VALUE = -9999. - met_missing = -99999999. - - @staticmethod - def get_num_array(nc_group, var_name): - nc_var = nc_group.variables.get(var_name, None) - #return [] if nc_var is None else nc_var[:].filled(nc_var._FillValue if '_FillValue' in nc_var.ncattrs() else nc_tools.met_missing) - return [] if nc_var is None else nc_var[:] - @staticmethod - def get_ncbyte_array_to_str(nc_var): - nc_str_data = nc_var[:] - if nc_var.datatype.name == 'bytes8': - nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:] ] - return nc_str_data + def get_prompt(): + return get_prompt() @staticmethod - def get_string_array(nc_group, var_name): - nc_var = nc_group.variables.get(var_name, None) - return [] if nc_var is None else nc_tools.get_ncbyte_array_to_str(nc_var) + def is_python_prefix(user_cmd): + return user_cmd.startswith(base_met_point_obs.python_prefix) # This is a sample drived class @@ -568,12 +637,15 @@ def read_data(self, arg_map={}): self.obs_var_table = [ "TMP", "RH" ] self.obs_qty_table = [ "NA" ] -def convert_point_data(point_data, check_all_records=False): - _csv_point_data = csv_point_obs(point_data) - if _csv_point_data.is_grib_code(): - _csv_point_data.use_var_id = False - _csv_point_data.check_csv_point_data(check_all_records) - return _csv_point_data.convert_point_data() +def convert_point_data(point_data, check_all_records=False, input_type='csv'): + tmp_point_data = {} + if 'csv' == input_type: + csv_point_data = csv_point_obs(point_data) + csv_point_data.check_csv_point_data(check_all_records) + tmp_point_data = csv_point_data.get_point_data() + else: + base_met_point_obs.error_msg('Not supported input type: {input_type}') + return tmp_point_data def main(): args = {} # or args = [] diff --git a/scripts/python/met_point_obs_nc.py b/scripts/python/met_point_obs_nc.py new file mode 100644 index 0000000000..e6680c0689 --- /dev/null +++ b/scripts/python/met_point_obs_nc.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 + +''' +Separated from read_met_point_obs on Feb 09, 2023 + +@author: hsoh + +This script reads the MET point observation NetCDF file like MET tools do. +''' + +import os +import sys +from datetime import datetime +import numpy as np +import netCDF4 as nc + +from met_point_obs import met_point_obs, base_met_point_obs, get_prompt + +DO_PRINT_DATA = False +ARG_PRINT_DATA = 'show_data' + +# Note: caller should import netCDF4 +# the argements nc_group(dataset) and nc_var should not be None +class nc_tools(): + + met_missing = -99999999. + + @staticmethod + def get_num_array(nc_group, var_name): + nc_var = nc_group.variables.get(var_name, None) + return [] if nc_var is None else nc_var[:] + + @staticmethod + def get_ncbyte_array_to_str(nc_var): + nc_str_data = nc_var[:] + if nc_var.datatype.name == 'bytes8': + nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:] ] + return nc_str_data + + @staticmethod + def get_string_array(nc_group, var_name): + nc_var = nc_group.variables.get(var_name, None) + return [] if nc_var is None else nc_tools.get_ncbyte_array_to_str(nc_var) + + +class nc_point_obs(met_point_obs): + + # args should be string, list, or dictionary + def get_nc_filename(self, args): + nc_filename = None + if isinstance(args, dict): + nc_filename = args.get('nc_name',None) + elif isinstance(args, list): + nc_filename = args[0] + elif args != ARG_PRINT_DATA: + nc_filename = args + + return nc_filename + + def read_data(self, nc_filename): + if nc_filename is None: + self.log_error_msg("The input NetCDF filename is missing") + elif not os.path.exists(nc_filename): + self.log_error_msg(f"input NetCDF file ({nc_filename}) does not exist") + else: + dataset = nc.Dataset(nc_filename, 'r') + + attr_name = 'use_var_id' + use_var_id_str = dataset.getncattr(attr_name) if attr_name in dataset.ncattrs() else "false" + self.use_var_id = use_var_id_str.lower() == 'true' + + # Header + self.hdr_typ = dataset['hdr_typ'][:] + self.hdr_sid = dataset['hdr_sid'][:] + self.hdr_vld = dataset['hdr_vld'][:] + self.hdr_lat = dataset['hdr_lat'][:] + self.hdr_lon = dataset['hdr_lon'][:] + self.hdr_elv = dataset['hdr_elv'][:] + self.hdr_typ_table = nc_tools.get_string_array(dataset, 'hdr_typ_table') + self.hdr_sid_table = nc_tools.get_string_array(dataset, 'hdr_sid_table') + self.hdr_vld_table = nc_tools.get_string_array(dataset, 'hdr_vld_table') + + nc_var = dataset.variables.get('obs_unit', None) + if nc_var: + self.obs_var_unit = nc_var[:] + nc_var = dataset.variables.get('obs_desc', None) + if nc_var: + self.obs_var_desc = nc_var[:] + + nc_var = dataset.variables.get('hdr_prpt_typ', None) + if nc_var: + self.hdr_prpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_irpt_typ', None) + if nc_var: + self.hdr_irpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_inst_typ', None) + if nc_var: + self.hdr_inst_typ =nc_var[:] + + #Observation data + self.hdr_sid = dataset['hdr_sid'][:] + self.obs_qty = np.array(dataset['obs_qty'][:]) + self.obs_hid = np.array(dataset['obs_hid'][:]) + self.obs_lvl = np.array(dataset['obs_lvl'][:]) + self.obs_hgt = np.array(dataset['obs_hgt'][:]) + self.obs_val = np.array(dataset['obs_val'][:]) + nc_var = dataset.variables.get('obs_vid', None) + if nc_var is None: + self.use_var_id = False + nc_var = dataset.variables.get('obs_gc', None) + else: + self.obs_var_table = nc_tools.get_string_array(dataset, 'obs_var') + if nc_var: + self.obs_vid = np.array(nc_var[:]) + + self.obs_qty_table = nc_tools.get_string_array(dataset, 'obs_qty_table') + + def save_ncfile(self, nc_filename): + met_data = self.get_point_data() + with nc.Dataset(nc_filename, 'w') as nc_dataset: + self.set_nc_data(nc_dataset) + return met_data + + def set_nc_data(self, nc_dataset): + return nc_point_obs.write_nc_data(nc_dataset, self) + + @staticmethod + def write_nc_file(nc_filename, point_obs): + with nc.Dataset(nc_filename, 'w') as nc_dataset: + nc_point_obs.set_nc_data(nc_dataset, point_obs) + + @staticmethod + def write_nc_data(nc_dataset, point_obs): + do_nothing = False + if 0 == point_obs.nhdr: + do_nothing = True + base_met_point_obs.info_msg("the header is empty") + if 0 == point_obs.nobs: + do_nothing = True + base_met_point_obs.info_msg("the observation data is empty") + if do_nothing: + print() + return + + # Set global attributes + nc_dataset.MET_Obs_version = "1.02" ; + nc_dataset.use_var_id = "true" if point_obs.use_var_id else "false" + + # Create dimensions + nc_dataset.createDimension('mxstr', 16) + nc_dataset.createDimension('mxstr2', 40) + nc_dataset.createDimension('mxstr3', 80) + nc_dataset.createDimension('nhdr', point_obs.nhdr) + nc_dataset.createDimension('nobs', point_obs.nobs) + #npbhdr = len(point_obs.hdr_prpt_typ) + if 0 < point_obs.npbhdr: + nc_dataset.createDimension('npbhdr', point_obs.npbhdr) + nc_dataset.createDimension('nhdr_typ', point_obs.nhdr_typ) + nc_dataset.createDimension('nhdr_sid', point_obs.nhdr_sid) + nc_dataset.createDimension('nhdr_vld', point_obs.nhdr_vld) + nc_dataset.createDimension('nobs_qty', point_obs.nobs_qty) + nc_dataset.createDimension('obs_var_num', point_obs.nobs_var) + + type_for_string = 'S1' # np.byte + dims_hdr = ('nhdr',) + dims_obs = ('nobs',) + + # Create header and observation variables + var_hdr_typ = nc_dataset.createVariable('hdr_typ', np.int32, dims_hdr, fill_value=-9999) + var_hdr_sid = nc_dataset.createVariable('hdr_sid', np.int32, dims_hdr, fill_value=-9999) + var_hdr_vld = nc_dataset.createVariable('hdr_vld', np.int32, dims_hdr, fill_value=-9999) + var_hdr_lat = nc_dataset.createVariable('hdr_lat', np.float32, dims_hdr, fill_value=-9999.) + var_hdr_lon = nc_dataset.createVariable('hdr_lon', np.float32, dims_hdr, fill_value=-9999.) + var_hdr_elv = nc_dataset.createVariable('hdr_elv', np.float32, dims_hdr, fill_value=-9999.) + + var_obs_qty = nc_dataset.createVariable('obs_qty', np.int32, dims_obs, fill_value=-9999) + var_obs_hid = nc_dataset.createVariable('obs_hid', np.int32, dims_obs, fill_value=-9999) + var_obs_vid = nc_dataset.createVariable('obs_vid', np.int32, dims_obs, fill_value=-9999) + var_obs_lvl = nc_dataset.createVariable('obs_lvl', np.float32, dims_obs, fill_value=-9999.) + var_obs_hgt = nc_dataset.createVariable('obs_hgt', np.float32, dims_obs, fill_value=-9999.) + var_obs_val = nc_dataset.createVariable('obs_val', np.float32, dims_obs, fill_value=-9999.) + + if 0 == point_obs.npbhdr: + var_hdr_prpt_typ = None + var_hdr_irpt_typ = None + var_hdr_inst_typ = None + else: + dims_npbhdr = ('npbhdr',) + var_hdr_prpt_typ = nc_dataset.createVariable('hdr_prpt_typ', np.int32, dims_npbhdr, fill_value=-9999.) + var_hdr_irpt_typ = nc_dataset.createVariable('hdr_irpt_typ', np.int32, dims_npbhdr, fill_value=-9999.) + var_hdr_inst_typ = nc_dataset.createVariable('hdr_inst_typ', np.int32, dims_npbhdr, fill_value=-9999.) + + var_hdr_typ_table = nc_dataset.createVariable('hdr_typ_table', type_for_string, ('nhdr_typ','mxstr2')) + var_hdr_sid_table = nc_dataset.createVariable('hdr_sid_table', type_for_string, ('nhdr_sid','mxstr2')) + var_hdr_vld_table = nc_dataset.createVariable('hdr_vld_table', type_for_string, ('nhdr_vld','mxstr')) + var_obs_qty_table = nc_dataset.createVariable('obs_qty_table', type_for_string, ('nobs_qty','mxstr')) + var_obs_var_table = nc_dataset.createVariable('obs_var', type_for_string, ('obs_var_num','mxstr2')) + var_obs_var_unit = nc_dataset.createVariable('obs_unit', type_for_string, ('obs_var_num','mxstr2')) + var_obs_var_desc = nc_dataset.createVariable('obs_desc', type_for_string, ('obs_var_num','mxstr3')) + + # Set variables + var_hdr_typ[:] = point_obs.hdr_typ[:] + var_hdr_sid[:] = point_obs.hdr_sid[:] + var_hdr_vld[:] = point_obs.hdr_vld[:] + var_hdr_lat[:] = point_obs.hdr_lat[:] + var_hdr_lon[:] = point_obs.hdr_lon[:] + var_hdr_elv[:] = point_obs.hdr_elv[:] + for i in range(0, point_obs.nhdr_typ): + for j in range(0, len(point_obs.hdr_typ_table[i])): + var_hdr_typ_table[i,j] = point_obs.hdr_typ_table[i][j] + for i in range(0, point_obs.nhdr_sid): + for j in range(0, len(point_obs.hdr_sid_table[i])): + var_hdr_sid_table[i,j] = point_obs.hdr_sid_table[i][j] + for i in range(0, point_obs.nhdr_vld): + for j in range(0, len(point_obs.hdr_vld_table[i])): + var_hdr_vld_table[i,j] = point_obs.hdr_vld_table[i][j] + if 0 < point_obs.npbhdr: + var_hdr_prpt_typ[:] = point_obs.hdr_prpt_typ[:] + var_hdr_irpt_typ[:] = point_obs.hdr_irpt_typ[:] + var_hdr_inst_typ[:] = point_obs.hdr_inst_typ[:] + + var_obs_qty[:] = point_obs.obs_qty[:] + var_obs_hid[:] = point_obs.obs_hid[:] + var_obs_vid[:] = point_obs.obs_vid[:] + var_obs_lvl[:] = point_obs.obs_lvl[:] + var_obs_hgt[:] = point_obs.obs_hgt[:] + var_obs_val[:] = point_obs.obs_val[:] + for i in range(0, point_obs.nobs_var): + for j in range(0, len(point_obs.obs_var_table[i])): + var_obs_var_table[i,j] = point_obs.obs_var_table[i][j] + var_obs_var_unit[i] = "" if i >= len(point_obs.obs_var_unit) else point_obs.obs_var_unit[i] + var_obs_var_desc[i] = "" if i >= len(point_obs.obs_var_desc) else point_obs.obs_var_desc[i] + for i in range(0, point_obs.nobs_qty): + for j in range(0, len(point_obs.obs_qty_table[i])): + var_obs_qty_table[i,j] = point_obs.obs_qty_table[i][j] + + # Set variable attributes + var_hdr_typ.long_name = "index of message type" + var_hdr_sid.long_name = "index of station identification" + var_hdr_vld.long_name = "index of valid time" + var_hdr_lat.long_name = "latitude" + var_hdr_lat.units = "degrees_north" + var_hdr_lon.long_name = "longitude" + var_hdr_lon.units = "degrees_east" + var_hdr_elv.long_name = "elevation" + var_hdr_elv.units = "meters above sea level (msl)" + + var_obs_qty.long_name = "index of quality flag" + var_obs_hid.long_name = "index of matching header data" + var_obs_vid.long_name = "index of BUFR variable corresponding to the observation type" + var_obs_lvl.long_name = "pressure level (hPa) or accumulation interval (sec)" + var_obs_hgt.long_name = "height in meters above sea level (msl)" + var_obs_val.long_name = "observation value" + var_hdr_typ_table.long_name = "message type" + var_hdr_sid_table.long_name = "station identification" + var_hdr_vld_table.long_name = "valid time" + var_hdr_vld_table.units = "YYYYMMDD_HHMMSS UTC" + var_obs_qty_table.long_name = "quality flag" + var_obs_var_table.long_name = "variable names" + var_obs_var_unit.long_name = "variable units" + var_obs_var_desc.long_name = "variable descriptions" + + +def main(argv): + if len(argv) != 1 and argv[1] != ARG_PRINT_DATA: + netcdf_filename = argv[1] + tmp_nc_name = 'tmp_met_point.nc' + point_obs_data = nc_point_obs() + point_obs_data.read_data(point_obs_data.get_nc_filename(netcdf_filename)) + met_point_data = point_obs_data.save_ncfile(tmp_nc_name) + print(f'{get_prompt()} saved met_point_data to {tmp_nc_name}') + met_point_data['met_point_data'] = point_obs_data + + if DO_PRINT_DATA or ARG_PRINT_DATA == argv[-1]: + met_point_obs.print_point_data(met_point_data) + +if __name__ == '__main__': + start_time = datetime.now() + main(sys.argv) + run_time = datetime.now() - start_time + print(f'{get_prompt()} Done python script {sys.argv[0]} took {run_time}') diff --git a/scripts/python/read_met_point_obs.py b/scripts/python/read_met_point_obs.py index 60f2c62065..57ccd22e7a 100755 --- a/scripts/python/read_met_point_obs.py +++ b/scripts/python/read_met_point_obs.py @@ -1,121 +1,91 @@ +#!/usr/bin/env python3 ''' Created on Nov 10, 2021 @author: hsoh This script reads the MET point observation NetCDF file like MET tools do. + +Usage: + + python3 read_met_point_obs.py + python3 read_met_point_obs.py + : 11 columns + 'typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs' + string columns: 'typ', 'sid', 'vld', 'var', , 'qc' + numeric columns 'lat', 'lon', 'elv', 'lvl', 'hgt', 'qc', 'obs' + python3 read_met_point_obs.py + ''' import os import sys from datetime import datetime -import numpy as np -import netCDF4 as nc + +met_base_dir = os.getenv('MET_BASE',None) +if met_base_dir is not None: + sys.path.append(os.path.join(met_base_dir, 'python')) from met_point_obs import met_point_obs, sample_met_point_obs +from met_point_obs_nc import nc_point_obs DO_PRINT_DATA = False ARG_PRINT_DATA = 'show_data' -class nc_point_obs(met_point_obs): - - def read_data(self, args): - # args should be list or dictionaryu - if isinstance(args, dict): - nc_filename = args.get('nc_name',None) - elif isinstance(args, list): - nc_filename = args[0] - else: - nc_filename = args - if nc_filename is None: - print("==ERROR== The input NetCDF filename is missing") - elif not os.path.exists(nc_filename): - print("==ERROR== input NetCDF file ({f}) does not exist".format(f=nc_filename)) - else: - dataset = nc.Dataset(nc_filename, 'r') - # Header - self.hdr_typ = dataset['hdr_typ'][:] - self.hdr_sid = dataset['hdr_sid'][:] - self.hdr_vld = dataset['hdr_vld'][:] - self.hdr_lat = dataset['hdr_lat'][:] - self.hdr_lon = dataset['hdr_lon'][:] - self.hdr_elv = dataset['hdr_elv'][:] - self.hdr_typ_table = nc_tools.get_string_array(dataset, 'hdr_typ_table') - self.hdr_sid_table = nc_tools.get_string_array(dataset, 'hdr_sid_table') - self.hdr_vld_table = nc_tools.get_string_array(dataset, 'hdr_vld_table') - - nc_var = dataset.variables.get('hdr_prpt_typ', None) - if nc_var: - self.hdr_prpt_typ = nc_var[:] - nc_var = dataset.variables.get('hdr_irpt_typ', None) - if nc_var: - self.hdr_irpt_typ = nc_var[:] - nc_var = dataset.variables.get('hdr_inst_typ', None) - if nc_var: - self.hdr_inst_typ =nc_var[:] - - #Observation data - self.hdr_sid = dataset['hdr_sid'][:] - self.obs_qty = np.array(dataset['obs_qty'][:]) - self.obs_hid = np.array(dataset['obs_hid'][:]) - self.obs_lvl = np.array(dataset['obs_lvl'][:]) - self.obs_hgt = np.array(dataset['obs_hgt'][:]) - self.obs_val = np.array(dataset['obs_val'][:]) - nc_var = dataset.variables.get('obs_vid', None) - if nc_var is None: - self.use_var_id = False - nc_var = dataset.variables.get('obs_gc', None) - else: - self.obs_var_table = nc_tools.get_string_array(dataset, 'obs_var') - if nc_var: - self.obs_vid = np.array(nc_var[:]) - - self.obs_qty_table = nc_tools.get_string_array(dataset, 'obs_qty_table') - - -class nc_tools(): - - @staticmethod - def get_num_array(dataset, var_name): - nc_var = dataset.variables.get(var_name, None) - return nc_var[:].filled(nc_var._FillValue if '_FillValue' in nc_var.ncattrs() else -9999) if nc_var else [] - - @staticmethod - def get_ncbyte_array_to_str(nc_var): - nc_str_data = nc_var[:] - if nc_var.datatype.name == 'bytes8': - nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:]] - return nc_str_data - - @staticmethod - def get_string_array(dataset, var_name): - nc_var = dataset.variables.get(var_name, None) - return nc_tools.get_ncbyte_array_to_str(nc_var) if nc_var else [] - - start_time = datetime.now() +prompt = met_point_obs.get_prompt() point_obs_data = None -if len(sys.argv) == 1: +if len(sys.argv) == 1 or ARG_PRINT_DATA == sys.argv[1]: point_obs_data = sample_met_point_obs() point_obs_data.read_data([]) +elif met_point_obs.is_python_prefix(sys.argv[1]): + import importlib.util + + print("{p} Python Script:\t".format(p=prompt) + repr(sys.argv[0])) + print("{p} User Command:\t".format(p=prompt) + repr(' '.join(sys.argv[2:]))) + + pyembed_module_name = sys.argv[2] + sys.argv = sys.argv[1:] + + # append user script dir to system path + pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) + if pyembed_dir: + sys.path.insert(0, pyembed_dir) + + if not pyembed_module_name.endswith('.py'): + pyembed_module_name += '.py' + os.environ[met_point_obs.MET_ENV_RUN] = "TRUE" + + user_base = os.path.basename(pyembed_module_name).replace('.py','') + + spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) + met_in = importlib.util.module_from_spec(spec) + spec.loader.exec_module(met_in) + + met_point_obs = met_in.met_point_obs + print("met_point_obs: ", met_point_obs) + met_point_data = met_in.met_point_data + print("met_point_data: ", met_point_data) + #print(hasattr("met_in: ", dir(met_in))) + #met_point_data = met_point_obs.get_point_data() + #met_point_data = None if met_in.get('met_point_data', None) else met_in.met_point_data + #met_data = None if met_in.get('met_data', None) else met_in.met_data + print(met_point_data) else: netcdf_filename = sys.argv[1] args = [ netcdf_filename ] #args = { 'nc_name': netcdf_filename } point_obs_data = nc_point_obs() - point_obs_data.read_data(args) - - if ARG_PRINT_DATA == sys.argv[-1]: - DO_PRINT_DATA = True + point_obs_data.read_data(point_obs_data.get_nc_filename(args)) if point_obs_data is not None: met_point_data = point_obs_data.get_point_data() met_point_data['met_point_data'] = point_obs_data - if DO_PRINT_DATA: - met_point_obs.print_point_data(met_point_data) + if DO_PRINT_DATA or ARG_PRINT_DATA == sys.argv[-1]: + point_obs_data.dump() run_time = datetime.now() - start_time -print('Done python script {s} took {t}'.format(s=sys.argv[0], t=run_time)) +print('{p} Done python script {s} took {t}'.format(p=prompt, s=sys.argv[0], t=run_time)) diff --git a/src/libcode/vx_data2d_python/python_dataplane.cc b/src/libcode/vx_data2d_python/python_dataplane.cc index f588db9c03..af886d7efe 100644 --- a/src/libcode/vx_data2d_python/python_dataplane.cc +++ b/src/libcode/vx_data2d_python/python_dataplane.cc @@ -23,7 +23,7 @@ //////////////////////////////////////////////////////////////////////// -GlobalPython GP; // this needs external linkage +extern GlobalPython GP; // this needs external linkage //////////////////////////////////////////////////////////////////////// @@ -35,12 +35,8 @@ static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_datapla static const char read_tmp_nc [] = "read_tmp_dataplane"; // NO ".py" suffix -static const char tmp_nc_base_name [] = "tmp_met_nc"; - static const char tmp_nc_var_name [] = "met_info"; -static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; - //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/python_pointdata.cc b/src/libcode/vx_pointdata_python/python_pointdata.cc index e99cc49596..8dc462ee13 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.cc +++ b/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -13,15 +13,30 @@ #include "python_pointdata.h" #include "pointdata_from_array.h" #include "vx_util.h" +#include "vx_python3_utils.h" #include "global_python.h" #include "wchar_argv.h" //////////////////////////////////////////////////////////////////////// +extern GlobalPython GP; // this needs external linkage + //////////////////////////////////////////////////////////////////////// +static const char * user_ppath = 0; + +static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_point_nc.py"; + +static const char read_tmp_nc [] = "read_tmp_point_nc"; // NO ".py" suffix + +//////////////////////////////////////////////////////////////////////// + + +static bool tmp_nc_point_obs(const char * script_name, int user_script_argc, + char ** user_script_argv, MetPointDataPython &met_pd_out); + static bool straight_python_point_data(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, MetPointDataPython &met_pd_out); @@ -52,13 +67,174 @@ bool python_point_data(const char * script_name, int script_argc, char ** script { -bool status = straight_python_point_data(script_name, script_argc, script_argv, - use_xarray, met_pd_out); +bool status = false; + +if ( user_ppath == 0 ) user_ppath = getenv(user_python_path_env); + +if ( user_ppath != 0 ) { + // do_tmp_nc = true; + + status = tmp_nc_point_obs(script_name, script_argc, script_argv, + met_pd_out); +} +else { + + status = straight_python_point_data(script_name, script_argc, script_argv, + use_xarray, met_pd_out); +} return ( status ); } +//////////////////////////////////////////////////////////////////////// + +bool process_python_point_data(PyObject *module_obj, MetPointDataPython &met_pd_out) +{ + +int int_value; +PyObject *module_dict_obj = 0; +PyObject *python_value = 0; +PyObject *python_met_point_data = 0; +ConcatString cs, user_dir, user_base; +const char *method_name = "process_python_point_data -> "; +const char *method_name_s = "process_python_point_data()"; + + // + // get the namespace for the module (as a dictionary) + // + +module_dict_obj = PyModule_GetDict (module_obj); + + // + // get handles to the objects of interest from the module_dict + // + +python_met_point_data = PyDict_GetItemString (module_dict_obj, python_key_point_data); + +python_value = PyDict_GetItemString (python_met_point_data, python_use_var_id); + +bool use_var_id = pyobject_as_bool(python_value); +met_pd_out.set_use_var_id(use_var_id); +mlog << Debug(9) << method_name << "use_var_id: \"" << use_var_id + << "\" from python. is_using_var_id(): " << met_pd_out.is_using_var_id() << "\n"; + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nhdr); + +int_value = pyobject_as_int(python_value); +if (int_value == 0) { + mlog << Error << "\n" << method_name + << "The header is empty. Please check if python input exists\n\n"; + exit (1); +} +met_pd_out.set_hdr_cnt(int_value); + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nobs); +int_value = pyobject_as_int(python_value); +if (int_value == 0) { + mlog << Error << "\n" << method_name + << "The point observation data is empty. Please check if python input is processed properly\n\n"; + exit (1); +} + +met_pd_out.allocate(int_value); + +MetPointObsData *obs_data = met_pd_out.get_point_obs_data(); +MetPointHeader *header_data = met_pd_out.get_header_data(); + + + // look up the data array variable name from the dictionary + + set_array_from_python(python_met_point_data, numpy_array_hdr_typ, &header_data->typ_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_sid, &header_data->sid_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_vld, &header_data->vld_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lat, &header_data->lat_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lon, &header_data->lon_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_elv, &header_data->elv_array); + if (header_data->typ_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_typ is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->sid_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_sid is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->vld_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_vld is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->lat_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_lat is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->lon_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_lon is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->elv_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_elv is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + + set_str_array_from_python(python_met_point_data, numpy_array_hdr_typ_table, &header_data->typ_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_sid_table, &header_data->sid_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_vld_table, &header_data->vld_array); + if (header_data->typ_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_typ_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->sid_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_sid_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->vld_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_vld_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + set_array_from_python(python_met_point_data, numpy_array_prpt_typ_table, &header_data->prpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_irpt_typ_table, &header_data->irpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_inst_typ_table, &header_data->inst_typ_array, false); + + set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids); + set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); + set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); + set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); + set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); + set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); + + set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); + set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); + if (obs_data->qty_names.n() == 0) { + mlog << Error << "\n" << method_name + << "The obs_qty_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (use_var_id && obs_data->var_names.n() == 0) { + mlog << Error << "\n" << method_name + << "The obs_var_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + + if(mlog.verbosity_level()>=point_data_debug_level) print_met_data(obs_data, header_data, method_name_s); + + // + // done + // + +return ( true ); + +} + + //////////////////////////////////////////////////////////////////////// @@ -75,8 +251,6 @@ ConcatString cs, user_dir, user_base; const char *method_name = "straight_python_point_data -> "; const char *method_name_s = "straight_python_point_data()"; -mlog << Debug(3) << "Running user's python script (" - << script_name << ").\n"; cs = script_name; user_dir = cs.dirname(); @@ -110,6 +284,11 @@ if ( PyErr_Occurred() ) { } + +mlog << Debug(3) << "Running MET compile time python instance (" + << MET_PYTHON_BIN_EXE << ") to run user's python script (" + << script_name << ").\n"; + // // set the arguments // @@ -170,153 +349,175 @@ if ( ! module_obj ) { } + +return process_python_point_data(module_obj, met_pd_out); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool tmp_nc_point_obs(const char * user_script_name, int user_script_argc, + char ** user_script_argv, MetPointDataPython &met_pd_out) + +{ + +int j; +int status; +ConcatString command; +ConcatString path; +ConcatString tmp_nc_path; +const char * tmp_dir = 0; +Wchar_Argv wa; +const char *method_name = "tmp_nc_point_obs() -> "; + // - // get the namespace for the module (as a dictionary) + // if the global python object has already been initialized, + // we need to reload the module // -module_dict_obj = PyModule_GetDict (module_obj); +bool do_reload = GP.is_initialized; + +GP.initialize(); // - // get handles to the objects of interest from the module_dict + // start up the python interpreter // -python_met_point_data = PyDict_GetItemString (module_dict_obj, python_key_point_data); +if ( PyErr_Occurred() ) { -python_value = PyDict_GetItemString (python_met_point_data, python_use_var_id); + PyErr_Print(); -bool use_var_id = pyobject_as_bool(python_value); -met_pd_out.set_use_var_id(use_var_id); -mlog << Debug(9) << method_name << "use_var_id: \"" << use_var_id << "\" from python. is_using_var_id(): " << met_pd_out.is_using_var_id() << "\n"; + mlog << Warning << "\n" << method_name + << "an error occurred initializing python\n\n"; -python_value = PyDict_GetItemString (python_met_point_data, python_key_nhdr); + return ( false ); -int_value = pyobject_as_int(python_value); -if (int_value == 0) { - mlog << Error << "\n" << method_name - << "The header is empty. Please check if python input exists\n\n"; - exit (1); } -met_pd_out.set_hdr_cnt(int_value); -python_value = PyDict_GetItemString (python_met_point_data, python_key_nobs); -int_value = pyobject_as_int(python_value); -if (int_value == 0) { +run_python_string("import sys"); +command << cs_erase + << "sys.path.append(\"" + << replace_path(python_dir) + << "\")"; +run_python_string(command.text()); + +mlog << Debug(3) << "Running user-specified python instance (MET_PYTHON_EXE=" << user_ppath + << ") to run user's python script (" << user_script_name << ").\n"; + + +tmp_dir = getenv ("MET_TMP_DIR"); + +if ( ! tmp_dir ) tmp_dir = default_tmp_dir; + +path << cs_erase + << tmp_dir << '/' + << tmp_nc_base_name; + +tmp_nc_path = make_temp_file_name(path.text(), 0); + +command << cs_erase + << user_ppath << ' ' // user's path to python + << replace_path(write_tmp_nc) << ' ' // write_tmp_nc.py + << tmp_nc_path << ' ' // tmp_nc output filename + << user_script_name; // user's script name + +for (j=1; jtyp_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_sid, &header_data->sid_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_vld, &header_data->vld_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_lat, &header_data->lat_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_lon, &header_data->lon_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_elv, &header_data->elv_array); - if (header_data->typ_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_typ is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->sid_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_sid is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->vld_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_vld is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->lat_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_lat is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->lon_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_lon is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->elv_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_elv is empty. Please check if python input is processed properly\n\n"; - exit (1); - } + module_obj = PyImport_ReloadModule (module_obj); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_typ_table, &header_data->typ_array); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_sid_table, &header_data->sid_array); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_vld_table, &header_data->vld_array); - if (header_data->typ_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_typ_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->sid_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_sid_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->vld_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_vld_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - set_array_from_python(python_met_point_data, numpy_array_prpt_typ_table, &header_data->prpt_typ_array, false); - set_array_from_python(python_met_point_data, numpy_array_irpt_typ_table, &header_data->irpt_typ_array, false); - set_array_from_python(python_met_point_data, numpy_array_inst_typ_table, &header_data->inst_typ_array, false); +} - set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids); - set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); - set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); - set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); - set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); - set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); +if ( PyErr_Occurred() ) { - set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); - set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); - if (obs_data->qty_names.n() == 0) { - mlog << Error << "\n" << method_name - << "The obs_qty_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (use_var_id && obs_data->var_names.n() == 0) { - mlog << Error << "\n" << method_name - << "The obs_var_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - - if(mlog.verbosity_level()>=point_data_debug_level) print_met_data(obs_data, header_data, method_name_s); + PyErr_Print(); + + mlog << Warning << "\n" << method_name + << "an error occurred importing module " + << '\"' << path << "\"\n\n"; + + return ( false ); } +if ( ! module_obj ) { + + mlog << Warning << "\n" << method_name + << "error running python script\n\n"; + + return ( false ); + +} + + // + // read the tmp_nc file + // + + // + // get the namespace for the module (as a dictionary) + // + + +process_python_point_data(module_obj, met_pd_out); + + + // + // cleanup + // + +remove_temp_file(tmp_nc_path); + // // done // diff --git a/src/libcode/vx_python3_utils/global_python.h b/src/libcode/vx_python3_utils/global_python.h index 50dbe22986..ca8c9e8e26 100644 --- a/src/libcode/vx_python3_utils/global_python.h +++ b/src/libcode/vx_python3_utils/global_python.h @@ -61,7 +61,7 @@ inline void GlobalPython::initialize() if ( ! is_initialized ) { - mlog << Debug(3) << "Initializing python: " << MET_PYTHON_BIN_EXE << "\n"; + mlog << Debug(3) << "Initializing MET compile time python instance: " << MET_PYTHON_BIN_EXE << "\n"; wchar_t *python_path = Py_DecodeLocale(MET_PYTHON_BIN_EXE, NULL); Py_SetProgramName(python_path); diff --git a/src/libcode/vx_python3_utils/python3_util.cc b/src/libcode/vx_python3_utils/python3_util.cc index ab17acd7b8..f454c837b1 100644 --- a/src/libcode/vx_python3_utils/python3_util.cc +++ b/src/libcode/vx_python3_utils/python3_util.cc @@ -15,8 +15,13 @@ using namespace std; #include "vx_math.h" #include "python3_util.h" +#include "global_python.h" +//////////////////////////////////////////////////////////////////////// + +GlobalPython GP; // this needs external linkage + //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_python3_utils/python3_util.h b/src/libcode/vx_python3_utils/python3_util.h index 9464142566..e81dfe260d 100644 --- a/src/libcode/vx_python3_utils/python3_util.h +++ b/src/libcode/vx_python3_utils/python3_util.h @@ -32,6 +32,11 @@ static const char wrappers_dir [] = "MET_BASE/wrappers"; static const char python_dir [] = "MET_BASE/python"; +static const char tmp_nc_base_name [] = "tmp_met_nc"; + +static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; + +static const char tmp_nc_point_var_name[] = "met_point_data"; //////////////////////////////////////////////////////////////////////// @@ -58,7 +63,7 @@ extern PyObject * get_attribute(PyObject *, const char * attribute_name); extern int pyobject_as_int (PyObject *); -extern bool pyobject_as_bool (PyObject *); +extern bool pyobject_as_bool (PyObject *); extern double pyobject_as_double (PyObject *); extern std::string pyobject_as_string (PyObject *); extern ConcatString pyobject_as_concat_string (PyObject *);