From 443882d8ef59335aac3c53fe0e68d7f19fbf4ce8 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Tue, 6 Jun 2023 11:51:59 +0200 Subject: [PATCH 1/4] Update for new smecv grid --- .github/workflows/ci.yml | 5 +- environment.yml | 5 +- setup.cfg | 4 +- src/esa_cci_sm/interface.py | 42 ++++++---- .../metadata/esa_cci_sm_v08_ACTIVE.ini | 82 +++++++++++++++++++ .../metadata/esa_cci_sm_v08_COMBINED.ini | 82 +++++++++++++++++++ .../metadata/esa_cci_sm_v08_PASSIVE.ini | 79 ++++++++++++++++++ src/esa_cci_sm/reshuffle.py | 9 +- tests/test_grid.py | 56 ------------- tests/test_interface_v033.py | 29 +++---- tests/test_interface_v042.py | 55 +++++++------ tests/test_interface_v052.py | 53 +++++++----- 12 files changed, 355 insertions(+), 146 deletions(-) create mode 100644 src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini create mode 100644 src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini create mode 100644 src/esa_cci_sm/metadata/esa_cci_sm_v08_PASSIVE.ini delete mode 100644 tests/test_grid.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 280abf7..bedc003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,12 +29,13 @@ jobs: lfs: true - name: Checkout LFS objects run: git lfs checkout - - uses: conda-incubator/setup-miniconda@v2.0.1 + - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" auto-update-conda: true python-version: ${{ matrix.python-version }} environment-file: environment.yml + mamba-version: "*" activate-environment: esa_cci_sm auto-activate-base: false - name: Print environment infos @@ -116,4 +117,4 @@ jobs: verify_metadata: true packages_dir: Artifacts/dist/ user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} # todo: add pypi token to GHA \ No newline at end of file + password: ${{ secrets.PYPI_API_TOKEN }} # todo: add pypi token to GHA diff --git a/environment.yml b/environment.yml index 109e623..ea4725f 100644 --- a/environment.yml +++ b/environment.yml @@ -5,7 +5,6 @@ channels: - conda-forge - defaults dependencies: - - python>=3.8 - pandas - netcdf4 - pyresample @@ -16,8 +15,8 @@ dependencies: - repurpose - parse - configparser - - smecv-grid==0.2.1 - - pygeogrids==0.3.0 + - smecv-grid + - pygeogrids - more_itertools - sphinx_rtd_theme - pytest diff --git a/setup.cfg b/setup.cfg index a63b2ef..d7d286b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,8 +40,8 @@ install_requires = numpy pynetcf repurpose parse - pygeogrids<=0.3.0 - smecv_grid<=0.2.1 + pygeogrids + smecv_grid more_itertools # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov diff --git a/src/esa_cci_sm/interface.py b/src/esa_cci_sm/interface.py index d7cf5d7..4c8e4f6 100644 --- a/src/esa_cci_sm/interface.py +++ b/src/esa_cci_sm/interface.py @@ -20,6 +20,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import warnings import numpy as np import os @@ -41,15 +42,16 @@ class CCI_SM_025Img(ImageBase): Parameters ---------- - filename: string + filename: str Filename of the ESA CCI SM netcdf file - mode: string, optional (default: 'r') + mode: str, optional (default: 'r') Mode of opening the file, only 'r' is implemented at the moment - parameter : string or list, optional (default: 'sm') + parameter : str or list[str,...], optional (default: 'sm') One or list of parameters to read, see ESA CCI documentation for more information array_1D: boolean, optional (default: False) - If set then the data is read into 1D arrays. Needed for some legacy code. + If set then the data is read into 1D arrays. Where the first element + refers to the lower left data point in the 2d image. """ def __init__(self, filename, mode='r', parameter=None, subgrid=None, @@ -62,16 +64,26 @@ def __init__(self, filename, mode='r', parameter=None, subgrid=None, self.array_1D = array_1D def read(self, timestamp=None): - # Returns the selected parameters for a ESA CCI SM image and according metadata + """ + Read data from loaded netcdf file. + + Parameters + ---------- + timestamp: datetime + Time stamp for this image. + + Returns + ------- + img: Image + Image object as implemented in pygeobase + """ return_img = {} return_metadata = {} try: dataset = Dataset(self.filename) except IOError as e: - print(e) - print("{}: Cannot open file".format(self.filename)) - raise e + raise IOError(f"Could not open file {self.filename}: {e}") if self.parameters is None: param_names = [p for p in dataset.variables.keys() if p not in ['time', 'lat', 'lon']] @@ -102,22 +114,24 @@ def read(self, timestamp=None): return_img[parameter] except KeyError: path, thefile = os.path.split(self.filename) - print ('%s in %s is corrupt - filling image with NaN values' % (parameter, thefile)) + warnings.warn(f"{parameter} in {thefile} is corrupt - " + f"filling image with NaN values") return_img[parameter] = np.empty(self.grid.n_gpi).fill(np.nan) dataset.close() + if self.array_1D: return Image(self.grid.activearrlon, self.grid.activearrlat, return_img, return_metadata, timestamp) else: yres, xres = self.grid.shape for key in return_img: - return_img[key] = return_img[key].reshape((xres, yres)) + return_img[key] = return_img[key].reshape((yres, xres)) return Image( - self.grid.activearrlon.reshape((xres, yres)), - self.grid.activearrlat.reshape((xres, yres)), - return_img, + self.grid.activearrlon.reshape((yres, xres)), + np.flipud(self.grid.activearrlat.reshape((yres, xres))), + {k: np.flipud(v) for k, v in return_img.items()}, return_metadata, timestamp) @@ -225,5 +239,3 @@ def __init__(self, ts_path, grid_path=None, **kwargs): grid = load_grid(grid_path) super(CCITs, self).__init__(ts_path, grid, **kwargs) - - diff --git a/src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini b/src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini new file mode 100644 index 0000000..b1fc42a --- /dev/null +++ b/src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini @@ -0,0 +1,82 @@ + +# global attributes + +[GLOBAL] +product_version : 8 +title : ESA CCI Surface Soil Moisture merged ACTIVE Product +institution : TU Wien (AUT); VanderSat B.V. (NL), Planet Labs (NL); CESBIO (FR); EODC Gmbh (AUT) +contact : cci_sm_contact@eodc.eu +references : http://www.esa-soilmoisture-cci.org; Dorigo, W.A., Wagner, W., Albergel, C., Albrecht, F., Balsamo, G., Brocca, L., Chung, D., Ertl, M., Forkel, M., Gruber, A., Haas, E., Hamer, D. P. Hirschi, M., Ikonen, J., De Jeu, R. Kidd, R. Lahoz, W., Liu, Y.Y., Miralles, D., Lecomte, P. (2017) ESA CCI Soil Moisture for improved Earth system understanding: State-of-the art and future directions. In Remote Sensing of Environment, 2017, ISSN 0034-4257, https://doi.org/10.1016/j.rse.2017.07.001; Gruber, A., Scanlon, T., van der Schalie, R., Wagner, W., Dorigo, W. (2019) Evolution of the ESA CCI Soil Moisture Climate Data Records and their underlying merging methodology. Earth System Science Data 11, 717-739, https://doi.org/10.5194/essd-11-717-2019; Gruber, A., Dorigo, W. A., Crow, W., Wagner W. (2017). Triple Collocation-Based Merging of Satellite Soil Moisture Retrievals. IEEE Transactions on Geoscience and Remote Sensing. PP. 1-13. https://doi.org/10.1109/TGRS.2017.2734070 +standard_name_vocabulary : NetCDF Climate and Forecast (CF) Metadata Convention +keywords : Soil Moisture/Water Content +naming_authority : TU Wien +keywords_vocabulary : NASA Global Change Master Directory (GCMD) Science Keywords +creator_name : Department of Geodesy and Geoinformation, Technical University of Vienna +creator_url : https://climers.geo.tuwien.ac.at +creator_email : CCI_SM_CONTACT@EODC.EU +project : Climate Change Initiative - European Space Agency +license : Data use is free and open for all registered users. +time_coverage_duration : P41Y +time_coverage_resolution : P1D +geospatial_lat_min : -90.0 +geospatial_lat_max : 90.0 +geospatial_lon_min : -180.0 +geospatial_lon_max : 180.0 +geospatial_vertical_min : 0.0 +geospatial_vertical_max : 0.0 +geospatial_lat_units : degrees_north +geospatial_lon_units : degrees_east +geospatial_lat_resolution : 0.25 degree +geospatial_lon_resolution : 0.25 degree +spatial_resolution: 25km + +[DNFLAG] +long_name : Day / Night Flag +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, day, night, day_night_combination +_FillValue : 0 + +[FLAG] +long_name : Flag +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000 +bit_meanings: no_data_inconsistency_detected, snow_coverage_or_temperature_below_zero, dense_vegetation, others_no_convergence_in_the_model_thus_no_valid_sm_estimates, soil_moisture_value_exceeds_physical_boundary, weight_of_measurement_below_threshold, all_datasets_deemed_unreliable, barren_ground_advisory_flag, not_used; +_FillValue : 127 + +[MODE] +long_name : Satellite Mode +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, ascending, descending, ascending_descending_combination +_FillValue : 0 + +[SENSOR] +long_name : Sensor +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000, 0b1000000000, 0b10000000000, 0b100000000000, 0b1000000000000, 0b10000000000000, 0b100000000000000, 0b1000000000000000, 0b10000000000000000 +bit_meanings: NaN, SMMR, SSMI, TMI, AMSRE, WindSat, AMSR2, SMOS, AMIWS, ASCATA, ASCATB, SMAP, MODEL, GPM, FY3B, FY3D, ASCATC, FY3C +_FillValue : 0 + +[SM] +long_name : Percent of Saturation Soil Moisture +units : percent +valid_range : 0. 100. +_FillValue : -9999.0 + +[SM_UNCERTAINTY] +long_name : Percent of Saturation Soil Moisture Uncertainty +description : Uncertainty is the estimated standard deviation of the observation +units : percent +valid_range : 0. 100. +_FillValue : -9999.0 + +[FREQBANDID] # attention this changed for earlier versions +long_name : Frequency Band Identification +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000 +bit_meanings: NaN, L14, C53, C66, C68, C69, C73, X107, K194, MODEL +_FillValue : 0. + +[T0] +long_name : Observation Timestamp +units : days since 1970-01-01 00:00:00 UTC +_FillValue : -9999.0 + + + diff --git a/src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini b/src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini new file mode 100644 index 0000000..ee347dc --- /dev/null +++ b/src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini @@ -0,0 +1,82 @@ + +# global attributes + +[GLOBAL] +product_version : 8 +title : ESA CCI Surface Soil Moisture merged COMBINED Product +institution : TU Wien (AUT); VanderSat B.V. (NL), Planet Labs (NL); CESBIO (FR); EODC Gmbh (AUT) +contact : cci_sm_contact@eodc.eu +references : http://www.esa-soilmoisture-cci.org; Dorigo, W.A., Wagner, W., Albergel, C., Albrecht, F., Balsamo, G., Brocca, L., Chung, D., Ertl, M., Forkel, M., Gruber, A., Haas, E., Hamer, D. P. Hirschi, M., Ikonen, J., De Jeu, R. Kidd, R. Lahoz, W., Liu, Y.Y., Miralles, D., Lecomte, P. (2017) ESA CCI Soil Moisture for improved Earth system understanding: State-of-the art and future directions. In Remote Sensing of Environment, 2017, ISSN 0034-4257, https://doi.org/10.1016/j.rse.2017.07.001; Gruber, A., Scanlon, T., van der Schalie, R., Wagner, W., Dorigo, W. (2019) Evolution of the ESA CCI Soil Moisture Climate Data Records and their underlying merging methodology. Earth System Science Data 11, 717-739, https://doi.org/10.5194/essd-11-717-2019; Gruber, A., Dorigo, W. A., Crow, W., Wagner W. (2017). Triple Collocation-Based Merging of Satellite Soil Moisture Retrievals. IEEE Transactions on Geoscience and Remote Sensing. PP. 1-13. https://doi.org/10.1109/TGRS.2017.2734070 +standard_name_vocabulary : NetCDF Climate and Forecast (CF) Metadata Convention +keywords : Soil Moisture/Water Content +naming_authority : TU Wien +keywords_vocabulary : NASA Global Change Master Directory (GCMD) Science Keywords +creator_name : Department of Geodesy and Geoinformation, Technical University of Vienna +creator_url : https://climers.geo.tuwien.ac.at +creator_email : CCI_SM_CONTACT@EODC.EU +project : Climate Change Initiative - European Space Agency +license : Data use is free and open for all registered users. +time_coverage_duration : P41Y +time_coverage_resolution : P1D +geospatial_lat_min : -90.0 +geospatial_lat_max : 90.0 +geospatial_lon_min : -180.0 +geospatial_lon_max : 180.0 +geospatial_vertical_min : 0.0 +geospatial_vertical_max : 0.0 +geospatial_lat_units : degrees_north +geospatial_lon_units : degrees_east +geospatial_lat_resolution : 0.25 degree +geospatial_lon_resolution : 0.25 degree +spatial_resolution: 25km + +[DNFLAG] +long_name : Day / Night Flag +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, day, night, day_night_combination +_FillValue : 0 + +[FLAG] +long_name : Flag +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000 +bit_meanings: no_data_inconsistency_detected, snow_coverage_or_temperature_below_zero, dense_vegetation, others_no_convergence_in_the_model_thus_no_valid_sm_estimates, soil_moisture_value_exceeds_physical_boundary, weight_of_measurement_below_threshold, all_datasets_deemed_unreliable, barren_ground_advisory_flag, not_used; +_FillValue : 127 + +[MODE] +long_name : Satellite Mode +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, ascending, descending, ascending_descending_combination +_FillValue : 0 + +[SENSOR] +long_name : Sensor +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000, 0b1000000000, 0b10000000000, 0b100000000000, 0b1000000000000, 0b10000000000000, 0b100000000000000, 0b1000000000000000, 0b10000000000000000 +bit_meanings: NaN, SMMR, SSMI, TMI, AMSRE, WindSat, AMSR2, SMOS, AMIWS, ASCATA, ASCATB, SMAP, MODEL, GPM, FY3B, FY3D, ASCATC, FY3C +_FillValue : 0 + +[SM] +long_name : Volumetric Soil Moisture +units : m3 m-3 +valid_range : 0. 1. +_FillValue : -9999.0 + +[SM_UNCERTAINTY] +long_name : Volumetric Soil Moisture Uncertainty +description : Uncertainty is the estimated standard deviation of the observation +units : m3 m-3 +valid_range : 0. 1. +_FillValue : -9999.0 + +[FREQBANDID] # attention this changed for earlier versions +long_name : Frequency Band Identification +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000 +bit_meanings: NaN, L14, C53, C66, C68, C69, C73, X107, K194, MODEL +_FillValue : 0. + +[T0] +long_name : Observation Timestamp +units : days since 1970-01-01 00:00:00 UTC +_FillValue : -9999.0 + + + diff --git a/src/esa_cci_sm/metadata/esa_cci_sm_v08_PASSIVE.ini b/src/esa_cci_sm/metadata/esa_cci_sm_v08_PASSIVE.ini new file mode 100644 index 0000000..13ab589 --- /dev/null +++ b/src/esa_cci_sm/metadata/esa_cci_sm_v08_PASSIVE.ini @@ -0,0 +1,79 @@ +[GLOBAL] +product_version : 8 +title : ESA CCI Surface Soil Moisture merged PASSIVE Product +institution : TU Wien (AUT); VanderSat B.V. (NL), Planet Labs (NL); CESBIO (FR); EODC Gmbh (AUT) +contact : cci_sm_contact@eodc.eu +references : http://www.esa-soilmoisture-cci.org; Dorigo, W.A., Wagner, W., Albergel, C., Albrecht, F., Balsamo, G., Brocca, L., Chung, D., Ertl, M., Forkel, M., Gruber, A., Haas, E., Hamer, D. P. Hirschi, M., Ikonen, J., De Jeu, R. Kidd, R. Lahoz, W., Liu, Y.Y., Miralles, D., Lecomte, P. (2017) ESA CCI Soil Moisture for improved Earth system understanding: State-of-the art and future directions. In Remote Sensing of Environment, 2017, ISSN 0034-4257, https://doi.org/10.1016/j.rse.2017.07.001; Gruber, A., Scanlon, T., van der Schalie, R., Wagner, W., Dorigo, W. (2019) Evolution of the ESA CCI Soil Moisture Climate Data Records and their underlying merging methodology. Earth System Science Data 11, 717-739, https://doi.org/10.5194/essd-11-717-2019; Gruber, A., Dorigo, W. A., Crow, W., Wagner W. (2017). Triple Collocation-Based Merging of Satellite Soil Moisture Retrievals. IEEE Transactions on Geoscience and Remote Sensing. PP. 1-13. https://doi.org/10.1109/TGRS.2017.2734070 +standard_name_vocabulary : NetCDF Climate and Forecast (CF) Metadata Convention +keywords : Soil Moisture/Water Content +naming_authority : TU Wien +keywords_vocabulary : NASA Global Change Master Directory (GCMD) Science Keywords +creator_name : Department of Geodesy and Geoinformation, Technical University of Vienna +creator_url : https://climers.geo.tuwien.ac.at +creator_email : cci_sm_contact@eodc.eu +project : Climate Change Initiative - European Space Agency +license : Data use is free and open for all registered users. +time_coverage_duration : P41Y +time_coverage_resolution : P1D +geospatial_lat_min : -90.0 +geospatial_lat_max : 90.0 +geospatial_lon_min : -180.0 +geospatial_lon_max : 180.0 +geospatial_vertical_min : 0.0 +geospatial_vertical_max : 0.0 +geospatial_lat_units : degrees_north +geospatial_lon_units : degrees_east +geospatial_lat_resolution : 0.25 degree +geospatial_lon_resolution : 0.25 degree +spatial_resolution: 25km + +[DNFLAG] +long_name : Day / Night Flag +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, day, night, day_night_combination +_FillValue : 0 + +[FLAG] +long_name : Flag +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000 +bit_meanings: no_data_inconsistency_detected, snow_coverage_or_temperature_below_zero, dense_vegetation, others_no_convergence_in_the_model_thus_no_valid_sm_estimates, soil_moisture_value_exceeds_physical_boundary, weight_of_measurement_below_threshold, all_datasets_deemed_unreliable, barren_ground_advisory_flag, not_used; +_FillValue : 127 + +[MODE] +long_name : Satellite Mode +flag_values : 0, 1, 2, 3 +flag_meanings : NaN, ascending, descending, ascending_descending_combination +_FillValue : 0 + +[SENSOR] +long_name : Sensor +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000, 0b1000000000, 0b10000000000, 0b100000000000, 0b1000000000000, 0b10000000000000, 0b100000000000000, 0b1000000000000000, 0b10000000000000000 +bit_meanings: NaN, SMMR, SSMI, TMI, AMSRE, WindSat, AMSR2, SMOS, AMIWS, ASCATA, ASCATB, SMAP, MODEL, GPM, FY3B, FY3D, ASCATC, FY3C +_FillValue : 0 + +[SM] +long_name : Volumetric Soil Moisture +units : m3 m-3 +valid_range : 0. 1. +_FillValue : -9999.0 + +[SM_UNCERTAINTY] +long_name : Volumetric Soil Moisture Uncertainty +description : Uncertainty is the estimated standard deviation of the observation +units : m3 m-3 +valid_range : 0. 1. +_FillValue : -9999.0 + +[FREQBANDID] # attention this changed for earlier versions +long_name : Frequency Band Identification +bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000 +bit_meanings: NaN, L14, C53, C66, C68, C69, C73, X107, K194, MODEL +_FillValue : 0. + +[T0] +long_name : Observation Timestamp +units : days since 1970-01-01 00:00:00 UTC +_FillValue : -9999.0 + + + diff --git a/src/esa_cci_sm/reshuffle.py b/src/esa_cci_sm/reshuffle.py index bf1d85a..3f4dbbc 100644 --- a/src/esa_cci_sm/reshuffle.py +++ b/src/esa_cci_sm/reshuffle.py @@ -200,7 +200,7 @@ def reshuffle(input_root, outputpath, global_attr['time_coverage_start'] = str(startdate) global_attr['time_coverage_end'] = str(enddate) else: - global_attr = {'product' : 'ESA CCI SM'} + global_attr = {'product': 'ESA CCI SM'} ts_attributes = None @@ -213,7 +213,7 @@ def reshuffle(input_root, outputpath, def parse_args(args): - ''' + """ Parse command line parameters for conversion from image to timeseries Parameters @@ -225,7 +225,7 @@ def parse_args(args): ------- params : argparse.Namespace Command line parameters - ''' + """ parser = argparse.ArgumentParser( description="Convert ESA CCI image data to time series format.") @@ -275,6 +275,3 @@ def main(args): def run(): main(sys.argv[1:]) - - - diff --git a/tests/test_grid.py b/tests/test_grid.py deleted file mode 100644 index efddf23..0000000 --- a/tests/test_grid.py +++ /dev/null @@ -1,56 +0,0 @@ -# -*- coding: utf-8 -*- - -from esa_cci_sm.grid import CCICellGrid, CCILandGrid -import unittest -import numpy as np - -class GridTest(unittest.TestCase): - - def test_CCICellGrid(self): - grid = CCICellGrid() - gp, dist = grid.find_nearest_gpi(75.625, 14.625) - assert gp == 602942 - lon, lat = grid.gpi2lonlat(602942) - assert lon == 75.625 - assert lat == 14.625 - assert np.where(grid.get_grid_points()[0] == 602942)[0][0] == 434462 # index - assert grid.get_grid_points()[1][434462] == lon - assert grid.get_grid_points()[2][434462] == lat - assert grid.gpi2cell(602942) == 1856 - assert grid.gpis.size == 1036800 - assert grid.gpis[0] == 1035360 - assert np.unique(grid.get_grid_points()[3]).size == 2592 - - - lon, lat = grid.gpi2lonlat(642933) - assert lon == -6.625 - assert lat == 21.625 - assert np.where(grid.get_grid_points()[0] == 642933)[0][0] == 393813 # index - assert grid.get_grid_points()[1][393813] == lon - assert grid.get_grid_points()[2][393813] == lat - assert grid.gpi2cell(642933) == 1246 - - - - def test_landgrid(self): - grid = CCILandGrid() - gp, dist = grid.find_nearest_gpi(75.625, 14.625) - assert gp == 602942 - lon, lat = grid.gpi2lonlat(602942) - assert lon == 75.625 - assert lat == 14.625 - assert np.where(grid.get_grid_points()[0] == 602942)[0][0] == 177048 # index - assert grid.get_grid_points()[1][177048] == lon - assert grid.get_grid_points()[2][177048] == lat - assert grid.gpi2cell(602942) == 1856 - assert grid.gpis.size == 1036800 - assert grid.activegpis.size == 244243 - assert grid.gpis[0] == 1035360 - assert grid.activegpis[0] == 999942 - assert np.unique(grid.get_grid_points()[3]).size == 1001 - - - -if __name__ == '__main__': - unittest.main() - diff --git a/tests/test_interface_v033.py b/tests/test_interface_v033.py index 110e2ec..47be2d5 100644 --- a/tests/test_interface_v033.py +++ b/tests/test_interface_v033.py @@ -20,8 +20,7 @@ def test_CCI_SM_v033_025Ds_img_reading(): assert sorted(image_c.data.keys()) == sorted(parameter) assert image_c.timestamp == datetime(2016, 1, 1, 0) - - assert abs(image_c.data['sm'][273, 693] - 0.142998) <= 1e-5 + nptest.assert_almost_equal(image_c.data['sm'][273, 693], 0.142998, 5) assert image_c.lon.shape == image_c.lat.shape == (720, 1440) @@ -33,7 +32,7 @@ def test_CCI_SM_v033_025Ds_img_reading(): assert sorted(image_a.data.keys()) == sorted(parameter) assert image_a.timestamp == datetime(2016, 1, 1, 0) - assert abs(image_a.data['sm'][273, 693] - 18.92771) <= 1e-5 + nptest.assert_almost_equal(image_a.data['sm'][273, 693], 18.92771, 5) assert image_a.lon.shape == image_a.lat.shape == (720, 1440) @@ -46,7 +45,7 @@ def test_CCI_SM_v033_025Ds_img_reading(): assert sorted(image_p.data.keys()) == sorted(parameter) assert image_p.timestamp == datetime(2016, 1, 1, 0) - assert abs(image_p.data['sm'][273, 693] - 0.0700) <= 1e-5 + nptest.assert_almost_equal(image_p.data['sm'][273, 693], 0.0700, 5) assert image_p.lon.shape == image_p.lat.shape == (720, 1440) @@ -108,15 +107,15 @@ def test_CCI_SM_v033_025Img_img_reading_1D_combined(): image_c = img_c.read() - ref_lat = image_c.lat[1440 * 273 + 693] + ref_lat = image_c.lat[1440 * (719-273) + 693] ref_lon = image_c.lon[1440 * 273 + 693] assert ref_lon == -6.625 - assert ref_lat== 21.625 + assert ref_lat == 21.625 assert sorted(image_c.data.keys()) == sorted(parameter) - ref_sm = image_c.data['sm'][1440 * 273 + 693] + ref_sm = image_c.data['sm'][1440 * (719-273) + 693] nptest.assert_almost_equal(ref_sm, 0.142998, 5) ###### land grid @@ -150,15 +149,15 @@ def test_CCI_SM_v033_025Img_img_reading_1D_active(): image_a = img_a.read() - ref_lat = image_a.lat[1440 * 273 + 693] + ref_lat = image_a.lat[1440 * (719-273) + 693] ref_lon = image_a.lon[1440 * 273 + 693] assert ref_lon == -6.625 - assert ref_lat== 21.625 + assert ref_lat == 21.625 assert sorted(image_a.data.keys()) == sorted(parameter) - ref_sm = image_a.data['sm'][1440 * 273 + 693] + ref_sm = image_a.data['sm'][1440 * (719-273) + 693] nptest.assert_almost_equal(ref_sm, 18.92771, 5) image_a = img_a.read() @@ -194,7 +193,7 @@ def test_CCI_SM_v033_025Img_img_reading_1D_passive(): image_p = img_p.read() - ref_lat = image_p.lat[1440 * 273 + 693] + ref_lat = image_p.lat[1440 * (719-273) + 693] ref_lon = image_p.lon[1440 * 273 + 693] assert ref_lon == -6.625 @@ -202,7 +201,7 @@ def test_CCI_SM_v033_025Img_img_reading_1D_passive(): assert sorted(image_p.data.keys()) == sorted(parameter) - ref_sm = image_p.data['sm'][1440 * 273 + 693] + ref_sm = image_p.data['sm'][1440 * (719-273) + 693] nptest.assert_almost_equal(ref_sm, 0.0700, 5) @@ -248,7 +247,6 @@ def test_CCI_SM_v33_025Img_img_reading_2D(): assert image_c.lon.shape == image_c.lat.shape == (720, 1440) - parameter = ['sm'] img_a = CCI_SM_025Img( os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data", @@ -268,7 +266,6 @@ def test_CCI_SM_v33_025Img_img_reading_2D(): assert image_a.lon.shape == image_a.lat.shape == (720, 1440) - parameter = ['sm'] img_p = CCI_SM_025Img( os.path.join(os.path.dirname(__file__), "esa_cci_sm-test-data", @@ -286,7 +283,3 @@ def test_CCI_SM_v33_025Img_img_reading_2D(): assert image_p.lat[719, 0] == -89.875 assert abs(image_p.data['sm'][203, 693] - 0.322685) <= 1e-5 assert image_p.lon.shape == image_p.lat.shape == (720, 1440) - - - - diff --git a/tests/test_interface_v042.py b/tests/test_interface_v042.py index c9d826f..b91bc96 100644 --- a/tests/test_interface_v042.py +++ b/tests/test_interface_v042.py @@ -4,6 +4,7 @@ from esa_cci_sm.interface import CCI_SM_025Ds, CCI_SM_025Img from esa_cci_sm.grid import CCILandGrid +import numpy as np import numpy.testing as nptest @@ -26,7 +27,7 @@ def test_CCI_SM_v042_025Ds_img_reading(): assert sorted(image_c.data.keys()) == sorted(parameter) assert image_c.timestamp == datetime(2016, 6, 7, 0) - assert abs(image_c.data['sm'][122400] - 0.1755) <= 1e-3 # 1440 * 85 = 1222400 + assert abs(image_c.data['sm'][912960] - 0.1755) <= 1e-3 # 1440 * 634 = 912960 assert image_c.lon.shape == image_c.lat.shape == (720 * 1440,) @@ -40,7 +41,7 @@ def test_CCI_SM_v042_025Ds_img_reading(): assert sorted(image_a.data.keys()) == sorted(parameter) assert image_a.timestamp == datetime(2016, 6, 6, 0) - assert abs(image_a.data['sm'][133920] - 100) <= 1e-3 # 93*1440 + assert abs(image_a.data['sm'][901440] - 100) <= 1e-3 # 626*1440 assert image_a.lon.shape == (360 * 180 * (1 / 0.25)**2,) assert image_a.lon.shape == image_a.lat.shape @@ -54,7 +55,7 @@ def test_CCI_SM_v042_025Ds_img_reading(): assert sorted(image_p.data.keys()) == sorted(parameter) assert image_p.timestamp == datetime(2016, 6, 6, 0) - assert abs(image_p.data['sm'][123840] - 0.31) <= 1e-3 # 86*1440 + assert abs(image_p.data['sm'][911520] - 0.31) <= 1e-3 # 633*1440 assert image_p.lon.shape == (360 * 180 * (1 / 0.25)**2,) assert image_p.lon.shape == image_p.lat.shape @@ -124,13 +125,13 @@ def test_CCI_SM_v042_025Img_img_reading_1D_combined(): image_c = img_c.read() - ref_lat = image_c.lat[1440 * 85] # = 122400 - ref_lon = image_c.lon[1440 * 85] + ref_lat = image_c.lat[1440 * (719-85)] # = 912960 + ref_lon = image_c.lon[1440 * (719-85)] assert ref_lon == -179.875 assert ref_lat == 68.625 assert sorted(image_c.data.keys()) == sorted(parameter) - ref_sm = image_c.data['sm'][122400] + ref_sm = image_c.data['sm'][912960] assert abs(ref_sm - 0.1755) <= 1e-3 ###land_grid @@ -142,9 +143,10 @@ def test_CCI_SM_v042_025Img_img_reading_1D_combined(): image_c = img_c.read() - sm = image_c.data['sm'][24434] - lat = image_c.lat[24434] - lon = image_c.lon[24434] + idx = np.where(land_grid.activegpis == 912960)[0][0] + sm = image_c.data['sm'][idx] + lat = image_c.lat[idx] + lon = image_c.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -168,13 +170,13 @@ def test_CCI_SM_v042_025Img_img_reading_1D_active(): image_a = img_a.read() - ref_lat = image_a.lat[1440 * 93] # = 133920 - ref_lon = image_a.lon[1440 * 93] + ref_lat = image_a.lat[1440 * (719-93)] # = 901440 + ref_lon = image_a.lon[1440 * (719-93)] assert ref_lon == -179.875 assert ref_lat == 66.625 assert sorted(image_a.data.keys()) == sorted(parameter) - ref_sm = image_a.data['sm'][133920] + ref_sm = image_a.data['sm'][901440] assert abs(ref_sm - 100.) <= 1e-3 ###land_grid @@ -185,10 +187,11 @@ def test_CCI_SM_v042_025Img_img_reading_1D_active(): assert img_a.grid.find_nearest_gpi(-179.875, 66.625) == (901440, 0) image_a = img_a.read() + idx = np.where(land_grid.activegpis == 901440)[0][0] - sm = image_a.data['sm'][32830] - lat = image_a.lat[32830] - lon = image_a.lon[32830] + sm = image_a.data['sm'][idx] + lat = image_a.lat[idx] + lon = image_a.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -206,13 +209,13 @@ def test_CCI_SM_v042_025Img_img_reading_1D_passive(): image_p = img_p.read() - ref_lat = image_p.lat[1440 * 86] # = 123840 - ref_lon = image_p.lon[1440 * 86] + ref_lat = image_p.lat[1440 * (719-86)] # = 911520 + ref_lon = image_p.lon[1440 * (719-86)] assert ref_lon == -179.875 assert ref_lat == 68.375 assert sorted(image_p.data.keys()) == sorted(parameter) - ref_sm = image_p.data['sm'][123840] + ref_sm = image_p.data['sm'][911520] assert abs(ref_sm - 0.31) <= 1e-3 ###land_grid @@ -223,10 +226,11 @@ def test_CCI_SM_v042_025Img_img_reading_1D_passive(): assert img_a.grid.find_nearest_gpi(-179.875, 68.375) == (911520, 0) image_a = img_a.read() + idx = np.where(land_grid.activegpis == 911520)[0][0] - sm = image_a.data['sm'][25410] - lat = image_a.lat[25410] - lon = image_a.lon[25410] + sm = image_a.data['sm'][idx] + lat = image_a.lat[idx] + lon = image_a.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -300,5 +304,10 @@ def test_CCI_SM_v042_025Img_img_reading_2D(): assert image_p.lon.shape == (720, 1440) assert image_p.lon.shape == image_p.lat.shape - - +if __name__ == '__main__': + test_CCI_SM_v042_025Ds_img_reading() + test_CCI_SM_v042_025Img_img_reading_2D() + test_CCI_SM_v042_025Ds_timestamps_for_daterange() + test_CCI_SM_v042_025Img_img_reading_1D_active() + test_CCI_SM_v042_025Img_img_reading_1D_combined() + test_CCI_SM_v042_025Img_img_reading_1D_passive() diff --git a/tests/test_interface_v052.py b/tests/test_interface_v052.py index b9d1f11..6cc3d26 100644 --- a/tests/test_interface_v052.py +++ b/tests/test_interface_v052.py @@ -3,6 +3,7 @@ from datetime import datetime from esa_cci_sm.interface import CCI_SM_025Ds, CCI_SM_025Img from esa_cci_sm.grid import CCILandGrid +import numpy as np import numpy.testing as nptest def test_CCI_SM_v052_025Ds_img_reading(): @@ -104,21 +105,21 @@ def test_CCI_SM_v052_025Img_img_reading_1D_combined(): "esa_cci_sm_dailyImages", "v05.2", "combined", "2016", "ESACCI-SOILMOISTURE-L3S-SSMV-COMBINED-20160607000000-fv05.2.nc") - img_c = CCI_SM_025Img(filename=filename, parameter=parameter, subgrid=None, + reader = CCI_SM_025Img(filename=filename, parameter=parameter, subgrid=None, array_1D=True) - image_c = img_c.read() + image_c = reader.read() - assert img_c.grid.find_nearest_gpi(75.625, 14.625) == (602942, 0) + assert reader.grid.find_nearest_gpi(75.625, 14.625) == (602942, 0) - ref_lat = image_c.lat[434462] - ref_lon = image_c.lon[434462] + ref_lat = image_c.lat[1440 * (719-301) + 1022] + ref_lon = image_c.lon[1440 * (719-301) + 1022] assert ref_lon == 75.625 assert ref_lat== 14.625 assert sorted(image_c.data.keys()) == sorted(parameter) - ref_sm = image_c.data['sm'][1440 * 301 + 1022] + ref_sm = image_c.data['sm'][1440 * (719-301) + 1022] nptest.assert_almost_equal(ref_sm, 0.37016, 5) ###### land grid @@ -129,10 +130,11 @@ def test_CCI_SM_v052_025Img_img_reading_1D_combined(): assert img_c.grid.find_nearest_gpi(75.625, 14.625) == (602942, 0) image_c = img_c.read() + idx = np.where(land_grid.activegpis == 602942)[0][0] - sm = image_c.data['sm'][177048] - lat = image_c.lat[177048] - lon = image_c.lon[177048] + sm = image_c.data['sm'][idx] + lat = image_c.lat[idx] + lon = image_c.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -153,15 +155,15 @@ def test_CCI_SM_v052_025Img_img_reading_1D_active(): assert img_a.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0) - ref_lat = image_a.lat[1440 * 273 + 693] - ref_lon = image_a.lon[1440 * 273 + 693] + ref_lat = image_a.lat[1440 * (719-273) + 693] + ref_lon = image_a.lon[1440 * (719-273) + 693] assert ref_lon == -6.625 assert ref_lat== 21.625 assert sorted(image_a.data.keys()) == sorted(parameter) - ref_sm = image_a.data['sm'][1440 * 273 + 693] + ref_sm = image_a.data['sm'][1440 * (719-273) + 693] nptest.assert_almost_equal(ref_sm, 0.6519703, 5) ###### land grid @@ -170,12 +172,13 @@ def test_CCI_SM_v052_025Img_img_reading_1D_active(): array_1D=True) assert img_a.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0) + idx = np.where(land_grid.activegpis == 642933)[0][0] image_a = img_a.read() - sm = image_a.data['sm'][164759] - lat = image_a.lat[164759] - lon = image_a.lon[164759] + sm = image_a.data['sm'][idx] + lat = image_a.lat[idx] + lon = image_a.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -194,15 +197,15 @@ def test_CCI_SM_v052_025Img_img_reading_1D_passive(): assert img_p.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0) - ref_lat = image_p.lat[393813] - ref_lon = image_p.lon[393813] + ref_lat = image_p.lat[1440*(719-273)+693] + ref_lon = image_p.lon[1440*(719-273)+693] assert ref_lon == -6.625 assert ref_lat== 21.625 assert sorted(image_p.data.keys()) == sorted(parameter) - ref_sm = image_p.data['sm'][1440 * 273 + 693] + ref_sm = image_p.data['sm'][1440 * (719-273) + 693] nptest.assert_almost_equal(ref_sm, 0.0600, 5) ###### land grid @@ -213,10 +216,11 @@ def test_CCI_SM_v052_025Img_img_reading_1D_passive(): assert img_p.grid.find_nearest_gpi(-6.625, 21.625) == (642933, 0) image_p = img_p.read() + idx = np.where(land_grid.activegpis == 642933)[0][0] - sm = image_p.data['sm'][164759] - lat = image_p.lat[164759] - lon = image_p.lon[164759] + sm = image_p.data['sm'][idx] + lat = image_p.lat[idx] + lon = image_p.lon[idx] assert ref_lat == lat assert ref_lon == lon @@ -286,3 +290,10 @@ def test_CCI_SM_v052_025Img_img_reading_2D(): +if __name__ == '__main__': + test_CCI_SM_v052_025Ds_img_reading() + test_CCI_SM_v052_025Img_img_reading_2D() + test_CCI_SM_v052_025Ds_timestamps_for_daterange() + test_CCI_SM_v052_025Img_img_reading_1D_active() + test_CCI_SM_v052_025Img_img_reading_1D_combined() + test_CCI_SM_v052_025Img_img_reading_1D_passive() From afa7227cdc72227ddc6e9f78aebcd506fe53ecc1 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Tue, 6 Jun 2023 12:10:58 +0200 Subject: [PATCH 2/4] Update supported python versions --- .github/workflows/ci.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1a88eb..73c51c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.7', '3.8', '3.9'] + python-version: ['3.8', '3.9', '3.10'] os: ["ubuntu-latest", "windows-latest"] steps: diff --git a/setup.cfg b/setup.cfg index d7d286b..96f9370 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,7 +46,7 @@ install_requires = numpy # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov # Require a specific Python version, e.g. Python 2.7 or >= 3.4 -python_requires = >=3.6 +python_requires = >=3.8 [options.packages.find] where = src From 6f9620ea95dcbef7299df4444c8c9fd987fb417d Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Tue, 6 Jun 2023 12:17:49 +0200 Subject: [PATCH 3/4] Update changelog --- CHANGES.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b2dde77..f361976 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,22 +2,37 @@ Changelog ========= -Unreleased ... -============== +Unreleased changes (master) +=========================== + - +Version v0.4.0 +============== + +- Update for cci v8 +- Uses the new smecv_grid now (with latitudes sorted from - to +) + +Version v0.3.0 +============== + +- Update for cci v7 + Version v0.2.0 ============== + - Update for cci v6 Version v0.1.1 ============== + - Update time series reader class - Unmask smecv grid - Separate libnetcdf version for python3 Version v0.1 ============== + - pypi release - Homogenize classes for past ESA CCI SM versions - Update Readme and Documentation From bd7ff985e2655bddbe6c53291ef40f4865e5c5d7 Mon Sep 17 00:00:00 2001 From: Wolfgang Preimesberger Date: Tue, 6 Jun 2023 12:28:18 +0200 Subject: [PATCH 4/4] Set grid version --- environment.yml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index ea4725f..114e034 100644 --- a/environment.yml +++ b/environment.yml @@ -15,7 +15,7 @@ dependencies: - repurpose - parse - configparser - - smecv-grid + - smecv-grid>=0.3 - pygeogrids - more_itertools - sphinx_rtd_theme diff --git a/setup.cfg b/setup.cfg index 96f9370..cfc0a8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ install_requires = numpy repurpose parse pygeogrids - smecv_grid + smecv_grid>=0.3 more_itertools # The usage of test_requires is discouraged, see `Dependency Management` docs # tests_require = pytest; pytest-cov