Merge pull request #22 from TUW-GEO/v8_reader

V8 reader
TUW-GEO · Jun 6, 2023 · 33a8a45 · 33a8a45
2 parents 155f689 + bd7ff98
commit 33a8a45
Show file tree

Hide file tree

Showing 13 changed files with 373 additions and 149 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,7 +19,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.8', '3.9', '3.10']
         os: ["ubuntu-latest", "windows-latest"]
 
     steps:
@@ -30,12 +30,13 @@ jobs:
           lfs: true
       - name: Checkout LFS objects
         run: git lfs checkout
-      - uses: conda-incubator/setup-miniconda@v2.0.1
+      - uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           environment-file: environment.yml
+          mamba-version: "*"
           activate-environment: esa_cci_sm
           auto-activate-base: false
       - name: Print environment infos

diff --git a/CHANGES.rst b/CHANGES.rst
@@ -2,22 +2,37 @@
 Changelog
 =========
 
-Unreleased ...
-==============
+Unreleased changes (master)
+===========================
+
 -
 
+Version v0.4.0
+==============
+
+- Update for cci v8
+- Uses the new smecv_grid now (with latitudes sorted from - to +)
+
+Version v0.3.0
+==============
+
+- Update for cci v7
+
 Version v0.2.0
 ==============
+
 - Update for cci v6
 
 Version v0.1.1
 ==============
+
 - Update time series reader class
 - Unmask smecv grid
 - Separate libnetcdf version for python3
 
 Version v0.1
 ==============
+
 - pypi release
 - Homogenize classes for past ESA CCI SM versions
 - Update Readme and Documentation

diff --git a/environment.yml b/environment.yml
@@ -5,7 +5,6 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python>=3.8
   - pandas
   - netcdf4
   - pyresample
@@ -16,8 +15,8 @@ dependencies:
     - repurpose
     - parse
     - configparser
-    - smecv-grid==0.2.1
-    - pygeogrids==0.3.0
+    - smecv-grid>=0.3
+    - pygeogrids
     - more_itertools
     - sphinx_rtd_theme
     - pytest

diff --git a/setup.cfg b/setup.cfg
@@ -40,13 +40,13 @@ install_requires = numpy
                    pynetcf
                    repurpose
                    parse
-                   pygeogrids<=0.3.0
-                   smecv_grid<=0.2.1
+                   pygeogrids
+                   smecv_grid>=0.3
                    more_itertools
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 # tests_require = pytest; pytest-cov
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4
-python_requires = >=3.6
+python_requires = >=3.8
 
 [options.packages.find]
 where = src

diff --git a/src/esa_cci_sm/interface.py b/src/esa_cci_sm/interface.py
@@ -20,6 +20,7 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
+import warnings
 
 import numpy as np
 import os
@@ -41,15 +42,16 @@ class CCI_SM_025Img(ImageBase):
 
     Parameters
     ----------
-    filename: string
+    filename: str
         Filename of the ESA CCI SM netcdf file
-    mode: string, optional (default: 'r')
+    mode: str, optional (default: 'r')
         Mode of opening the file, only 'r' is implemented at the moment
-    parameter : string or list, optional (default: 'sm')
+    parameter : str or list[str,...], optional (default: 'sm')
         One or list of parameters to read, see ESA CCI documentation for
         more information
     array_1D: boolean, optional (default: False)
-        If set then the data is read into 1D arrays. Needed for some legacy code.
+        If set then the data is read into 1D arrays. Where the first element
+        refers to the lower left data point in the 2d image.
     """
 
     def __init__(self, filename, mode='r', parameter=None, subgrid=None,
@@ -62,16 +64,26 @@ def __init__(self, filename, mode='r', parameter=None, subgrid=None,
         self.array_1D = array_1D
 
     def read(self, timestamp=None):
-        # Returns the selected parameters for a ESA CCI SM image and according metadata
+        """
+        Read data from loaded netcdf file.
+
+        Parameters
+        ----------
+        timestamp: datetime
+            Time stamp for this image.
+
+        Returns
+        -------
+        img: Image
+           Image object as implemented in pygeobase
+        """
         return_img = {}
         return_metadata = {}
 
         try:
             dataset = Dataset(self.filename)
         except IOError as e:
-            print(e)
-            print("{}: Cannot open file".format(self.filename))
-            raise e
+            raise IOError(f"Could not open file {self.filename}: {e}")
 
         if self.parameters is None:
             param_names = [p for p in dataset.variables.keys() if p not in ['time', 'lat', 'lon']]
@@ -102,22 +114,24 @@ def read(self, timestamp=None):
                     return_img[parameter]
                 except KeyError:
                     path, thefile = os.path.split(self.filename)
-                    print ('%s in %s is corrupt - filling image with NaN values' % (parameter, thefile))
+                    warnings.warn(f"{parameter} in {thefile} is corrupt - "
+                                  f"filling image with NaN values")
                     return_img[parameter] = np.empty(self.grid.n_gpi).fill(np.nan)
 
         dataset.close()
+
         if self.array_1D:
             return Image(self.grid.activearrlon, self.grid.activearrlat,
                          return_img, return_metadata, timestamp)
         else:
             yres, xres = self.grid.shape
             for key in return_img:
-                return_img[key] = return_img[key].reshape((xres, yres))
+                return_img[key] = return_img[key].reshape((yres, xres))
 
             return Image(
-                self.grid.activearrlon.reshape((xres, yres)),
-                self.grid.activearrlat.reshape((xres, yres)),
-                return_img,
+                self.grid.activearrlon.reshape((yres, xres)),
+                np.flipud(self.grid.activearrlat.reshape((yres, xres))),
+                {k: np.flipud(v) for k, v in return_img.items()},
                 return_metadata,
                 timestamp)
 
@@ -225,5 +239,3 @@ def __init__(self, ts_path, grid_path=None, **kwargs):
 
         grid = load_grid(grid_path)
         super(CCITs, self).__init__(ts_path, grid, **kwargs)
-
-
diff --git a/src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini b/src/esa_cci_sm/metadata/esa_cci_sm_v08_ACTIVE.ini
@@ -0,0 +1,82 @@
+
+# global attributes
+
+[GLOBAL]
+product_version : 8
+title : ESA CCI Surface Soil Moisture merged ACTIVE Product
+institution : TU Wien (AUT); VanderSat B.V. (NL), Planet Labs (NL); CESBIO (FR); EODC Gmbh (AUT)
+contact : cci_sm_contact@eodc.eu
+references : http://www.esa-soilmoisture-cci.org; Dorigo, W.A., Wagner, W., Albergel, C., Albrecht, F.,  Balsamo, G., Brocca, L., Chung, D., Ertl, M., Forkel, M., Gruber, A., Haas, E., Hamer, D. P. Hirschi, M., Ikonen, J., De Jeu, R. Kidd, R. Lahoz, W., Liu, Y.Y., Miralles, D., Lecomte, P. (2017) ESA CCI Soil Moisture for improved Earth system understanding: State-of-the art and future directions. In Remote Sensing of Environment, 2017, ISSN 0034-4257, https://doi.org/10.1016/j.rse.2017.07.001; Gruber, A., Scanlon, T., van der Schalie, R., Wagner, W., Dorigo, W. (2019) Evolution of the ESA CCI Soil Moisture Climate Data Records and their underlying merging methodology. Earth System Science Data 11, 717-739, https://doi.org/10.5194/essd-11-717-2019; Gruber, A., Dorigo, W. A., Crow, W., Wagner W. (2017). Triple Collocation-Based Merging of Satellite Soil Moisture Retrievals. IEEE Transactions on Geoscience and Remote Sensing. PP. 1-13. https://doi.org/10.1109/TGRS.2017.2734070
+standard_name_vocabulary : NetCDF Climate and Forecast (CF) Metadata Convention
+keywords : Soil Moisture/Water Content
+naming_authority : TU Wien
+keywords_vocabulary : NASA Global Change Master Directory (GCMD) Science Keywords
+creator_name : Department of Geodesy and Geoinformation, Technical University of Vienna
+creator_url : https://climers.geo.tuwien.ac.at
+creator_email : CCI_SM_CONTACT@EODC.EU
+project : Climate Change Initiative - European Space Agency
+license : Data use is free and open for all registered users.
+time_coverage_duration : P41Y
+time_coverage_resolution : P1D
+geospatial_lat_min : -90.0
+geospatial_lat_max : 90.0
+geospatial_lon_min : -180.0
+geospatial_lon_max : 180.0
+geospatial_vertical_min : 0.0
+geospatial_vertical_max : 0.0
+geospatial_lat_units : degrees_north
+geospatial_lon_units : degrees_east
+geospatial_lat_resolution : 0.25 degree
+geospatial_lon_resolution : 0.25 degree
+spatial_resolution: 25km
+
+[DNFLAG]
+long_name : Day / Night Flag
+flag_values : 0, 1, 2, 3
+flag_meanings : NaN, day, night, day_night_combination
+_FillValue : 0
+
+[FLAG]
+long_name : Flag
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000
+bit_meanings: no_data_inconsistency_detected, snow_coverage_or_temperature_below_zero, dense_vegetation, others_no_convergence_in_the_model_thus_no_valid_sm_estimates, soil_moisture_value_exceeds_physical_boundary, weight_of_measurement_below_threshold, all_datasets_deemed_unreliable, barren_ground_advisory_flag, not_used;
+_FillValue : 127
+
+[MODE]
+long_name : Satellite Mode
+flag_values : 0, 1, 2, 3
+flag_meanings : NaN, ascending, descending, ascending_descending_combination
+_FillValue : 0
+
+[SENSOR]
+long_name :  Sensor
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000, 0b1000000000, 0b10000000000, 0b100000000000, 0b1000000000000, 0b10000000000000, 0b100000000000000, 0b1000000000000000, 0b10000000000000000
+bit_meanings: NaN, SMMR, SSMI, TMI, AMSRE, WindSat, AMSR2, SMOS, AMIWS, ASCATA, ASCATB, SMAP, MODEL, GPM, FY3B, FY3D, ASCATC, FY3C
+_FillValue : 0
+
+[SM]
+long_name : Percent of Saturation Soil Moisture
+units : percent
+valid_range : 0. 100.
+_FillValue : -9999.0
+
+[SM_UNCERTAINTY]
+long_name : Percent of Saturation Soil Moisture Uncertainty
+description : Uncertainty is the estimated standard deviation of the observation
+units : percent
+valid_range : 0. 100.
+_FillValue : -9999.0
+
+[FREQBANDID] # attention this changed for earlier versions
+long_name : Frequency Band Identification
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000
+bit_meanings: NaN, L14, C53, C66, C68, C69, C73, X107, K194, MODEL
+_FillValue : 0.
+
+[T0]
+long_name : Observation Timestamp
+units : days since 1970-01-01 00:00:00 UTC
+_FillValue : -9999.0
+
+
+
diff --git a/src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini b/src/esa_cci_sm/metadata/esa_cci_sm_v08_COMBINED.ini
@@ -0,0 +1,82 @@
+
+# global attributes
+
+[GLOBAL]
+product_version : 8
+title : ESA CCI Surface Soil Moisture merged COMBINED Product
+institution : TU Wien (AUT); VanderSat B.V. (NL), Planet Labs (NL); CESBIO (FR); EODC Gmbh (AUT)
+contact : cci_sm_contact@eodc.eu
+references : http://www.esa-soilmoisture-cci.org; Dorigo, W.A., Wagner, W., Albergel, C., Albrecht, F.,  Balsamo, G., Brocca, L., Chung, D., Ertl, M., Forkel, M., Gruber, A., Haas, E., Hamer, D. P. Hirschi, M., Ikonen, J., De Jeu, R. Kidd, R. Lahoz, W., Liu, Y.Y., Miralles, D., Lecomte, P. (2017) ESA CCI Soil Moisture for improved Earth system understanding: State-of-the art and future directions. In Remote Sensing of Environment, 2017, ISSN 0034-4257, https://doi.org/10.1016/j.rse.2017.07.001; Gruber, A., Scanlon, T., van der Schalie, R., Wagner, W., Dorigo, W. (2019) Evolution of the ESA CCI Soil Moisture Climate Data Records and their underlying merging methodology. Earth System Science Data 11, 717-739, https://doi.org/10.5194/essd-11-717-2019; Gruber, A., Dorigo, W. A., Crow, W., Wagner W. (2017). Triple Collocation-Based Merging of Satellite Soil Moisture Retrievals. IEEE Transactions on Geoscience and Remote Sensing. PP. 1-13. https://doi.org/10.1109/TGRS.2017.2734070
+standard_name_vocabulary : NetCDF Climate and Forecast (CF) Metadata Convention
+keywords : Soil Moisture/Water Content
+naming_authority : TU Wien
+keywords_vocabulary : NASA Global Change Master Directory (GCMD) Science Keywords
+creator_name : Department of Geodesy and Geoinformation, Technical University of Vienna
+creator_url : https://climers.geo.tuwien.ac.at
+creator_email : CCI_SM_CONTACT@EODC.EU
+project : Climate Change Initiative - European Space Agency
+license : Data use is free and open for all registered users.
+time_coverage_duration : P41Y
+time_coverage_resolution : P1D
+geospatial_lat_min : -90.0
+geospatial_lat_max : 90.0
+geospatial_lon_min : -180.0
+geospatial_lon_max : 180.0
+geospatial_vertical_min : 0.0
+geospatial_vertical_max : 0.0
+geospatial_lat_units : degrees_north
+geospatial_lon_units : degrees_east
+geospatial_lat_resolution : 0.25 degree
+geospatial_lon_resolution : 0.25 degree
+spatial_resolution: 25km
+
+[DNFLAG]
+long_name : Day / Night Flag
+flag_values : 0, 1, 2, 3
+flag_meanings : NaN, day, night, day_night_combination
+_FillValue : 0
+
+[FLAG]
+long_name : Flag
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000
+bit_meanings: no_data_inconsistency_detected, snow_coverage_or_temperature_below_zero, dense_vegetation, others_no_convergence_in_the_model_thus_no_valid_sm_estimates, soil_moisture_value_exceeds_physical_boundary, weight_of_measurement_below_threshold, all_datasets_deemed_unreliable, barren_ground_advisory_flag, not_used;
+_FillValue : 127
+
+[MODE]
+long_name : Satellite Mode
+flag_values : 0, 1, 2, 3
+flag_meanings : NaN, ascending, descending, ascending_descending_combination
+_FillValue : 0
+
+[SENSOR]
+long_name :  Sensor
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000, 0b1000000000, 0b10000000000, 0b100000000000, 0b1000000000000, 0b10000000000000, 0b100000000000000, 0b1000000000000000, 0b10000000000000000
+bit_meanings: NaN, SMMR, SSMI, TMI, AMSRE, WindSat, AMSR2, SMOS, AMIWS, ASCATA, ASCATB, SMAP, MODEL, GPM, FY3B, FY3D, ASCATC, FY3C
+_FillValue : 0
+
+[SM]
+long_name : Volumetric Soil Moisture
+units : m3 m-3
+valid_range : 0. 1.
+_FillValue : -9999.0
+
+[SM_UNCERTAINTY]
+long_name : Volumetric Soil Moisture Uncertainty
+description : Uncertainty is the estimated standard deviation of the observation
+units : m3 m-3
+valid_range : 0. 1.
+_FillValue : -9999.0
+
+[FREQBANDID] # attention this changed for earlier versions
+long_name : Frequency Band Identification
+bits: 0b0, 0b1, 0b10, 0b100, 0b1000, 0b10000, 0b100000, 0b1000000, 0b10000000, 0b100000000
+bit_meanings: NaN, L14, C53, C66, C68, C69, C73, X107, K194, MODEL
+_FillValue : 0.
+
+[T0]
+long_name : Observation Timestamp
+units : days since 1970-01-01 00:00:00 UTC
+_FillValue : -9999.0
+
+
+