Merge remote-tracking branch 'upstream/v0.16.x' into cirrus-16

* upstream/v0.16.x: Docstest 0v16 (SciTools#244) Remove eccodes bug workaround added in SciTools#208. (SciTools#224) Update requirements to pick up Iris 3. (SciTools#243) Add 'main' conda-forge channel, needed for docs builds. (SciTools#240) Fix RC date in release notes (about to cut). (SciTools#235) Version 0.16 release candidate (SciTools#232) Cosmetic change : rename the travis iris-test-version options (SciTools#234) Travis test with both Iris latest-release and latest-master. (SciTools#231)
jamesp · Feb 23, 2021 · d1b192e · d1b192e
2 parents 4fb5ccd + df97291
commit d1b192e
Show file tree

Hide file tree

Showing 12 changed files with 108 additions and 99 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -3,8 +3,10 @@ dist: xenial
 
 env:
   matrix:
-    - PYTHON_VERSION=36
-    - PYTHON_VERSION=37
+    - PYTHON_VERSION=36 IRIS_SOURCE=master
+    - PYTHON_VERSION=37 IRIS_SOURCE=master
+    - PYTHON_VERSION=36 IRIS_SOURCE=release
+    - PYTHON_VERSION=37 IRIS_SOURCE=release
 
 install:
   # Download iris-test-data
@@ -40,6 +42,22 @@ install:
     export ENV_NAME="iris-grib-dev";
     export ENV_FILE="${TRAVIS_BUILD_DIR}/requirements/ci/py${PYTHON_VERSION}.yml";
 
+  # Optionally download latest Iris master, from the github repo,
+  # and replace the Iris dependency with Iris' underlying dependencies.
+  - >
+    if [[ "${IRIS_SOURCE}" == "master" ]]; then
+      IRIS_REF="https://github.com/SciTools/iris/archive/master.zip";
+      export IRIS_LOCATION="${HOME}/iris";
+      mkdir ${IRIS_LOCATION};
+      wget -O ${IRIS_LOCATION}/iris.zip ${IRIS_REF};
+      unzip -q ${IRIS_LOCATION}/iris.zip -d ${IRIS_LOCATION};
+
+      IRIS_ENV_FILE="${IRIS_LOCATION}/iris-master/requirements/ci/py${PYTHON_VERSION}.yml";
+      sed -i "/- iris/d" ${ENV_FILE};
+      echo "# IRIS DEPENDENCIES." >> ${ENV_FILE};
+      sed -ne "/dependencies:/,$ p" ${IRIS_ENV_FILE} | sed "1d" >> ${ENV_FILE};
+    fi;
+
   # Create environment.
   - >
     conda env create --quiet --file=${ENV_FILE};
@@ -61,6 +79,13 @@ install:
   # --------------
   - echo "Configuring Iris";
 
+  # Optionally install latest Iris master, from downloaded source.
+  - >
+    if [[ "${IRIS_SOURCE}" == "master" ]]; then
+      cd ${IRIS_LOCATION}/iris-master;
+      python setup.py --quiet install;
+    fi;
+
   # Locate Iris installation.
   - export IRIS_DIR=$(python -c "import iris; import os.path; print(os.path.dirname(iris.__file__))")
 

diff --git a/docs/ref/release_notes.rst b/docs/ref/release_notes.rst
@@ -1,12 +1,12 @@
 Release Notes
 =============
 
-=======
+
 What's new in iris-grib v0.16
 -----------------------------
 
 :Release: 0.16.0
-:Date: ?? Sep 2020
+:Date: 27 Jan 2021
 
 Features
 ^^^^^^^^
@@ -64,6 +64,24 @@ Bugs Fixed
   only the Iris coordinate system has changed.
   `(PR#223) <https://github.com/SciTools/iris-grib/pull/223>`_
 
+* `@pp-mo <https://github.com/pp-mo>`_ fixed a problem where cubes were loading from GRIB 1 with a changed coordinate
+  system, since eccodes versions >= 1.19.  This resulted from a change to eccodes, which now returns a different
+  'shapeOfTheEarth' parameter : see `eccodes issue ECC-811 <https://jira.ecmwf.int/browse/ECC-811>`_ .  This resulted
+  in a coordinate system with a different earth radius.
+  For backwards compatibilty, the earth radius has now been fixed to the same value as previously.
+  However, pending further investigation, this value may be technically incorrect and we may
+  yet decide to change it in a future release.
+  `(PR#240) <https://github.com/SciTools/iris-grib/pull/240>`_
+
+
+Dependencies
+^^^^^^^^^^^^
+
+* now requires Iris version >= 3.0
+  Needed for the bugfix in
+  `PR#223 <https://github.com/SciTools/iris-grib/pull/223>`_ .
+
+
 
 What's new in iris-grib v0.15.1
 -------------------------------

diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py
@@ -33,7 +33,7 @@
 from .message import GribMessage
 
 
-__version__ = '0.15.1'
+__version__ = '0.16.0'
 
 __all__ = ['load_cubes', 'save_grib2', 'load_pairs_from_fields',
            'save_pairs_from_cube', 'save_messages']
@@ -342,56 +342,30 @@ def _compute_extra_keys(self):
         self.extra_keys['_forecastTimeUnit'] = self._timeunit_string()
 
         # shape of the earth
-
-        # pre-defined sphere
-        if self.shapeOfTheEarth == 0:
+        soe_code = self.shapeOfTheEarth
+        # As this class is now *only* for GRIB1, 'shapeOfTheEarth' is not a
+        # value read from the actual file :  It is really a GRIB2 param, and
+        # the value is merely what eccodes (gribapi) gives as the default.
+        # This was always = 6, until eccodes 0.19, when it changed to 0.
+        # See https://jira.ecmwf.int/browse/ECC-811
+        # The two represent different sized spherical earths.
+        if soe_code not in (6, 0):
+            raise ValueError('Unexpected shapeOfTheEarth value =', soe_code)
+
+        soe_code = 6
+        # *FOR NOW* maintain the old behaviour (radius=6371229) in all cases,
+        # for backwards compatibility.
+        # However, this does not match the 'radiusOfTheEarth' default from the
+        # gribapi so is probably incorrect (see above issue ECC-811).
+        # So we may change this in future.
+
+        if soe_code == 0:
+            # New supposedly-correct default value, matches 'radiusOfTheEarth'.
             geoid = coord_systems.GeogCS(semi_major_axis=6367470)
-
-        # custom sphere
-        elif self.shapeOfTheEarth == 1:
-            geoid = coord_systems.GeogCS(
-                self.scaledValueOfRadiusOfSphericalEarth *
-                10 ** -self.scaleFactorOfRadiusOfSphericalEarth)
-
-        # IAU65 oblate sphere
-        elif self.shapeOfTheEarth == 2:
-            geoid = coord_systems.GeogCS(6378160, inverse_flattening=297.0)
-
-        # custom oblate spheroid (km)
-        elif self.shapeOfTheEarth == 3:
-            geoid = coord_systems.GeogCS(
-                semi_major_axis=self.scaledValueOfEarthMajorAxis *
-                10 ** -self.scaleFactorOfEarthMajorAxis * 1000.,
-                semi_minor_axis=self.scaledValueOfEarthMinorAxis *
-                10 ** -self.scaleFactorOfEarthMinorAxis * 1000.)
-
-        # IAG-GRS80 oblate spheroid
-        elif self.shapeOfTheEarth == 4:
-            geoid = coord_systems.GeogCS(6378137, None, 298.257222101)
-
-        # WGS84
-        elif self.shapeOfTheEarth == 5:
-            geoid = \
-                coord_systems.GeogCS(6378137, inverse_flattening=298.257223563)
-
-        # pre-defined sphere
-        elif self.shapeOfTheEarth == 6:
+        elif soe_code == 6:
+            # Old value, does *not* match the 'radiusOfTheEarth' parameter.
             geoid = coord_systems.GeogCS(6371229)
 
-        # custom oblate spheroid (m)
-        elif self.shapeOfTheEarth == 7:
-            geoid = coord_systems.GeogCS(
-                semi_major_axis=self.scaledValueOfEarthMajorAxis *
-                10 ** -self.scaleFactorOfEarthMajorAxis,
-                semi_minor_axis=self.scaledValueOfEarthMinorAxis *
-                10 ** -self.scaleFactorOfEarthMinorAxis)
-
-        elif self.shapeOfTheEarth == 8:
-            raise ValueError("unhandled shape of earth : grib earth shape = 8")
-
-        else:
-            raise ValueError("undefined shape of earth")
-
         gridType = gribapi.grib_get_string(self.grib_message, "gridType")
 
         if gridType in ["regular_ll", "regular_gg", "reduced_ll",

diff --git a/iris_grib/_grib1_load_rules.py b/iris_grib/_grib1_load_rules.py
@@ -238,9 +238,9 @@ def add_bounded_time_coords(aux_coords_and_dims, grib):
     if \
             (grib.levelType == 'ml') and \
             (hasattr(grib, 'pv')):
-        aux_coords_and_dims.append((AuxCoord(grib.level, standard_name='model_level_number', attributes={'positive': 'up'}), None))
+        aux_coords_and_dims.append((AuxCoord(grib.level, standard_name='model_level_number', units=1, attributes={'positive': 'up'}), None))
         aux_coords_and_dims.append((DimCoord(grib.pv[grib.level], long_name='level_pressure', units='Pa'), None))
-        aux_coords_and_dims.append((AuxCoord(grib.pv[grib.numberOfCoordinatesValues//2 + grib.level], long_name='sigma'), None))
+        aux_coords_and_dims.append((AuxCoord(grib.pv[grib.numberOfCoordinatesValues//2 + grib.level], long_name='sigma', units=1), None))
         factories.append(Factory(HybridPressureFactory, [{'long_name': 'level_pressure'}, {'long_name': 'sigma'}, Reference('surface_pressure')]))
 
     if grib._originatingCentre != 'unknown':

diff --git a/iris_grib/_load_convert.py b/iris_grib/_load_convert.py
@@ -1545,7 +1545,7 @@ def hybrid_factories(section, metadata):
             # Create the model level number scalar coordinate.
             scaledValue = section['scaledValueOfFirstFixedSurface']
             coord = DimCoord(scaledValue, standard_name='model_level_number',
-                             attributes=dict(positive='up'))
+                             units=1, attributes=dict(positive='up'))
             metadata['aux_coords_and_dims'].append((coord, None))
 
             if typeOfFirstFixedSurface == 118:
@@ -1576,7 +1576,7 @@ def hybrid_factories(section, metadata):
             metadata['aux_coords_and_dims'].append((coord, None))
             # Create the sigma scalar coordinate.
             offset = NV // 2 + scaledValue
-            coord = AuxCoord(pv[offset], long_name='sigma')
+            coord = AuxCoord(pv[offset], long_name='sigma', units=1)
             metadata['aux_coords_and_dims'].append((coord, None))
             # Create the associated factory reference.
             factory = Factory(factory_class, factory_args)
@@ -2286,19 +2286,22 @@ def satellite_common(section, metadata):
     if NB > 0:
         # Create the satellite series coordinate.
         satelliteSeries = section['satelliteSeries']
-        coord = AuxCoord(satelliteSeries, long_name='satellite_series')
+        coord = AuxCoord(satelliteSeries, long_name='satellite_series',
+                         units=1)
         # Add the satellite series coordinate to the metadata aux coords.
         metadata['aux_coords_and_dims'].append((coord, None))
 
         # Create the satellite number coordinate.
         satelliteNumber = section['satelliteNumber']
-        coord = AuxCoord(satelliteNumber, long_name='satellite_number')
+        coord = AuxCoord(satelliteNumber, long_name='satellite_number',
+                         units=1)
         # Add the satellite number coordinate to the metadata aux coords.
         metadata['aux_coords_and_dims'].append((coord, None))
 
         # Create the satellite instrument type coordinate.
         instrumentType = section['instrumentType']
-        coord = AuxCoord(instrumentType, long_name='instrument_type')
+        coord = AuxCoord(instrumentType, long_name='instrument_type',
+                         units=1)
         # Add the instrument type coordinate to the metadata aux coords.
         metadata['aux_coords_and_dims'].append((coord, None))
 

diff --git a/iris_grib/_save_rules.py b/iris_grib/_save_rules.py
@@ -1490,12 +1490,6 @@ def data_section(cube, grib):
         gribapi.grib_set(grib, "bitmapPresent", 1)
         gribapi.grib_set_double(grib, "missingValue", fill_value)
 
-    # A segmentation fault is raised by `gribapi.grib_set_double_array` if it
-    # tries to cast large data to float64. As a temporary fix we cast the data
-    # upfront
-    # TODO: remove the `astype` command once eccodes (gribapi) has been fixed.
-    if data.dtype != np.float64:
-        data = data.astype(np.float64)
     gribapi.grib_set_double_array(grib, "values", data.flatten())
 
     # todo: check packing accuracy?

diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml
@@ -15,16 +15,16 @@
         <dimCoord id="ecc4ad5a" long_name="height" points="[2.0]" shape="(1,)" units="Unit('m')" value_type="float64"/>
       </coord>
       <coord datadims="[1]">
-        <dimCoord id="0f43c07d" points="[-2396487.0124, -2392487.0124, -2388487.0124,
+        <dimCoord id="f429d179" points="[-2396487.0124, -2392487.0124, -2388487.0124,
 		..., 2387512.9876, 2391512.9876, 2395512.9876]" shape="(1199,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord datadims="[0]">
-        <dimCoord id="1e5b5e41" points="[-3870311.24926, -3866311.24926, -3862311.24926,
+        <dimCoord id="51f2bb31" points="[-3870311.24926, -3866311.24926, -3862311.24926,
 		..., -686311.249256, -682311.249256,
 		-678311.249256]" shape="(799,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord>

diff --git a/iris_grib/tests/results/integration/load_convert/sample_file_loads/lambert_grib1.cml b/iris_grib/tests/results/integration/load_convert/sample_file_loads/lambert_grib1.cml
@@ -16,16 +16,16 @@
         <auxCoord id="61bde96d" long_name="originating_centre" points="[	US National Weather Service, National Centres for Environmental Prediction]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
       </coord>
       <coord datadims="[1]">
-        <dimCoord id="07a25078" points="[-2396487.0124, -2392487.0124, -2388487.0124,
+        <dimCoord id="62845c7c" points="[-2396487.0124, -2392487.0124, -2388487.0124,
 		..., 2387512.9876, 2391512.9876, 2395512.9876]" shape="(1199,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60" central_lon="-98.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="-98.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord datadims="[0]">
-        <dimCoord id="fef4faee" points="[-3870311.24926, -3866311.24926, -3862311.24926,
+        <dimCoord id="c75f3634" points="[-3870311.24926, -3866311.24926, -3862311.24926,
 		..., -686311.249256, -682311.249256,
 		-678311.249256]" shape="(799,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60" central_lon="-98.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="-98.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord>

diff --git a/iris_grib/tests/results/integration/load_convert/sample_file_loads/lambert_grib2.cml b/iris_grib/tests/results/integration/load_convert/sample_file_loads/lambert_grib2.cml
@@ -15,16 +15,16 @@
         <dimCoord id="ecc4ad5a" long_name="height" points="[2.0]" shape="(1,)" units="Unit('m')" value_type="float64"/>
       </coord>
       <coord datadims="[1]">
-        <dimCoord id="0f43c07d" points="[-2396487.0124, -2392487.0124, -2388487.0124,
+        <dimCoord id="f429d179" points="[-2396487.0124, -2392487.0124, -2388487.0124,
 		..., 2387512.9876, 2391512.9876, 2395512.9876]" shape="(1199,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord datadims="[0]">
-        <dimCoord id="1e5b5e41" points="[-3870311.24926, -3866311.24926, -3862311.24926,
+        <dimCoord id="51f2bb31" points="[-3870311.24926, -3866311.24926, -3862311.24926,
 		..., -686311.249256, -682311.249256,
 		-678311.249256]" shape="(799,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
-          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0.0" false_northing="0.0" secant_latitudes="(60.0, 30.0)"/>
         </dimCoord>
       </coord>
       <coord>

diff --git a/iris_grib/tests/unit/grib1_load_rules/test_grib1_convert.py b/iris_grib/tests/unit/grib1_load_rules/test_grib1_convert.py
@@ -9,11 +9,10 @@
 # importing anything else
 import iris_grib.tests as tests
 
-import cf_units
 import gribapi
 from unittest import mock
 
-import iris
+from iris.aux_factory import HybridPressureFactory
 from iris.exceptions import TranslationError
 from iris.fileformats.rules import Reference
 
@@ -109,27 +108,23 @@ def test_grib1_hybrid_height(self):
         results = grib1_convert(gw)
 
         factory, = results[0]
-        self.assertEqual(factory.factory_class,
-                         iris.aux_factory.HybridPressureFactory)
+        self.assertEqual(factory.factory_class, HybridPressureFactory)
         delta, sigma, ref = factory.args
         self.assertEqual(delta, {'long_name': 'level_pressure'})
         self.assertEqual(sigma, {'long_name': 'sigma'})
         self.assertEqual(ref, Reference(name='surface_pressure'))
 
-        ml_ref = iris.coords.CoordDefn('model_level_number', None, None,
-                                       cf_units.Unit('1'),
-                                       {'positive': 'up'}, None, False)
-        lp_ref = iris.coords.CoordDefn(None, 'level_pressure', None,
-                                       cf_units.Unit('Pa'),
-                                       {}, None, False)
-        s_ref = iris.coords.CoordDefn(None, 'sigma', None,
-                                      cf_units.Unit('1'),
-                                      {}, None, False)
-
-        aux_coord_defns = [coord._as_defn() for coord, dim in results[8]]
-        self.assertIn(ml_ref, aux_coord_defns)
-        self.assertIn(lp_ref, aux_coord_defns)
-        self.assertIn(s_ref, aux_coord_defns)
+        coords_and_dims = results[8]
+        coord, = [co for co, _ in coords_and_dims
+                  if co.name() == 'model_level_number']
+        self.assertEqual(coord.units, '1')
+        self.assertEqual(coord.attributes['positive'], 'up')
+        coord, = [co for co, _ in coords_and_dims
+                  if co.name() == 'level_pressure']
+        self.assertEqual(coord.units, 'Pa')
+        coord, = [co for co, _ in coords_and_dims
+                  if co.name() == 'sigma']
+        self.assertEqual(coord.units, '1')
 
 
 if __name__ == "__main__":

diff --git a/iris_grib/tests/unit/load_convert/test_satellite_common.py b/iris_grib/tests/unit/load_convert/test_satellite_common.py
@@ -43,11 +43,11 @@ def _check(self, factors=1, values=111):
 
         # Check the result.
         expected = empty_metadata()
-        coord = AuxCoord(series, long_name='satellite_series')
+        coord = AuxCoord(series, long_name='satellite_series', units=1)
         expected['aux_coords_and_dims'].append((coord, None))
-        coord = AuxCoord(number, long_name='satellite_number')
+        coord = AuxCoord(number, long_name='satellite_number', units=1)
         expected['aux_coords_and_dims'].append((coord, None))
-        coord = AuxCoord(instrument, long_name='instrument_type')
+        coord = AuxCoord(instrument, long_name='instrument_type', units=1)
         expected['aux_coords_and_dims'].append((coord, None))
         standard_name = 'sensor_band_central_radiation_wavenumber'
         coord = AuxCoord(values / (10.0 ** factors),

diff --git a/requirements/core.txt b/requirements/core.txt
@@ -1,4 +1,4 @@
 # Core dependencies.
 
-scitools-iris>=2.4
+scitools-iris>=3
 eccodes-python