PI:3358 (WIP) Ensure flags load/save without units (#3613)

* change dependencies : NOTE these changes need removing when remerging to master

lib/iris/fileformats/_pyke_rules/fc_rules_cf.krb

@@ -1689,6 +1689,9 @@ fc_extras
         if np.issubdtype(cf_var.dtype, np.str_):
             attr_units = cf_units._NO_UNIT_STRING
 
+        if any(hasattr(cf_var.cf_data, name) for name in ("flag_values", "flag_masks", "flag_meanings")):
+            attr_units = cf_units._NO_UNIT_STRING
+
         # Get any assoicated calendar for a time reference coordinate.
         if cf_units.as_unit(attr_units).is_time_reference():
             attr_calendar = getattr(cf_var, CF_ATTR_CALENDAR, None)

lib/iris/fileformats/netcdf.py

@@ -1776,7 +1776,7 @@ def _inner_create_cf_cellmeasure_or_ancil_variable(
         # Add the data to the CF-netCDF variable.
         cf_var[:] = data
 
-        if dimensional_metadata.units != "unknown":
+        if dimensional_metadata.units not in ("no_unit", "unknown"):
             _setncattr(cf_var, "units", str(dimensional_metadata.units))
 
         if dimensional_metadata.standard_name is not None:
@@ -1942,7 +1942,7 @@ def _create_cf_coord_variable(self, cube, dimension_names, coord):
         # Deal with CF-netCDF units and standard name.
         standard_name, long_name, units = self._cf_coord_identity(coord)
 
-        if units != "unknown":
+        if units not in ("no_unit", "unknown"):
             _setncattr(cf_var, "units", units)
 
         if standard_name is not None:
@@ -2387,7 +2387,7 @@ def store(data, cf_var, fill_value):
         if cube.long_name:
             _setncattr(cf_var, "long_name", cube.long_name)
 
-        if cube.units != "unknown":
+        if cube.units not in ("no_unit", "unknown"):
             _setncattr(cf_var, "units", str(cube.units))
 
         # Add the CF-netCDF calendar attribute.

lib/iris/tests/results/netcdf/netcdf_save_no_name.cdl

@@ -10,7 +10,6 @@ variables:
 	double dim1(dim1) ;
 		dim1:units = "m" ;
 	char unknown_scalar(string6) ;
-		unknown_scalar:units = "no_unit" ;
 
 // global attributes:
 		:Conventions = "CF-1.7" ;

lib/iris/tests/test_netcdf.py

@@ -296,6 +296,55 @@ def test_ancillary_variables(self):
         )
         self.assertEqual(avs[0], expected)
 
+    def test_status_flags(self):
+        # Note: using a CDL string as a test data reference, rather than a binary file.
+        ref_cdl = """
+            netcdf cm_attr {
+            dimensions:
+                axv = 3 ;
+            variables:
+                int64 qqv(axv) ;
+                    qqv:long_name = "qq" ;
+                    qqv:units = "1" ;
+                    qqv:ancillary_variables = "my_av" ;
+                int64 axv(axv) ;
+                    axv:units = "1" ;
+                    axv:long_name = "x" ;
+                byte my_av(axv) ;
+                    my_av:long_name = "qq status_flag" ;
+                    my_av:flag_values = 1b, 2b ;
+                    my_av:flag_meanings = "a b" ;
+            data:
+                axv = 11, 21, 31;
+                my_av = 1b, 1b, 2b;
+            }
+            """
+        self.tmpdir = tempfile.mkdtemp()
+        cdl_path = os.path.join(self.tmpdir, "tst.cdl")
+        nc_path = os.path.join(self.tmpdir, "tst.nc")
+        # Write CDL string into a temporary CDL file.
+        with open(cdl_path, "w") as f_out:
+            f_out.write(ref_cdl)
+        # Use ncgen to convert this into an actual (temporary) netCDF file.
+        command = "ncgen -o {} {}".format(nc_path, cdl_path)
+        check_call(command, shell=True)
+        # Load with iris.fileformats.netcdf.load_cubes, and check expected content.
+        cubes = list(nc_load_cubes(nc_path))
+        self.assertEqual(len(cubes), 1)
+        avs = cubes[0].ancillary_variables()
+        self.assertEqual(len(avs), 1)
+        expected = AncillaryVariable(
+            np.ma.array([1, 1, 2], dtype=np.int8),
+            long_name="qq status_flag",
+            var_name="my_av",
+            units="no_unit",
+            attributes={
+                "flag_values": np.array([1, 2], dtype=np.int8),
+                "flag_meanings": "a b",
+            },
+        )
+        self.assertEqual(avs[0], expected)
+
     def test_cell_measures(self):
         # Note: using a CDL string as a test data reference, rather than a binary file.
         ref_cdl = """

lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_auxiliary_coordinate.py

@@ -91,7 +91,7 @@ def _get_per_test_bounds_var(_coord_unused):
     def _make_array_and_cf_data(cls, dimension_names):
         shape = tuple(cls.dim_names_lens[name]
                       for name in dimension_names)
-        cf_data = mock.Mock(_FillValue=None)
+        cf_data = mock.MagicMock(_FillValue=None, spec=[])
         cf_data.chunking = mock.MagicMock(return_value=shape)
         return np.zeros(shape), cf_data
 
@@ -144,7 +144,7 @@ class TestDtype(tests.IrisTest):
     def setUp(self):
         # Create coordinate cf variables and pyke engine.
         points = np.arange(6).reshape(2, 3)
-        cf_data = mock.Mock(_FillValue=None)
+        cf_data = mock.MagicMock(_FillValue=None)
         cf_data.chunking = mock.MagicMock(return_value=points.shape)
 
         self.cf_coord_var = mock.Mock(
@@ -218,7 +218,7 @@ def setUp(self):
             scale_factor=1,
             add_offset=0,
             cf_name='wibble',
-            cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None)),
+            cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None), spec=[]),
             standard_name=None,
             long_name='wibble',
             units='days since 1970-01-01',

lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_cube_metadata.py

@@ -28,7 +28,7 @@ def _make_engine(global_attributes=None, standard_name=None, long_name=None):
 
     cf_group = mock.Mock(global_attributes=global_attributes)
 
-    cf_var = mock.Mock(
+    cf_var = mock.MagicMock(
         cf_name='wibble',
         standard_name=standard_name,
         long_name=long_name,

lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_build_dimension_coordinate.py

@@ -75,6 +75,7 @@ def _set_cf_coord_var(self, points):
         self.cf_coord_var = mock.Mock(
             dimensions=('foo',),
             cf_name='wibble',
+            cf_data=mock.Mock(spec=[]),
             standard_name=None,
             long_name='wibble',
             units='days since 1970-01-01',
@@ -226,6 +227,7 @@ def setUp(self):
             cf_name='wibble',
             standard_name=None,
             long_name='wibble',
+            cf_data=mock.Mock(spec=[]),
             units='m',
             shape=points.shape,
             dtype=points.dtype,
@@ -332,11 +334,12 @@ def setUp(self):
 
     def _make_vars(self, points, bounds=None, units='degrees'):
         points = np.array(points)
-        self.cf_coord_var = mock.Mock(
+        self.cf_coord_var = mock.MagicMock(
             dimensions=('foo',),
             cf_name='wibble',
             standard_name=None,
             long_name='wibble',
+            cf_data=mock.Mock(spec=[]),
             units=units,
             shape=points.shape,
             dtype=points.dtype,
@@ -435,6 +438,7 @@ def _make_vars(self, bounds):
             standard_name=None,
             long_name='wibble',
             units='degrees',
+            cf_data=mock.Mock(spec=[]),
             shape=(),
             dtype=points.dtype,
             __getitem__=lambda self, key: points[key])

lib/iris/tests/unit/fileformats/pyke_rules/compiled_krb/fc_rules_cf_fc/test_get_attr_units.py

@@ -29,8 +29,9 @@ def _make_cf_var(global_attributes=None):
 
         cf_group = mock.Mock(global_attributes=global_attributes)
 
-        cf_var = mock.Mock(
+        cf_var = mock.MagicMock(
             cf_name='sound_frequency',
+            cf_data=mock.Mock(spec=[]),
             standard_name=None,
             long_name=None,
             units=u'\u266b',

requirements/core.txt

@@ -6,9 +6,9 @@
 cartopy>=0.12
 #conda: proj4<6
 cf-units>=2
-cftime
+cftime<1.1
 dask[array]>=2  #conda: dask>=2
-matplotlib
+matplotlib<=3.1
 netcdf4
-numpy>=1.14
+numpy<=1.17
 scipy

requirements/test.txt

@@ -3,7 +3,8 @@
 
 black==19.10b0  #conda: black=19.10b0
 filelock
-imagehash>=4.0
+pillow<7
+imagehash
 nose
 pre-commit
 requests