Merge pull request #91 from carloshorn/correct_qflags

Invert bit position in quality flags
pytroll · Dec 3, 2020 · 2962a86 · 2962a86
2 parents 28a97a4 + cf4fea5
commit 2962a86
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 58 deletions.
diff --git a/pygac/klm_reader.py b/pygac/klm_reader.py
@@ -66,33 +66,33 @@ class KLM_QualityIndicator(IntFlag):
         Table 8.3.1.3.3.2-1. and Table 8.3.1.4.3.2-1. define bit: 21 as
         "flywheeling detected during this frame"
     """
-    FATAL_FLAG = 2**0  # Data should not be used for product generation
-    TIME_ERROR = 2**1  # Time sequence error detected within this scan
-    DATA_GAP = 2**2  # Data gap precedes this scan
-    CALIBRATION = 2**3  # Insufficient data for calibration
-    NO_EARTH_LOCATION = 2**4  # Earth location data not available
-    CLOCK_UPDATE = 2**5  # First good time following a clock update (nominally 0)
-    INSTRUMENT_CHANGE = 2**6  # Instrument status changed with this scan
-    BIT_SYNC_STATUS = 2**7  # Sync lock dropped during this frame
-    SYNC_ERROR = 2**8  # Frame sync word error greater than zero
-    FRAME_SYNC_LOCK = 2**9  # Frame sync previously dropped lock
-    SYNC_INVALID = 2**10  # Frame sync word not valid
-    FLYWHEELING = 2**10  # Flywheeling detected during this frame
-    BIT_SLIPPAGE = 2**11  # Bit slippage detected during this frame
-    # Note: 2**12 to 2**22 are not defined for KLMs
-    TIP_PARITY = 2**23  # TIP parity error detected
+    FATAL_FLAG = 2**31  # Data should not be used for product generation
+    TIME_ERROR = 2**30  # Time sequence error detected within this scan
+    DATA_GAP = 2**29  # Data gap precedes this scan
+    CALIBRATION = 2**28  # Insufficient data for calibration
+    NO_EARTH_LOCATION = 2**27  # Earth location data not available
+    CLOCK_UPDATE = 2**26  # First good time following a clock update (nominally 0)
+    INSTRUMENT_CHANGE = 2**25  # Instrument status changed with this scan
+    BIT_SYNC_STATUS = 2**24  # Sync lock dropped during this frame
+    SYNC_ERROR = 2**23  # Frame sync word error greater than zero
+    FRAME_SYNC_LOCK = 2**22  # Frame sync previously dropped lock
+    SYNC_INVALID = 2**21  # Frame sync word not valid
+    FLYWHEELING = 2**21  # Flywheeling detected during this frame
+    BIT_SLIPPAGE = 2**20  # Bit slippage detected during this frame
+    # Note: Bit 19 - 9 are not defined for KLMs
+    TIP_PARITY = 2**8  # TIP parity error detected
     # Reflected Sunlight (RS) detected (solar blackbody contamination)
-    CH_3B_RS = 2**24
-    CH_3_CONTAMINATION = 2**24  # POD compatible alias
-    CH_3B_RS_ANOMALY = 2**25
-    CH_4_RS = 2**26
-    CH_4_CONTAMINATION = 2**26  # POD compatible alias
-    CH_4_RS_ANOMALY = 2**27
-    CH_5_RS = 2**28
-    CH_5_CONTAMINATION = 2**28  # POD compatible alias
-    CH_5_RS_ANOMALY = 2**29
-    DATA_JITTER = 2**30  # Resync occurred on this frame
-    PSEUDO_NOISE = 2**31  # Pseudo noise occurred on this frame
+    CH_3B_RS = 2**7
+    CH_3B_RS_ANOMALY = 2**6
+    CH_3_CONTAMINATION = CH_3B_RS | CH_3B_RS_ANOMALY  # POD compatible alias
+    CH_4_RS = 2**5
+    CH_4_RS_ANOMALY = 2**4
+    CH_4_CONTAMINATION = CH_4_RS | CH_4_RS_ANOMALY  # POD compatible alias
+    CH_5_RS = 2**3
+    CH_5_RS_ANOMALY = 2**2
+    CH_5_CONTAMINATION = CH_5_RS | CH_5_RS_ANOMALY  # POD compatible alias
+    DATA_JITTER = 2**1  # Resync occurred on this frame
+    PSEUDO_NOISE = 2**0  # Pseudo noise occurred on this frame
 
 
 # GAC header object
@@ -810,7 +810,6 @@ def _adjust_clock_drift(self):
             Clock drift correction is only applied to POD satellites.
             On the KLM series, the clock is updated daily.
         """
-        pass
 
     def get_tsm_pixels(self, channels):
         """Determine pixels affected by the scan motor issue.

diff --git a/pygac/pod_reader.py b/pygac/pod_reader.py
@@ -63,35 +63,35 @@ class POD_QualityIndicator(IntFlag):
         POD guide Table 3.1.2.1-2. Format of quality indicators.
     """
     # POD guide Table 3.1.2.1-2. Format of quality indicators.
-    FATAL_FLAG = 2**0  # Data should not be used for product generation
-    TIME_ERROR = 2**1  # A time sequence error was detected while Processing
+    FATAL_FLAG = 2**31  # Data should not be used for product generation
+    TIME_ERROR = 2**30  # A time sequence error was detected while Processing
     # this frame
-    DATA_GAP = 2**2  # A gap precedes this frame
-    DATA_JITTER = 2**3  # Resync occurred on this frame
-    CALIBRATION = 2**4  # Insufficient data for calibration
-    NO_EARTH_LOCATION = 2**5  # Earth location data not available
-    ASCEND_DESCEND = 2**6  # AVHRR Earth location indication of Ascending (=0)
+    DATA_GAP = 2**29  # A gap precedes this frame
+    DATA_JITTER = 2**28  # Resync occurred on this frame
+    CALIBRATION = 2**27  # Insufficient data for calibration
+    NO_EARTH_LOCATION = 2**26  # Earth location data not available
+    ASCEND_DESCEND = 2**25  # AVHRR Earth location indication of Ascending (=0)
     # or descending (=1) data
-    PSEUDO_NOISE = 2**7  # Pseudo Noise (P/N) occurred (=1) on the frame,
+    PSEUDO_NOISE = 2**24  # Pseudo Noise (P/N) occurred (=1) on the frame,
     # data not used for calibration computations
-    BIT_SYNC_STATUS = 2**8  # Drop lock during frame
-    SYNC_ERROR = 2**9  # Frame Sync word error greater than zero
-    FRAME_SYNC_LOCK = 2**10  # Frame Sync previously dropped lock
-    FLYWHEELING = 2**11  # Flywheeling detected during this frame
-    BIT_SLIPPAGE = 2**12  # Bit slippage detected during this frame
+    BIT_SYNC_STATUS = 2**23  # Drop lock during frame
+    SYNC_ERROR = 2**22  # Frame Sync word error greater than zero
+    FRAME_SYNC_LOCK = 2**21  # Frame Sync previously dropped lock
+    FLYWHEELING = 2**20  # Flywheeling detected during this frame
+    BIT_SLIPPAGE = 2**19  # Bit slippage detected during this frame
     # Solar blackbody contamination indicator
     # 0 = no correction
     # 1 = solar contamination corrected
-    CH_3_CONTAMINATION = 2**13  # Channel 3 solar blackbody contamination
-    CH_4_CONTAMINATION = 2**14  # Channel 4 solar blackbody contamination
-    CH_5_CONTAMINATION = 2**15  # Channel 5 solar blackbody contamination
+    CH_3_CONTAMINATION = 2**18  # Channel 3 solar blackbody contamination
+    CH_4_CONTAMINATION = 2**17  # Channel 4 solar blackbody contamination
+    CH_5_CONTAMINATION = 2**16  # Channel 5 solar blackbody contamination
     # TIP Parity
-    TIP_PARITY_1 = 2**16  # In first minor frame
-    TIP_PARITY_2 = 2**17  # In second minor frame
-    TIP_PARITY_3 = 2**18  # In third minor frame
-    TIP_PARITY_4 = 2**19  # In fourth minor frame
-    TIP_PARITY_5 = 2**20  # In fifth minor frame
-    # Note: 2**21 to 2**23, and 2**30 to 2**31 are spare bits. 2**24 to 2**29 define
+    TIP_PARITY_1 = 2**15  # In first minor frame
+    TIP_PARITY_2 = 2**14  # In second minor frame
+    TIP_PARITY_3 = 2**13  # In third minor frame
+    TIP_PARITY_4 = 2**12  # In fourth minor frame
+    TIP_PARITY_5 = 2**11  # In fifth minor frame
+    # Note: Bit 10 to 8, and 1 to 0 are spare bits. 7 to 2 define
     #       "SYNC ERRORS - Number of bit errors in frame sync" (6 bit integer?)
 
 

diff --git a/pygac/tests/test_pod.py b/pygac/tests/test_pod.py
@@ -154,20 +154,40 @@ def test_quality_indicators(self):
         reader = self.reader
         QFlag = reader.QFlag
         quality_indicators = np.array([
-            0,  # nothing flagged
-            -1,  # everything flagged
+            1,  # 00...001
+            QFlag.FATAL_FLAG,  # 100...00
             QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION,
             QFlag.TIME_ERROR | QFlag.DATA_GAP,
-            QFlag.FATAL_FLAG
-        ], dtype=np.uint32)
+        ], dtype='>u4')
+        # check if the bits look as expected
+        bits = np.unpackbits(quality_indicators.view(np.uint8)).reshape((-1, 32))
+        # For a big endian integer, the number 1 fills only the last of the 32 bits
+        self.assertEqual(bits[0].sum(), 1)  # only one bit is filled
+        self.assertEqual(bits[0][-1], 1)  # the last bit is filled
+        # The fatal flag fills only the first bit
+        self.assertEqual(bits[1].sum(), 1)  # only one bit is filled
+        self.assertEqual(bits[1][0], 1)  # the first bit is filled
+
+        # setup reader and test
         reader.scans = {self.reader._quality_indicators_key: quality_indicators}
-        # test mask, i.e. QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION
-        expected_mask = np.array([False, True, True, False, True], dtype=bool)
+
+        # default mask is QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION
+        expected_mask = np.array([False, True, True, False], dtype=bool)
         numpy.testing.assert_array_equal(reader.mask, expected_mask)
+
         # test individual flags
-        self.assertTrue(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).any())
-        # count the occurence (everything flagged and last entrance => 2)
-        self.assertEqual(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).sum(), 2)
+        expected_mask = np.array([False, False, False, True], dtype=bool)
+        numpy.testing.assert_array_equal(
+            reader._get_corrupt_mask(flags=QFlag.TIME_ERROR),
+            expected_mask
+        )
+        # test combination of flags
+        expected_mask = np.array([False, False, True, True], dtype=bool)
+        flags = QFlag.DATA_GAP | QFlag.NO_EARTH_LOCATION
+        numpy.testing.assert_array_equal(
+            reader._get_corrupt_mask(flags=flags),
+            expected_mask
+        )
 
     @mock.patch('pygac.pod_reader.get_lonlatalt')
     @mock.patch('pygac.pod_reader.compute_pixels')