From 58864d193001b6c732418cbe3f3398427087dc51 Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 09:41:16 +0000 Subject: [PATCH 1/6] invert bit position in quality flags --- pygac/klm_reader.py | 52 ++++++++++++++++++++++----------------------- pygac/pod_reader.py | 44 +++++++++++++++++++------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/pygac/klm_reader.py b/pygac/klm_reader.py index 84abd6e9..9535e6d7 100644 --- a/pygac/klm_reader.py +++ b/pygac/klm_reader.py @@ -66,33 +66,33 @@ class KLM_QualityIndicator(IntFlag): Table 8.3.1.3.3.2-1. and Table 8.3.1.4.3.2-1. define bit: 21 as "flywheeling detected during this frame" """ - FATAL_FLAG = 2**0 # Data should not be used for product generation - TIME_ERROR = 2**1 # Time sequence error detected within this scan - DATA_GAP = 2**2 # Data gap precedes this scan - CALIBRATION = 2**3 # Insufficient data for calibration - NO_EARTH_LOCATION = 2**4 # Earth location data not available - CLOCK_UPDATE = 2**5 # First good time following a clock update (nominally 0) - INSTRUMENT_CHANGE = 2**6 # Instrument status changed with this scan - BIT_SYNC_STATUS = 2**7 # Sync lock dropped during this frame - SYNC_ERROR = 2**8 # Frame sync word error greater than zero - FRAME_SYNC_LOCK = 2**9 # Frame sync previously dropped lock - SYNC_INVALID = 2**10 # Frame sync word not valid - FLYWHEELING = 2**10 # Flywheeling detected during this frame - BIT_SLIPPAGE = 2**11 # Bit slippage detected during this frame - # Note: 2**12 to 2**22 are not defined for KLMs - TIP_PARITY = 2**23 # TIP parity error detected + FATAL_FLAG = 2**31 # Data should not be used for product generation + TIME_ERROR = 2**30 # Time sequence error detected within this scan + DATA_GAP = 2**29 # Data gap precedes this scan + CALIBRATION = 2**28 # Insufficient data for calibration + NO_EARTH_LOCATION = 2**27 # Earth location data not available + CLOCK_UPDATE = 2**26 # First good time following a clock update (nominally 0) + INSTRUMENT_CHANGE = 2**25 # Instrument status changed with this scan + BIT_SYNC_STATUS = 2**24 # Sync lock dropped during this frame + SYNC_ERROR = 2**23 # Frame sync word error greater than zero + FRAME_SYNC_LOCK = 2**22 # Frame sync previously dropped lock + SYNC_INVALID = 2**21 # Frame sync word not valid + FLYWHEELING = 2**21 # Flywheeling detected during this frame + BIT_SLIPPAGE = 2**20 # Bit slippage detected during this frame + # Note: Bit 19 - 9 are not defined for KLMs + TIP_PARITY = 2**8 # TIP parity error detected # Reflected Sunlight (RS) detected (solar blackbody contamination) - CH_3B_RS = 2**24 - CH_3_CONTAMINATION = 2**24 # POD compatible alias - CH_3B_RS_ANOMALY = 2**25 - CH_4_RS = 2**26 - CH_4_CONTAMINATION = 2**26 # POD compatible alias - CH_4_RS_ANOMALY = 2**27 - CH_5_RS = 2**28 - CH_5_CONTAMINATION = 2**28 # POD compatible alias - CH_5_RS_ANOMALY = 2**29 - DATA_JITTER = 2**30 # Resync occurred on this frame - PSEUDO_NOISE = 2**31 # Pseudo noise occurred on this frame + CH_3B_RS = 2**7 + CH_3_CONTAMINATION = 2**7 # POD compatible alias + CH_3B_RS_ANOMALY = 2**6 + CH_4_RS = 2**5 + CH_4_CONTAMINATION = 2**5 # POD compatible alias + CH_4_RS_ANOMALY = 2**4 + CH_5_RS = 2**3 + CH_5_CONTAMINATION = 2**3 # POD compatible alias + CH_5_RS_ANOMALY = 2**2 + DATA_JITTER = 2**1 # Resync occurred on this frame + PSEUDO_NOISE = 2**0 # Pseudo noise occurred on this frame # GAC header object diff --git a/pygac/pod_reader.py b/pygac/pod_reader.py index 43f6f19c..be7e6cf6 100644 --- a/pygac/pod_reader.py +++ b/pygac/pod_reader.py @@ -63,35 +63,35 @@ class POD_QualityIndicator(IntFlag): POD guide Table 3.1.2.1-2. Format of quality indicators. """ # POD guide Table 3.1.2.1-2. Format of quality indicators. - FATAL_FLAG = 2**0 # Data should not be used for product generation - TIME_ERROR = 2**1 # A time sequence error was detected while Processing + FATAL_FLAG = 2**31 # Data should not be used for product generation + TIME_ERROR = 2**30 # A time sequence error was detected while Processing # this frame - DATA_GAP = 2**2 # A gap precedes this frame - DATA_JITTER = 2**3 # Resync occurred on this frame - CALIBRATION = 2**4 # Insufficient data for calibration - NO_EARTH_LOCATION = 2**5 # Earth location data not available - ASCEND_DESCEND = 2**6 # AVHRR Earth location indication of Ascending (=0) + DATA_GAP = 2**29 # A gap precedes this frame + DATA_JITTER = 2**28 # Resync occurred on this frame + CALIBRATION = 2**27 # Insufficient data for calibration + NO_EARTH_LOCATION = 2**26 # Earth location data not available + ASCEND_DESCEND = 2**25 # AVHRR Earth location indication of Ascending (=0) # or descending (=1) data - PSEUDO_NOISE = 2**7 # Pseudo Noise (P/N) occurred (=1) on the frame, + PSEUDO_NOISE = 2**24 # Pseudo Noise (P/N) occurred (=1) on the frame, # data not used for calibration computations - BIT_SYNC_STATUS = 2**8 # Drop lock during frame - SYNC_ERROR = 2**9 # Frame Sync word error greater than zero - FRAME_SYNC_LOCK = 2**10 # Frame Sync previously dropped lock - FLYWHEELING = 2**11 # Flywheeling detected during this frame - BIT_SLIPPAGE = 2**12 # Bit slippage detected during this frame + BIT_SYNC_STATUS = 2**23 # Drop lock during frame + SYNC_ERROR = 2**22 # Frame Sync word error greater than zero + FRAME_SYNC_LOCK = 2**21 # Frame Sync previously dropped lock + FLYWHEELING = 2**20 # Flywheeling detected during this frame + BIT_SLIPPAGE = 2**19 # Bit slippage detected during this frame # Solar blackbody contamination indicator # 0 = no correction # 1 = solar contamination corrected - CH_3_CONTAMINATION = 2**13 # Channel 3 solar blackbody contamination - CH_4_CONTAMINATION = 2**14 # Channel 4 solar blackbody contamination - CH_5_CONTAMINATION = 2**15 # Channel 5 solar blackbody contamination + CH_3_CONTAMINATION = 2**18 # Channel 3 solar blackbody contamination + CH_4_CONTAMINATION = 2**17 # Channel 4 solar blackbody contamination + CH_5_CONTAMINATION = 2**16 # Channel 5 solar blackbody contamination # TIP Parity - TIP_PARITY_1 = 2**16 # In first minor frame - TIP_PARITY_2 = 2**17 # In second minor frame - TIP_PARITY_3 = 2**18 # In third minor frame - TIP_PARITY_4 = 2**19 # In fourth minor frame - TIP_PARITY_5 = 2**20 # In fifth minor frame - # Note: 2**21 to 2**23, and 2**30 to 2**31 are spare bits. 2**24 to 2**29 define + TIP_PARITY_1 = 2**15 # In first minor frame + TIP_PARITY_2 = 2**14 # In second minor frame + TIP_PARITY_3 = 2**13 # In third minor frame + TIP_PARITY_4 = 2**12 # In fourth minor frame + TIP_PARITY_5 = 2**11 # In fifth minor frame + # Note: Bit 10 to 8, and 1 to 0 are spare bits. 7 to 2 define # "SYNC ERRORS - Number of bit errors in frame sync" (6 bit integer?) From 0b9c26361cecd3ef3bba3c912e4950aa75eabef4 Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 10:11:20 +0000 Subject: [PATCH 2/6] clean code --- pygac/klm_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pygac/klm_reader.py b/pygac/klm_reader.py index 9535e6d7..c0663bc4 100644 --- a/pygac/klm_reader.py +++ b/pygac/klm_reader.py @@ -810,7 +810,6 @@ def _adjust_clock_drift(self): Clock drift correction is only applied to POD satellites. On the KLM series, the clock is updated daily. """ - pass def get_tsm_pixels(self, channels): """Determine pixels affected by the scan motor issue. From f18d97ff2e74b8315f695c577a7e3247e113f559 Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 13:07:31 +0000 Subject: [PATCH 3/6] makde corrupt mask test more realistic --- pygac/tests/test_pod.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/pygac/tests/test_pod.py b/pygac/tests/test_pod.py index f626272c..b41ce445 100644 --- a/pygac/tests/test_pod.py +++ b/pygac/tests/test_pod.py @@ -154,20 +154,40 @@ def test_quality_indicators(self): reader = self.reader QFlag = reader.QFlag quality_indicators = np.array([ - 0, # nothing flagged - -1, # everything flagged + 1, # 00...001 + QFlag.FATAL_FLAG, QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION, QFlag.TIME_ERROR | QFlag.DATA_GAP, - QFlag.FATAL_FLAG - ], dtype=np.uint32) + ], dtype='>u4') + # check if the bits look as expected + bits = np.unpackbits(quality_indicators.view(np.uint8)).reshape((-1, 32)) + # For a big endian integer, the number 1 fills only the last of the 32 bits + self.assertEqual(bits[0].sum(), 1) # only one bit is filled + self.assertEqual(bits[0][-1], 1) # the last bit is filled + # The fatal flag fills only the first bit + self.assertEqual(bits[1].sum(), 1) # only one bit is filled + self.assertEqual(bits[1][0], 1) # only one bit is filled + + # setup reader and test reader.scans = {self.reader._quality_indicators_key: quality_indicators} - # test mask, i.e. QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION - expected_mask = np.array([False, True, True, False, True], dtype=bool) + + # default mask is QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION + expected_mask = np.array([False, True, True, False], dtype=bool) numpy.testing.assert_array_equal(reader.mask, expected_mask) + # test individual flags - self.assertTrue(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).any()) - # count the occurence (everything flagged and last entrance => 2) - self.assertEqual(reader._get_corrupt_mask(flags=QFlag.FATAL_FLAG).sum(), 2) + expected_mask = np.array([False, False, False, True], dtype=bool) + numpy.testing.assert_array_equal( + reader._get_corrupt_mask(flags=QFlag.TIME_ERROR), + expected_mask + ) + # test combination of flags + expected_mask = np.array([False, False, True, True], dtype=bool) + flags = QFlag.DATA_GAP | QFlag.NO_EARTH_LOCATION + numpy.testing.assert_array_equal( + reader._get_corrupt_mask(flags=flags), + expected_mask + ) @mock.patch('pygac.pod_reader.get_lonlatalt') @mock.patch('pygac.pod_reader.compute_pixels') From 097632d49c7fe17b745284251e49045fc702d006 Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 13:08:50 +0000 Subject: [PATCH 4/6] removed white space --- pygac/tests/test_pod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygac/tests/test_pod.py b/pygac/tests/test_pod.py index b41ce445..bb5a6931 100644 --- a/pygac/tests/test_pod.py +++ b/pygac/tests/test_pod.py @@ -174,7 +174,7 @@ def test_quality_indicators(self): # default mask is QFlag.FATAL_FLAG | QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION expected_mask = np.array([False, True, True, False], dtype=bool) numpy.testing.assert_array_equal(reader.mask, expected_mask) - + # test individual flags expected_mask = np.array([False, False, False, True], dtype=bool) numpy.testing.assert_array_equal( From 904ba0939f9ca2882bdb0206f84c8effe86a96af Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 13:46:50 +0000 Subject: [PATCH 5/6] correct contamination flag for KLMs --- pygac/klm_reader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygac/klm_reader.py b/pygac/klm_reader.py index c0663bc4..b9272519 100644 --- a/pygac/klm_reader.py +++ b/pygac/klm_reader.py @@ -83,14 +83,14 @@ class KLM_QualityIndicator(IntFlag): TIP_PARITY = 2**8 # TIP parity error detected # Reflected Sunlight (RS) detected (solar blackbody contamination) CH_3B_RS = 2**7 - CH_3_CONTAMINATION = 2**7 # POD compatible alias CH_3B_RS_ANOMALY = 2**6 + CH_3_CONTAMINATION = CH_3B_RS | CH_3B_RS_ANOMALY # POD compatible alias CH_4_RS = 2**5 - CH_4_CONTAMINATION = 2**5 # POD compatible alias CH_4_RS_ANOMALY = 2**4 + CH_4_CONTAMINATION = CH_4_RS | CH_4_RS_ANOMALY # POD compatible alias CH_5_RS = 2**3 - CH_5_CONTAMINATION = 2**3 # POD compatible alias CH_5_RS_ANOMALY = 2**2 + CH_5_CONTAMINATION = CH_5_RS | CH_5_RS_ANOMALY # POD compatible alias DATA_JITTER = 2**1 # Resync occurred on this frame PSEUDO_NOISE = 2**0 # Pseudo noise occurred on this frame From cf4fea51ef5d7ac4d41b80b758355f54fb377643 Mon Sep 17 00:00:00 2001 From: horn Date: Wed, 2 Dec 2020 15:05:09 +0000 Subject: [PATCH 6/6] correct comment line --- pygac/tests/test_pod.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygac/tests/test_pod.py b/pygac/tests/test_pod.py index bb5a6931..cb374c0f 100644 --- a/pygac/tests/test_pod.py +++ b/pygac/tests/test_pod.py @@ -155,7 +155,7 @@ def test_quality_indicators(self): QFlag = reader.QFlag quality_indicators = np.array([ 1, # 00...001 - QFlag.FATAL_FLAG, + QFlag.FATAL_FLAG, # 100...00 QFlag.CALIBRATION | QFlag.NO_EARTH_LOCATION, QFlag.TIME_ERROR | QFlag.DATA_GAP, ], dtype='>u4') @@ -166,7 +166,7 @@ def test_quality_indicators(self): self.assertEqual(bits[0][-1], 1) # the last bit is filled # The fatal flag fills only the first bit self.assertEqual(bits[1].sum(), 1) # only one bit is filled - self.assertEqual(bits[1][0], 1) # only one bit is filled + self.assertEqual(bits[1][0], 1) # the first bit is filled # setup reader and test reader.scans = {self.reader._quality_indicators_key: quality_indicators}