From 2fa2287c23bbb6a8b369f789c873eaaf152f2bee Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 20 Oct 2022 12:37:58 +0200 Subject: [PATCH 1/5] Add support for frac data --- pygac/lac_reader.py | 5 +++-- pygac/tests/test_reader.py | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/pygac/lac_reader.py b/pygac/lac_reader.py index 8f3aadac..c9f0cabd 100644 --- a/pygac/lac_reader.py +++ b/pygac/lac_reader.py @@ -32,6 +32,7 @@ LOG = logging.getLogger(__name__) + class LACReader(Reader): """Reader for LAC data.""" @@ -46,7 +47,7 @@ def __init__(self, *args, **kwargs): @classmethod def _validate_header(cls, header): - """Check if the header belongs to this reader""" + """Check if the header belongs to this reader.""" # call super to enter the Method Resolution Order (MRO) super(LACReader, cls)._validate_header(header) LOG.debug("validate header") @@ -54,5 +55,5 @@ def _validate_header(cls, header): # split header into parts creation_site, transfer_mode, platform_id = ( data_set_name.split('.')[:3]) - if transfer_mode not in ['LHRR', 'HRPT']: + if transfer_mode not in ["LHRR", "HRPT", "FRAC"]: raise ReaderError('Improper transfer mode "%s"!' % transfer_mode) diff --git a/pygac/tests/test_reader.py b/pygac/tests/test_reader.py index 29388f5c..b511af7a 100644 --- a/pygac/tests/test_reader.py +++ b/pygac/tests/test_reader.py @@ -31,20 +31,27 @@ import numpy as np import numpy.testing from pygac.gac_reader import GACReader, ReaderError +from pygac.lac_reader import LACReader from pygac.pod_reader import POD_QualityIndicator from pygac.gac_pod import scanline from pygac.reader import NoTLEData class TestPath(os.PathLike): + """Fake path class.""" + def __init__(self, path): + """Initialize the path.""" self.path = str(path) def __fspath__(self): + """Return the path.""" return self.path class FakeGACReader(GACReader): + """Fake GAC reader class.""" + QFlag = POD_QualityIndicator _quality_indicators_key = "quality_indicators" tsm_affected_intervals = {None: []} @@ -52,6 +59,7 @@ class FakeGACReader(GACReader): across_track = 4 def __init__(self): + """Initialize the fake reader.""" super(FakeGACReader, self).__init__() self.scan_width = self.across_track scans = np.zeros(self.along_track, dtype=scanline) @@ -68,9 +76,11 @@ def _get_times(self): return year, jday, msec def get_header_timestamp(self): + """Get the header timestamp.""" return datetime.datetime(1970, 1, 1) def get_telemetry(self): + """Get the telemetry.""" prt = 51 * np.ones(self.along_track) # prt threshold is 50 ict = 101 * np.ones((self.along_track, 3)) # ict threshold is 100 space = 101 * np.ones((self.along_track, 3)) # space threshold is 100 @@ -83,16 +93,20 @@ def _get_lonlat(self): pass def postproc(self, channels): + """Postprocess the data.""" pass def read(self, filename, fileobj=None): + """Read the data.""" pass @classmethod def read_header(cls, filename, fileobj=None): + """Read the header.""" pass def get_tsm_pixels(self, channels): + """Get the tsm pixels.""" pass @@ -192,6 +206,7 @@ def test__get_calibrated_channels_uniform_shape(self, get_channels): self.reader._get_calibrated_channels_uniform_shape() def test_get_calibrated_channels(self): + """Test getting calibrated channels.""" reader = FakeGACReader() res = reader.get_calibrated_channels() expected = np.full( @@ -325,9 +340,9 @@ def test_midnight_scanline(self): utcs2 = np.array([1, 2, 3]).astype('datetime64[ms]') scanline2 = None - for utcs, scanline in zip((utcs1, utcs2), (scanline1, scanline2)): + for utcs, scan_line in zip((utcs1, utcs2), (scanline1, scanline2)): self.reader.utcs = utcs - self.assertEqual(self.reader.get_midnight_scanline(), scanline, + self.assertEqual(self.reader.get_midnight_scanline(), scan_line, msg='Incorrect midnight scanline') def test_miss_lines(self): @@ -588,6 +603,7 @@ def test_update_metadata(self, get_miss_lines, get_midnight_scanline, get_sun_earth_distance_correction): + """Test updating the metadata.""" get_miss_lines.return_value = 'miss_lines' get_midnight_scanline.return_value = 'midn_line' get_sun_earth_distance_correction.return_value = 'factor' @@ -602,3 +618,20 @@ def test_update_metadata(self, 'gac_header': {'foo': 'bar'}, 'calib_coeffs_version': 'version'} self.assertDictEqual(self.reader.meta_data, mda_exp) + + +class TestLacReader(unittest.TestCase): + """Test the common LAC Reader.""" + + longMessage = True + + @mock.patch.multiple('pygac.lac_reader.LACReader', + __abstractmethods__=set()) + def setUp(self, *mocks): + """Set up the tests.""" + self.reader = LACReader() + + def test_lac_reader_accepts_FRAC(self): + """Test the header validation.""" + head = {'data_set_name': b'NSS.FRAC.M1.D19115.S2352.E0050.B3425758.SV'} + self.reader._validate_header(head) From c49bad4cce9e536f6dc20d26bedc77e1e825b432 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 20 Oct 2022 14:50:06 +0200 Subject: [PATCH 2/5] Fix assumption that files do not contain more than 15000 scanlines --- pygac/reader.py | 22 +++++++++++----------- pygac/tests/test_reader.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pygac/reader.py b/pygac/reader.py index 5b812652..b290167c 100644 --- a/pygac/reader.py +++ b/pygac/reader.py @@ -76,7 +76,8 @@ class ReaderError(ValueError): - """Raised in Reader.read if the given file does not correspond to it""" + """Raised in Reader.read if the given file does not correspond to it.""" + pass @@ -134,7 +135,7 @@ def __init__(self, interpolate_coords=True, adjust_clock_drift=True, @property def times(self): - """The UTCs as datetime.datetime""" + """Get the UTCs as datetime.datetime.""" return self.to_datetime(self.utcs) @property @@ -196,7 +197,7 @@ def read_header(cls, filename, fileobj=None): # pragma: no cover @classmethod def _correct_data_set_name(cls, header, filename): - """Replace invalid data_set_name from header with filename + """Replace invalid data_set_name from header with filename. Args: header (struct): file header @@ -221,7 +222,7 @@ def _correct_data_set_name(cls, header, filename): @classmethod def _validate_header(cls, header): - """Check if the header belongs to this reader + """Check if the header belongs to this reader. Note: according to https://www1.ncdc.noaa.gov/pub/data/satellite/ @@ -258,7 +259,7 @@ def _validate_header(cls, header): % header['data_set_name']) def _read_scanlines(self, buffer, count): - """Read the scanlines from the given buffer + """Read the scanlines from the given buffer. Args: buffer (bytes, bytearray): buffer to read from @@ -304,7 +305,7 @@ def can_read(cls, filename, fileobj=None): @classmethod def fromfile(cls, filename, fileobj=None): - """Create Reader from file (alternative constructor) + """Create Reader from file, alternative constructor. Args: filename (str): Path to GAC/LAC file @@ -323,14 +324,14 @@ def fromfile(cls, filename, fileobj=None): return instance def _get_calibrated_channels_uniform_shape(self): - """Prepare the channels as input for gac_io.save_gac""" + """Prepare the channels as input for gac_io.save_gac.""" channels = self.get_calibrated_channels() assert channels.shape[-1] == 6 return channels def save(self, start_line, end_line, output_file_prefix="PyGAC", output_dir="./", avhrr_dir=None, qual_dir=None, sunsatangles_dir=None): - """Convert the Reader instance content into hdf5 files""" + """Convert the Reader instance content into hdf5 files.""" avhrr_dir = avhrr_dir or output_dir qual_dir = qual_dir or output_dir sunsatangles_dir = sunsatangles_dir or output_dir @@ -775,7 +776,6 @@ def get_angles(self): and different ranges. Returns: - sat_azi: satellite azimuth angle degree clockwise from north in range ]-180, 180] @@ -894,7 +894,7 @@ def correct_scan_line_numbers(self): 'n_orig': self.scans['scan_line_number'].copy()} # Remove scanlines whose scanline number is outside the valid range - within_range = np.logical_and(self.scans["scan_line_number"] < 15000, + within_range = np.logical_and(self.scans["scan_line_number"] < len(self.scans) * 2, self.scans["scan_line_number"] >= 0) self.scans = self.scans[within_range] @@ -1131,7 +1131,7 @@ def get_tsm_pixels(self, channels): # pragma: no cover raise NotImplementedError def get_attitude_coeffs(self): - """Return the roll, pitch, yaw values""" + """Return the roll, pitch, yaw values.""" if self._rpy is None: if "constant_yaw_attitude_error" in self.head.dtype.fields: rpy = np.deg2rad([self.head["constant_roll_attitude_error"] / 1e3, diff --git a/pygac/tests/test_reader.py b/pygac/tests/test_reader.py index b511af7a..bf655f0d 100644 --- a/pygac/tests/test_reader.py +++ b/pygac/tests/test_reader.py @@ -535,8 +535,8 @@ def _get_scanline_numbers(self): Corrupted and corrected scanline numbers. """ - along_track = 12000 - scans = np.zeros(12000, dtype=[("scan_line_number", ">u2")]) + along_track = 16000 + scans = np.zeros(16000, dtype=[("scan_line_number", ">u2")]) scans["scan_line_number"] = np.arange(1, along_track+1) # ... with 500 missing scanlines at scanline 8000 From 2c06d7b62174885b78b3e6b157d4ef9e1aebb946 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 20 Oct 2022 15:57:30 +0200 Subject: [PATCH 3/5] Remove unneeded spaces --- pygac/reader.py | 22 +++++++++++----------- pygac/tests/test_reader.py | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pygac/reader.py b/pygac/reader.py index b290167c..bc00118e 100644 --- a/pygac/reader.py +++ b/pygac/reader.py @@ -49,29 +49,29 @@ # rpy values from # here:http://yyy.rsmas.miami.edu/groups/rrsl/pathfinder/Processing/proc_app_a.html rpy_coeffs = { - 'noaa7': {'roll': 0.000, + 'noaa7': {'roll': 0.000, 'pitch': 0.000, - 'yaw': 0.000, + 'yaw': 0.000, }, - 'noaa9': {'roll': 0.000, + 'noaa9': {'roll': 0.000, 'pitch': 0.0025, - 'yaw': 0.000, + 'yaw': 0.000, }, - 'noaa10': {'roll': 0.000, + 'noaa10': {'roll': 0.000, 'pitch': 0.000, - 'yaw': 0.000, + 'yaw': 0.000, }, 'noaa11': {'roll': -0.0019, 'pitch': -0.0037, - 'yaw': 0.000, + 'yaw': 0.000, }, - 'noaa12': {'roll': 0.000, + 'noaa12': {'roll': 0.000, 'pitch': 0.000, - 'yaw': 0.000, + 'yaw': 0.000, }, - 'noaa14': {'roll': 0.000, + 'noaa14': {'roll': 0.000, 'pitch': 0.000, - 'yaw': 0.000, + 'yaw': 0.000, }} diff --git a/pygac/tests/test_reader.py b/pygac/tests/test_reader.py index bf655f0d..e8c1cc43 100644 --- a/pygac/tests/test_reader.py +++ b/pygac/tests/test_reader.py @@ -420,19 +420,19 @@ def test_get_sat_angles_without_tle(self, get_tle_lines): # Test data correspond to columns 0:2, 201:208 and 407:409. Extracted like this: # self.lons[0:5, [0, 1, 201, 202, 203, 204, 205, 206, 207, -2, -1]] self.reader.lons = np.array([[69.41555135, 68.76815744, 28.04133742, 27.94671757, 27.85220562, - 27.7578125, 27.66354783, 27.5694182, 27.47542957, 2.66416611, + 27.7578125, 27.66354783, 27.5694182, 27.47542957, 2.66416611, 2.39739436], [69.41409536, 68.76979616, 28.00228658, 27.9076628, 27.8131467, - 27.71875, 27.62448295, 27.53035209, 27.43636312, 2.61727408, + 27.71875, 27.62448295, 27.53035209, 27.43636312, 2.61727408, 2.35049275], [69.42987929, 68.78543423, 27.96407251, 27.86936406, 27.77457923, - 27.6796875, 27.58467527, 27.48959853, 27.39453053, 2.5704025, + 27.6796875, 27.58467527, 27.48959853, 27.39453053, 2.5704025, 2.30362323], [69.44430772, 68.80064104, 27.91910034, 27.82340242, 27.72794715, - 27.6328125, 27.53805662, 27.44366144, 27.34959008, 2.53088093, + 27.6328125, 27.53805662, 27.44366144, 27.34959008, 2.53088093, 2.26729486], [69.47408815, 68.8259859, 27.87666513, 27.78224611, 27.68795045, - 27.59375, 27.49962326, 27.40557682, 27.31162435, 2.48359319, + 27.59375, 27.49962326, 27.40557682, 27.31162435, 2.48359319, 2.21976689]]) self.reader.lats = np.array([[71.62830288, 71.67081539, 69.90976034, 69.89297223, 69.87616536, 69.859375, 69.84262997, 69.82593315, 69.80928089, 61.61334632, From d615d1f2cded82491f8b49c2291030e6b8820240 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 20 Oct 2022 15:59:55 +0200 Subject: [PATCH 4/5] Remove unneeded spaces --- pygac/reader.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pygac/reader.py b/pygac/reader.py index bc00118e..b4d1333b 100644 --- a/pygac/reader.py +++ b/pygac/reader.py @@ -49,14 +49,14 @@ # rpy values from # here:http://yyy.rsmas.miami.edu/groups/rrsl/pathfinder/Processing/proc_app_a.html rpy_coeffs = { - 'noaa7': {'roll': 0.000, - 'pitch': 0.000, - 'yaw': 0.000, - }, - 'noaa9': {'roll': 0.000, - 'pitch': 0.0025, - 'yaw': 0.000, - }, + 'noaa7': {'roll': 0.000, + 'pitch': 0.000, + 'yaw': 0.000, + }, + 'noaa9': {'roll': 0.000, + 'pitch': 0.0025, + 'yaw': 0.000, + }, 'noaa10': {'roll': 0.000, 'pitch': 0.000, 'yaw': 0.000, From 2ecad40f4ddd390cec803aa7938db0beb31e19cb Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 24 Oct 2022 17:33:39 +0200 Subject: [PATCH 5/5] Use a different number of max scanlines in gac and lac --- pygac/gac_reader.py | 5 +++- pygac/lac_reader.py | 2 ++ pygac/reader.py | 2 +- pygac/tests/test_reader.py | 56 ++++++++++++++++++++++---------------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/pygac/gac_reader.py b/pygac/gac_reader.py index c91cb647..98f87064 100644 --- a/pygac/gac_reader.py +++ b/pygac/gac_reader.py @@ -34,11 +34,14 @@ LOG = logging.getLogger(__name__) + class GACReader(Reader): """Reader for GAC data.""" # Scanning frequency (scanlines per millisecond) scan_freq = 2.0 / 1000.0 + # Max scanlines + max_scanlines = 15000 def __init__(self, *args, **kwargs): """Init the GAC reader.""" @@ -48,7 +51,7 @@ def __init__(self, *args, **kwargs): @classmethod def _validate_header(cls, header): - """Check if the header belongs to this reader""" + """Check if the header belongs to this reader.""" # call super to enter the Method Resolution Order (MRO) super(GACReader, cls)._validate_header(header) LOG.debug("validate header") diff --git a/pygac/lac_reader.py b/pygac/lac_reader.py index c9f0cabd..e3690f40 100644 --- a/pygac/lac_reader.py +++ b/pygac/lac_reader.py @@ -38,6 +38,8 @@ class LACReader(Reader): # Scanning frequency (scanlines per millisecond) scan_freq = 6.0 / 1000.0 + # Max scanlines + max_scanlines = 65535 def __init__(self, *args, **kwargs): """Init the LAC reader.""" diff --git a/pygac/reader.py b/pygac/reader.py index b4d1333b..40ac30e9 100644 --- a/pygac/reader.py +++ b/pygac/reader.py @@ -894,7 +894,7 @@ def correct_scan_line_numbers(self): 'n_orig': self.scans['scan_line_number'].copy()} # Remove scanlines whose scanline number is outside the valid range - within_range = np.logical_and(self.scans["scan_line_number"] < len(self.scans) * 2, + within_range = np.logical_and(self.scans["scan_line_number"] < self.max_scanlines, self.scans["scan_line_number"] >= 0) self.scans = self.scans[within_range] diff --git a/pygac/tests/test_reader.py b/pygac/tests/test_reader.py index e8c1cc43..035d959e 100644 --- a/pygac/tests/test_reader.py +++ b/pygac/tests/test_reader.py @@ -528,31 +528,9 @@ def test_mask_tsm_pixels(self, get_tsm_pixels): self.reader.mask_tsm_pixels(channels) # masks in-place numpy.testing.assert_array_equal(channels, masked_exp) - def _get_scanline_numbers(self): - """Create artificial scanline numbers with some corruptions. - - Returns: - Corrupted and corrected scanline numbers. - - """ - along_track = 16000 - scans = np.zeros(16000, dtype=[("scan_line_number", ">u2")]) - scans["scan_line_number"] = np.arange(1, along_track+1) - - # ... with 500 missing scanlines at scanline 8000 - scans["scan_line_number"][8000:] += 500 - corrected = scans["scan_line_number"].copy() - - # ... and some spikes here and there - scans["scan_line_number"][3000] += 1E4 - scans["scan_line_number"][9000] -= 1E4 - corrected = np.delete(corrected, [3000, 9000]) - - return scans, corrected - def test_correct_scan_line_numbers(self): """Test scanline number correction.""" - scans, expected = self._get_scanline_numbers() + scans, expected = _get_scanline_numbers(14000) self.reader.scans = scans self.reader.correct_scan_line_numbers() numpy.testing.assert_array_equal(self.reader.scans['scan_line_number'], @@ -564,7 +542,7 @@ def test_correct_times_thresh(self, get_header_timestamp): header_time = datetime.datetime(2016, 8, 16, 16, 7, 36) # Create artificial timestamps - _, scan_line_numbers = self._get_scanline_numbers() + _, scan_line_numbers = _get_scanline_numbers(14000) t0 = np.array([header_time], dtype="datetime64[ms]").astype("i8")[0] shift = 1000 msecs = t0 + shift + scan_line_numbers / GACReader.scan_freq @@ -620,6 +598,28 @@ def test_update_metadata(self, self.assertDictEqual(self.reader.meta_data, mda_exp) +def _get_scanline_numbers(scanlines_along_track): + """Create artificial scanline numbers with some corruptions. + + Returns: + Corrupted and corrected scanline numbers. + + """ + scans = np.zeros(scanlines_along_track, dtype=[("scan_line_number", ">u2")]) + scans["scan_line_number"] = np.arange(1, scanlines_along_track + 1) + + # ... with 500 missing scanlines at scanline 8000 + scans["scan_line_number"][8000:] += 500 + corrected = scans["scan_line_number"].copy() + + # ... and some spikes here and there + scans["scan_line_number"][3000] += 1E4 + scans["scan_line_number"][9000] -= 1E4 + corrected = np.delete(corrected, [3000, 9000]) + + return scans, corrected + + class TestLacReader(unittest.TestCase): """Test the common LAC Reader.""" @@ -635,3 +635,11 @@ def test_lac_reader_accepts_FRAC(self): """Test the header validation.""" head = {'data_set_name': b'NSS.FRAC.M1.D19115.S2352.E0050.B3425758.SV'} self.reader._validate_header(head) + + def test_correct_scan_line_numbers(self): + """Test scanline number correction.""" + scans, expected = _get_scanline_numbers(22000) + self.reader.scans = scans + self.reader.correct_scan_line_numbers() + numpy.testing.assert_array_equal(self.reader.scans['scan_line_number'], + expected)