Merge pull request #125 from mraspaud/feature-eo-sip-support

Support reading EO-SIP LAC data

.github/workflows/ci.yaml

@@ -10,7 +10,7 @@ jobs:
  fail-fast: true
  matrix:
  os: ["ubuntu-latest"]
- python-version: ["3.8", "3.9", "3.10"]
+ python-version: ["3.10", "3.11", "3.12"]
  experimental: [false]
 
  env:

pygac/pod_reader.py

@@ -279,7 +279,7 @@ def read(self, filename, fileobj=None):
  # choose the right header depending on the date
  with file_opener(fileobj or filename) as fd_:
  self.tbm_head, self.head = self.read_header(
- filename, fileobj=fd_)
+ filename, fileobj=fd_, header_date=self.header_date)
  if self.tbm_head:
  tbm_offset = tbm_header.itemsize
  else:
@@ -302,12 +302,14 @@ def read(self, filename, fileobj=None):
  return self.head, self.scans
 
  @classmethod
- def read_header(cls, filename, fileobj=None):
+ def read_header(cls, filename, fileobj=None, header_date="auto"):
  """Read the file header.
 
  Args:
  filename (str): Path to GAC/LAC file
  fileobj: An open file object to read from. (optional)
+ header_date: date to use to choose the header.
+ Defaults to "auto" to use the data to pick the header corresponding to the date of the file.
 
  Returns:
  archive_header (struct): archive header
@@ -332,19 +334,7 @@ def read_header(cls, filename, fileobj=None):
  fd_.seek(0)
  tbm_head = None
  tbm_offset = 0
- # read header
- head0, = np.frombuffer(
- fd_.read(header0.itemsize),
- dtype=header0, count=1)
- year, jday, _ = cls.decode_timestamps(head0["start_time"])
- start_date = (datetime.date(year, 1, 1) +
- datetime.timedelta(days=int(jday) - 1))
- if start_date < datetime.date(1992, 9, 8):
- header = header1
- elif start_date <= datetime.date(1994, 11, 15):
- header = header2
- else:
- header = header3
+ header = cls.choose_header_based_on_timestamp(header_date, fd_)
  fd_.seek(tbm_offset, 0)
  # need to copy frombuffer to have write access on head
  head, = np.frombuffer(
@@ -354,6 +344,31 @@ def read_header(cls, filename, fileobj=None):
  cls._validate_header(head)
  return tbm_head, head
 
+ @classmethod
+ def choose_header_based_on_timestamp(cls, header_date, fd_):
+ """Choose the header dtype based on the timestamp."""
+ if header_date == "auto":
+ header_date = cls.get_start_date(fd_)
+ if header_date < datetime.date(1992, 9, 8):
+ header = header1
+ elif header_date <= datetime.date(1994, 11, 15):
+ header = header2
+ else:
+ header = header3
+ return header
+
+ @classmethod
+ def get_start_date(cls, fd_):
+ """Get the start time from the filestream."""
+ head0, = np.frombuffer(
+ fd_.read(header0.itemsize),
+ dtype=header0, count=1)
+ year, jday, _ = cls.decode_timestamps(head0["start_time"])
+ header_date = (datetime.date(year, 1, 1) +
+ datetime.timedelta(days=int(jday) - 1))
+
+ return header_date
+
  @classmethod
  def _validate_header(cls, header):
  """Check if the header belongs to this reader."""

pygac/reader.py

@@ -86,6 +86,10 @@ class NoTLEData(IndexError):
  """Raised if no TLE data available within time range."""
 
 
+class DecodingError(ValueError):
+ """Raised when decoding of some value fails."""
+
+
 class Reader(six.with_metaclass(ABCMeta)):
  """Reader for GAC and LAC format, POD and KLM platforms."""
 
@@ -95,7 +99,7 @@ class Reader(six.with_metaclass(ABCMeta)):
 
  def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
  tle_dir=None, tle_name=None, tle_thresh=7, creation_site=None,
- custom_calibration=None, calibration_file=None):
+ custom_calibration=None, calibration_file=None, header_date="auto"):
  """Init the reader.
 
  Args:
@@ -111,6 +115,7 @@ def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
  custom_calibration: dictionary with a subset of user defined satellite specific
  calibration coefficients
  calibration_file: path to json file containing default calibrations
+ header_date: the date to use for pod header choice. Defaults to "auto".
 
  """
  self.meta_data = {}
@@ -122,6 +127,7 @@ def __init__(self, interpolate_coords=True, adjust_clock_drift=True,
  self.creation_site = (creation_site or 'NSS').encode('utf-8')
  self.custom_calibration = custom_calibration
  self.calibration_file = calibration_file
+ self.header_date = header_date
  self.head = None
  self.scans = None
  self.spacecraft_name = None
@@ -205,22 +211,38 @@ def _correct_data_set_name(cls, header, filename):
  filename (str): path to file
  """
  filename = str(filename)
- data_set_name = header['data_set_name'].decode(errors='ignore')
- if not cls.data_set_pattern.match(data_set_name):
- LOG.debug('The data_set_name in header %s does not match.'
- ' Use filename instead.' % header['data_set_name'])
+ for encoding in "utf-8", "cp500":
+ data_set_name = header['data_set_name']
+ try:
+ data_set_name = cls._decode_data_set_name(data_set_name, encoding)
+ except DecodingError as err:
+ LOG.debug(str(err))
+ else:
+ header["data_set_name"] = data_set_name
+ break
+ else:
+ LOG.debug(f'The data_set_name in header {header["data_set_name"]} does not match.'
+ ' Use filename instead.')
  match = cls.data_set_pattern.search(filename)
  if match:
  data_set_name = match.group()
- LOG.debug("Set data_set_name, to filename %s"
- % data_set_name)
+ LOG.debug(f"Set data_set_name, to filename {data_set_name}")
  header['data_set_name'] = data_set_name.encode()
  else:
- LOG.debug("header['data_set_name']=%s; filename='%s'"
- % (header['data_set_name'], filename))
+ LOG.debug(f"header['data_set_name']={header['data_set_name']}; filename='{filename}'")
  raise ReaderError('Cannot determine data_set_name!')
  return header
 
+ @classmethod
+ def _decode_data_set_name(cls, data_set_name, encoding):
+ data_set_name = data_set_name.decode(encoding, errors='ignore')
+ if not cls.data_set_pattern.match(data_set_name):
+ raise DecodingError(f'The data_set_name in header {data_set_name} '
+ f'does not seem correct using encoding {encoding}.')
+ else:
+ data_set_name = data_set_name.encode()
+ return data_set_name
+
  @classmethod
  def _validate_header(cls, header):
  """Check if the header belongs to this reader.
@@ -274,7 +296,7 @@ def _read_scanlines(self, buffer, count):
  "Expected %d scan lines, but found %d!"
  % (count, line_count))
  warnings.warn("Unexpected number of scanlines!",
- category=RuntimeWarning)
+ category=RuntimeWarning, stacklevel=2)
  self.scans = np.frombuffer(
  buffer, dtype=self.scanline_type, count=line_count)
 

pygac/tests/test_klm.py

@@ -38,7 +38,7 @@
 class TestKLM:
  """Test the klm reader."""
 
- def setup(self):
+ def setup_method(self):
  """Set up the tests."""
  self.reader = GACKLMReader()
 
@@ -126,7 +126,7 @@ def test_quality_indicators(self):
 class TestGACKLM:
  """Tests for gac klm."""
 
- def setup(self):
+ def setup_method(self):
  """Set up the tests."""
  self.reader = GACKLMReader()
 
@@ -150,7 +150,7 @@ def test_get_tsm_pixels(self, get_tsm_idx):
 class TestLACKLM:
  """Tests for lac klm."""
 
- def setup(self):
+ def setup_method(self):
  """Set up the tests."""
  self.reader = LACKLMReader()
  self.reader.scans = np.ones(100, dtype=scanline)