MNT: space_packet_parser update

imap_processing/ccsds/ccsds_data.py

@@ -42,10 +42,7 @@ class CcsdsData:
     def __init__(self, packet_header: dict):
         attributes = [field.name for field in fields(self)]
 
-        for key, item in packet_header.items():
-            value = (
-                item.derived_value if item.derived_value is not None else item.raw_value
-            )
+        for key, value in packet_header.items():
             if key in attributes:
                 setattr(self, key, value)
             else:

imap_processing/cdf/config/imap_ultra_l1a_variable_attrs.yaml

@@ -7,39 +7,42 @@ ultra_metadata_attrs:
   FORMAT: I19
   LABLAXIS: Metadata
   SCALE_TYP: linear
-  UNITS: " "
+  UNITS: ” ”
   VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807
   VAR_TYPE: support_data
   DISPLAY_TYPE: time_series
-
 ultra_support_attrs:
   CATDESC: Metadata for Ultra data
   FIELDNAM: metadata
   FILLVAL: -9223370000000000000
   FORMAT: I19
-  LABLAXIS: "none"
+  LABLAXIS: “none”
   SCALE_TYP: linear
-  UNITS: " "
+  UNITS: ” ”
   VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807
   VAR_TYPE: support_data
   DISPLAY_TYPE: time_series
-
 string_base_attrs:
   CATDESC: string metadata
   FIELDNAM: string_metadata
-  FORMAT: A80
-  VAR_TYPE: metadata
+  FILLVAL: -9223370000000000000
+  FORMAT: I1
+  LABLAXIS: “none”
+  SCALE_TYP: linear
+  VALIDMIN: 0
+  VALIDMAX: 1
+  VAR_TYPE: support_data
   DISPLAY_TYPE: no_plot
   DEPEND_0: epoch
-
+  UNITS: ” ”
 packet_data_attrs:
   CATDESC: packet data
   FIELDNAM: packet_data
   FILLVAL: -9223370000000000000
   FORMAT: I19
-  LABLAXIS: "none"
+  LABLAXIS: “none”
   SCALE_TYP: linear
   VALIDMIN: -9223372036854775808
   VALIDMAX: 9223372036854775807

imap_processing/codice/codice_l1a.py

@@ -365,7 +365,7 @@ def set_data_product_config(
         self.packet_dataset = packet
 
         # Set various configurations of the data product
-        self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid)  # type: ignore
+        self.config: dict[str, Any] = constants.DATA_PRODUCT_CONFIGURATIONS.get(apid)
 
         # Gather and set the CDF attributes
         self.cdf_attrs = ImapCdfAttributes()
@@ -541,6 +541,8 @@ def process_codice_l1a(file_path: Path, data_version: str) -> xr.Dataset:
         elif apid in constants.APIDS_FOR_SCIENCE_PROCESSING:
             # Extract the data
             science_values = packet_dataset.data.data[0]
+            # TODO: improve this as needed
+            science_values = "".join(f"{byte:08b}" for byte in science_values)
 
             # Get the four "main" parameters for processing
             table_id, plan_id, plan_step, view_id = get_params(packet_dataset)

imap_processing/codice/utils.py

@@ -65,7 +65,7 @@ def add_metadata_to_array(packet: space_packet_parser, metadata_arrays: dict) ->
 
     Parameters
     ----------
-    packet : space_packet_parser.parser.Packet
+    packet : space_packet_parser.packets.CCSDSPacket
         CODICE data packet.
     metadata_arrays : dict
         Metadata arrays.
@@ -88,7 +88,7 @@ def add_metadata_to_array(packet: space_packet_parser, metadata_arrays: dict) ->
     for key, value in packet.header.items():
         metadata_arrays.setdefault(key, []).append(value.raw_value)
 
-    for key, value in packet.data.items():
+    for key, value in packet.user_data.items():
         if key not in ignore_list:
             metadata_arrays.setdefault(key, []).append(value.raw_value)
 

imap_processing/decom.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 from typing import Union
 
-from space_packet_parser import parser, xtcedef
+from space_packet_parser import definitions
 
 
 def decom_packets(
@@ -32,9 +32,8 @@ def decom_packets(
     list
         List of all the unpacked data.
     """
-    packet_definition = xtcedef.XtcePacketDefinition(xtce_packet_definition)
-    packet_parser = parser.PacketParser(packet_definition)
+    packet_definition = definitions.XtcePacketDefinition(xtce_packet_definition)
 
     with open(packet_file, "rb") as binary_data:
-        packet_generator = packet_parser.generator(binary_data)
+        packet_generator = packet_definition.packet_generator(binary_data)
         return list(packet_generator)

imap_processing/glows/l0/decom_glows.py

@@ -3,7 +3,7 @@
 from enum import Enum
 from pathlib import Path
 
-from space_packet_parser import parser, xtcedef
+from space_packet_parser import definitions
 
 from imap_processing import imap_module_directory
 from imap_processing.ccsds.ccsds_data import CcsdsData
@@ -49,39 +49,28 @@ def decom_packets(
         f"{imap_module_directory}/glows/packet_definitions/GLX_COMBINED.xml"
     )
 
-    packet_definition = xtcedef.XtcePacketDefinition(xtce_document)
-    glows_parser = parser.PacketParser(packet_definition)
+    packet_definition = definitions.XtcePacketDefinition(xtce_document)
 
     histdata = []
     dedata = []
 
     filename = packet_file_path.name
 
     with open(packet_file_path, "rb") as binary_data:
-        glows_packets = glows_parser.generator(binary_data)
+        glows_packets = packet_definition.packet_generator(binary_data)
 
         for packet in glows_packets:
-            apid = packet.header["PKT_APID"].derived_value
+            apid = packet.header["PKT_APID"]
             # Do something with the packet data
             if apid == GlowsParams.HIST_APID.value:
-                values = [
-                    item.derived_value
-                    if item.derived_value is not None
-                    else item.raw_value
-                    for item in packet.data.values()
-                ]
+                values = [item.raw_value for item in packet.user_data.values()]
                 hist_l0 = HistogramL0(
                     __version__, filename, CcsdsData(packet.header), *values
                 )
                 histdata.append(hist_l0)
 
             if apid == GlowsParams.DE_APID.value:
-                values = [
-                    item.derived_value
-                    if item.derived_value is not None
-                    else item.raw_value
-                    for item in packet.data.values()
-                ]
+                values = [item for item in packet.user_data.values()]
 
                 de_l0 = DirectEventL0(
                     __version__, filename, CcsdsData(packet.header), *values

imap_processing/glows/l0/glows_l0_data.py

@@ -114,18 +114,7 @@ class HistogramL0(GlowsL0):
     ELAVG: int
     ELVAR: int
     EVENTS: int
-    HISTOGRAM_DATA: bytearray
-
-    def __post_init__(self) -> None:
-        """Convert HISTOGRAM_DATA attribute from string to bytearray if needed."""
-        if isinstance(self.HISTOGRAM_DATA, str):
-            # Convert string output from space_packet_parser to bytearray
-            self.HISTOGRAM_DATA = bytearray(
-                int(self.HISTOGRAM_DATA, 2).to_bytes(
-                    len(self.HISTOGRAM_DATA) // 8, "big"
-                )
-            )
-
+    HISTOGRAM_DATA: bytes
 
 @dataclass
 class DirectEventL0(GlowsL0):

imap_processing/hi/l1a/histogram.py

@@ -60,9 +60,11 @@ def create_dataset(input_ds: xr.Dataset) -> xr.Dataset:
     #       It seems like we could try to reshape the arrays and do some numpy
     #       broadcasting rather than for-loops directly here
     for i_epoch, counters_binary_data in enumerate(input_ds["counters"].data):
+        # TODO: improve this as needed
+        binary_str_val = "".join(f"{byte:08b}" for byte in counters_binary_data)
         # unpack 24 arrays of 90 12-bit unsigned integers
         counter_ints = [
-            int(counters_binary_data[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
+            int(binary_str_val[i * 12 : (i + 1) * 12], 2) for i in range(90 * 24)
         ]
         # populate the dataset with the unpacked integers
         for i_counter, counter in enumerate(

imap_processing/hi/l1a/science_direct_event.py

@@ -328,8 +328,10 @@ def science_direct_event(packets_data: xr.Dataset) -> xr.Dataset:
     # end of the list. This way, I don't need to flatten
     # the list later.
     for i, data in enumerate(packets_data["de_tof"].data):
+        # TODO: improve this as needed
+        binary_str_val = "".join(f"{byte:08b}" for byte in data)
         # break binary stream data into unit of 48-bits
-        event_48bits_list = break_into_bits_size(data)
+        event_48bits_list = break_into_bits_size(binary_str_val)
         # parse 48-bits into meaningful data such as metaevent or direct event
         de_data_list.extend([parse_direct_event(event) for event in event_48bits_list])
         # add packet time to packet_met_time

imap_processing/hit/l0/data_classes/housekeeping.py

@@ -213,7 +213,7 @@ class Housekeeping(HITBase):
 
     def __init__(
         self,
-        packet: space_packet_parser.parser.Packet,
+        packet: space_packet_parser.packets.CCSDSPacket,
         software_version: str,
         packet_file_name: str,
     ):

imap_processing/hit/l0/decom_hit.py

@@ -357,7 +357,11 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset:
     # Convert sequence flags and counters to NumPy arrays for vectorized operations
     seq_flgs = sci_dataset.seq_flgs.values
     seq_ctrs = sci_dataset.src_seq_ctr.values
-    science_data = sci_dataset.science_data.values
+    # TODO: improve this as needed
+    binary_str_val = []
+    for data in sci_dataset.science_data.values:
+        binary_str_val.append("".join(f"{byte:08b}" for byte in data))
+    science_data = binary_str_val
     epoch_data = sci_dataset.epoch.values
 
     # Number of packets in the file

imap_processing/hit/l0/utils/hit_base.py

@@ -31,19 +31,19 @@ class HITBase:
     packet_file_name: str
     ccsds_header: CcsdsData
 
-    def parse_data(self, packet: space_packet_parser.parser.Packet) -> None:
+    def parse_data(self, packet: space_packet_parser.packets.CCSDSPacket) -> None:
         """
         Parse Lo L0 packet data.
 
         Parameters
         ----------
-        packet : space_packet_parser.parser.Packet
+        packet : space_packet_parser.packets.CCSDSPacket
             A single Lo L0 packet from space packet parser.
         """
         attributes = [field.name for field in fields(self)]
 
         # For each item in packet, assign it to the matching attribute in the class.
-        for key, item in packet.data.items():
+        for key, item in packet.user_data.items():
             value = (
                 item.derived_value if item.derived_value is not None else item.raw_value
             )