From 25a99f2b0fc0ef1bdbcf114f7e3ea5b46a8162d2 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 25 Mar 2024 10:41:32 -0600 Subject: [PATCH 01/12] Updating mag CDF generation to match new filenames --- imap_processing/mag/l1a/mag_l1a.py | 7 +++++++ imap_processing/tests/mag/test_mag_decom.py | 21 ++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index fe8f4b915..7fcca430a 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -28,3 +28,10 @@ def mag_l1a(packet_filepath): if mag_burst is not None: file = write_cdf(mag_burst) logger.info(f"Created CDF file at {file}") + + +if __name__ == "__main__": + filepath_burst = Path("mag_IT_data/MAG_SCI_BURST.bin") + filepath_norm = Path("mag_IT_data/MAG_SCI_NORM.bin") + mag_l1a(filepath_norm) + mag_l1a(filepath_burst) diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index d085b7451..68195a20b 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -59,23 +59,30 @@ def test_mag_raw_xarray(): assert burst_data.sizes["epoch"] == expected_burst_len -def test_mag_raw_cdf_generation(): +def test_mag_raw_cdf_generation(tmp_path): current_directory = Path(__file__).parent test_file = current_directory / "mag_l0_test_data.pkts" l0 = decom_packets(str(test_file)) + output_path = tmp_path / "imap" / "mag" / "l1a" / "2023" / "10" + norm_data, burst_data = export_to_xarray(l0) - output = write_cdf(norm_data) - assert output.exists() - assert output.name == "imap_mag_l1a_norm-raw_20231025_v001.cdf" + test_data_path_norm = output_path / "imap_mag_l1a_norm-raw_20231025_v001.cdf" + + assert not test_data_path_norm.exists() + output = write_cdf(norm_data, test_data_path_norm) + assert test_data_path_norm.exists() input_xarray = cdf_to_xarray(output) assert input_xarray.attrs.keys() == norm_data.attrs.keys() - output = write_cdf(burst_data) - assert output.exists() - assert output.name == "imap_mag_l1a_burst-raw_20231025_v001.cdf" + test_data_path_burst = output_path / "imap_mag_l1a_burst-raw_20231025_v001.cdf" + + assert not test_data_path_burst.exists() + output = write_cdf(burst_data, test_data_path_burst) + print(output) + assert test_data_path_burst.exists() input_xarray = cdf_to_xarray(output) assert input_xarray.attrs.keys() == burst_data.attrs.keys() From b30de9f98086b6b081ddb25bd1c1e624ce507ee3 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 25 Mar 2024 13:54:56 -0600 Subject: [PATCH 02/12] PR updates --- imap_processing/mag/l1a/mag_l1a.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index 7fcca430a..fe8f4b915 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -28,10 +28,3 @@ def mag_l1a(packet_filepath): if mag_burst is not None: file = write_cdf(mag_burst) logger.info(f"Created CDF file at {file}") - - -if __name__ == "__main__": - filepath_burst = Path("mag_IT_data/MAG_SCI_BURST.bin") - filepath_norm = Path("mag_IT_data/MAG_SCI_NORM.bin") - mag_l1a(filepath_norm) - mag_l1a(filepath_burst) From fe8975df7508c285bff93f10c7ec4901f7aba59a Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Wed, 27 Mar 2024 09:57:24 -0600 Subject: [PATCH 03/12] Updating test to remove path --- imap_processing/tests/mag/test_mag_decom.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index 68195a20b..d085b7451 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -59,30 +59,23 @@ def test_mag_raw_xarray(): assert burst_data.sizes["epoch"] == expected_burst_len -def test_mag_raw_cdf_generation(tmp_path): +def test_mag_raw_cdf_generation(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l0_test_data.pkts" l0 = decom_packets(str(test_file)) - output_path = tmp_path / "imap" / "mag" / "l1a" / "2023" / "10" - norm_data, burst_data = export_to_xarray(l0) - test_data_path_norm = output_path / "imap_mag_l1a_norm-raw_20231025_v001.cdf" - - assert not test_data_path_norm.exists() - output = write_cdf(norm_data, test_data_path_norm) - assert test_data_path_norm.exists() + output = write_cdf(norm_data) + assert output.exists() + assert output.name == "imap_mag_l1a_norm-raw_20231025_v001.cdf" input_xarray = cdf_to_xarray(output) assert input_xarray.attrs.keys() == norm_data.attrs.keys() - test_data_path_burst = output_path / "imap_mag_l1a_burst-raw_20231025_v001.cdf" - - assert not test_data_path_burst.exists() - output = write_cdf(burst_data, test_data_path_burst) - print(output) - assert test_data_path_burst.exists() + output = write_cdf(burst_data) + assert output.exists() + assert output.name == "imap_mag_l1a_burst-raw_20231025_v001.cdf" input_xarray = cdf_to_xarray(output) assert input_xarray.attrs.keys() == burst_data.attrs.keys() From 44439bf44619da5df78e04babe0f2aa8d643b827 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 28 Mar 2024 12:33:49 -0600 Subject: [PATCH 04/12] removing bytearray from magl0 --- imap_processing/mag/l0/mag_l0_data.py | 16 +++++++++++++--- imap_processing/tests/mag/test_mag_decom.py | 7 +++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/imap_processing/mag/l0/mag_l0_data.py b/imap_processing/mag/l0/mag_l0_data.py index edab08c77..63f7c6db3 100644 --- a/imap_processing/mag/l0/mag_l0_data.py +++ b/imap_processing/mag/l0/mag_l0_data.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from enum import IntEnum +import numpy as np + from imap_processing.ccsds.ccsds_data import CcsdsData @@ -90,7 +92,7 @@ class MagL0: PRI_FNTM: int SEC_COARSETM: int SEC_FNTM: int - VECTORS: bytearray + VECTORS: np.ndarray def __post_init__(self): """Convert Vectors attribute from string to bytearray if needed. @@ -100,9 +102,17 @@ def __post_init__(self): """ if isinstance(self.VECTORS, str): # Convert string output from space_packet_parser to bytearray - self.VECTORS = bytearray( - int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big") + self.VECTORS = np.frombuffer( + int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"), + dtype=np.dtype(">b"), ) + if isinstance(self.VECTORS, bytearray): + self.VECTORS = np.array(self.VECTORS, dtype=np.dtype(">b")) + + # Remove buffer from end of vectors + if len(self.VECTORS) % 2: + self.VECTORS = self.VECTORS[:-1] + self.PRI_VECSEC = 2**self.PRI_VECSEC self.SEC_VECSEC = 2**self.SEC_VECSEC diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index d085b7451..9840f0c22 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -33,6 +33,13 @@ def test_mag_decom(): assert test.SEC_COARSETM == expected_output["SEC_COARSETM"][index] assert test.SEC_FNTM == expected_output["SEC_FNTM"][index] + # Remove bytes for header and previous attributes from CCSDS_HEX, + # remaining bytes are vectors + assert ( + test.VECTORS.tobytes().hex() + == expected_output["CCSDS_HEX"][index][54:].lower() + ) + assert len(l0) == len(expected_output.index) From 079d8d08bcc2297b41bb01ae8e2b026a3ee743d0 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 28 Mar 2024 14:08:41 -0600 Subject: [PATCH 05/12] First pass at vector processing and tests --- imap_processing/mag/l1a/mag_l1a.py | 229 +++++++++++++++++- imap_processing/mag/l1a/mag_l1a_data.py | 105 ++++++++ .../tests/mag/mag_l1a_test_output.csv | 97 ++++++++ imap_processing/tests/mag/test_mag_l1a.py | 56 +++++ 4 files changed, 482 insertions(+), 5 deletions(-) create mode 100644 imap_processing/mag/l1a/mag_l1a_data.py create mode 100644 imap_processing/tests/mag/mag_l1a_test_output.csv create mode 100644 imap_processing/tests/mag/test_mag_l1a.py diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index fe8f4b915..cdec1321b 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -2,8 +2,12 @@ import logging +import numpy as np + from imap_processing.cdf.utils import write_cdf from imap_processing.mag.l0 import decom_mag +from imap_processing.mag.l0.mag_l0_data import MagL0 +from imap_processing.mag.l1a.mag_l1a_data import MagL1a, TimeTuple logger = logging.getLogger(__name__) @@ -19,12 +23,227 @@ def mag_l1a(packet_filepath): """ mag_l0 = decom_mag.decom_packets(packet_filepath) - mag_norm, mag_burst = decom_mag.export_to_xarray(mag_l0) + mag_norm_raw, mag_burst_raw = decom_mag.export_to_xarray(mag_l0) - if mag_norm is not None: - file = write_cdf(mag_norm) + if mag_norm_raw is not None: + file = write_cdf(mag_norm_raw) logger.info(f"Created CDF file at {file}") - if mag_burst is not None: - file = write_cdf(mag_burst) + if mag_burst_raw is not None: + file = write_cdf(mag_burst_raw) logger.info(f"Created CDF file at {file}") + + +def process_packets(mag_l0_list: list[MagL0]): + """ + Given a list of MagL0 packets, process them into MagO and MagI L1A data classes. + + Parameters + ---------- + mag_l0_list : list[MagL0] + List of Mag L0 packets to process + + Returns + ------- + (list[MagL1a], list[MagL1a]) + Tuple containing MagI and MagO L1A data classes + + """ + magi = [] + mago = [] + + for mag_l0 in mag_l0_list: + if mag_l0.COMPRESSION: + raise NotImplementedError("Unable to process compressed data") + + primary_start_time = TimeTuple(mag_l0.PRI_COARSETM, mag_l0.PRI_FNTM) + secondary_start_time = TimeTuple(mag_l0.SEC_COARSETM, mag_l0.SEC_FNTM) + + # seconds of data in this packet is the SUBTYPE plus 1 + seconds_per_packet = mag_l0.PUS_SSUBTYPE + 1 + + # now we know the number of secs of data in the packet, and the data rates of + # each sensor, we can calculate how much data is in this packet and where the + # byte boundaries are. + + # VECSEC is already decoded in mag_l0 + total_primary_vectors = seconds_per_packet * mag_l0.PRI_VECSEC + total_secondary_vectors = seconds_per_packet * mag_l0.SEC_VECSEC + + primary_vectors, secondary_vectors = process_vector_data( + mag_l0.VECTORS, total_primary_vectors, total_secondary_vectors + ) + + # Primary sensor is MAGO + if mag_l0.PRI_SENS == 0: + mago_l1a = MagL1a( + True, + bool(mag_l0.MAGO_ACT), + primary_start_time, + mag_l0.PRI_VECSEC, + total_primary_vectors, + seconds_per_packet, + mag_l0.SHCOARSE, + primary_vectors, + ) + + magi_l1a = MagL1a( + False, + bool(mag_l0.MAGI_ACT), + secondary_start_time, + mag_l0.SEC_VECSEC, + total_secondary_vectors, + seconds_per_packet, + mag_l0.SHCOARSE, + secondary_vectors, + ) + # Primary sensor is MAGI + if mag_l0.PRI_SENS == 1: + magi_l1a = MagL1a( + False, + bool(mag_l0.MAGI_ACT), + primary_start_time, + mag_l0.PRI_VECSEC, + total_primary_vectors, + seconds_per_packet, + mag_l0.SHCOARSE, + primary_vectors, + ) + + mago_l1a = MagL1a( + True, + bool(mag_l0.MAGO_ACT), + secondary_start_time, + mag_l0.SEC_VECSEC, + total_secondary_vectors, + seconds_per_packet, + mag_l0.SHCOARSE, + secondary_vectors, + ) + + magi.append(magi_l1a) + mago.append(mago_l1a) + + return magi, mago + + +def process_vector_data( + vector_data: np.ndarray, primary_count: int, secondary_count: int +) -> (list[tuple], list[tuple]): + """ + Given raw packet data, process into Vectors. + + Vectors are grouped into primary sensor and secondary sensor, and returned as a + tuple (primary sensor vectors, secondary sensor vectors) + + Written by MAG instrument team + + Parameters + ---------- + vector_data : np.ndarray + Raw vector data, in bytes. Contains both primary and secondary vector data + (first primary, then secondary) + primary_count : int + Count of the number of primary vectors + secondary_count : int + Count of the number of secondary vectors + + Returns + ------- + (primary, secondary) + Two arrays, each containing tuples of (x, y, z, sample_range) for each vector + sample. + """ + + # TODO: error handling + def to_signed16(n): + n = n & 0xFFFF + return n | (-(n & 0x8000)) + + pos = 0 + primary_vectors = [] + secondary_vectors = [] + + for i in range(primary_count + secondary_count): # 0..63 say + x, y, z, rng = 0, 0, 0, 0 + if i % 4 == 0: # start at bit 0, take 8 bits + 8bits + # pos = 0, 25, 50... + x = ( + ((vector_data[pos + 0] & 0xFF) << 8) + | ((vector_data[pos + 1] & 0xFF) << 0) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0xFF) << 8) + | ((vector_data[pos + 3] & 0xFF) << 0) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0xFF) << 8) + | ((vector_data[pos + 5] & 0xFF) << 0) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 6) & 0x3 + pos += 6 + elif i % 4 == 1: # start at bit 2, take 6 bits, 8 bit, 2 bits per vector + # pos = 6, 31... + x = ( + ((vector_data[pos + 0] & 0x3F) << 10) + | ((vector_data[pos + 1] & 0xFF) << 2) + | ((vector_data[pos + 2] >> 6) & 0x03) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x3F) << 10) + | ((vector_data[pos + 3] & 0xFF) << 2) + | ((vector_data[pos + 4] >> 6) & 0x03) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x3F) << 10) + | ((vector_data[pos + 5] & 0xFF) << 2) + | ((vector_data[pos + 6] >> 6) & 0x03) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 4) & 0x3 + pos += 6 + elif i % 4 == 2: # start at bit 4, take 4 bits, 8 bits, 4 bits per vector + # pos = 12, 37... + x = ( + ((vector_data[pos + 0] & 0x0F) << 12) + | ((vector_data[pos + 1] & 0xFF) << 4) + | ((vector_data[pos + 2] >> 4) & 0x0F) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x0F) << 12) + | ((vector_data[pos + 3] & 0xFF) << 4) + | ((vector_data[pos + 4] >> 4) & 0x0F) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x0F) << 12) + | ((vector_data[pos + 5] & 0xFF) << 4) + | ((vector_data[pos + 6] >> 4) & 0x0F) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 2) & 0x3 + pos += 6 + elif i % 4 == 3: # start at bit 6, take 2 bits, 8 bits, 6 bits per vector + # pos = 18, 43... + x = ( + ((vector_data[pos + 0] & 0x03) << 14) + | ((vector_data[pos + 1] & 0xFF) << 6) + | ((vector_data[pos + 2] >> 2) & 0x3F) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x03) << 14) + | ((vector_data[pos + 3] & 0xFF) << 6) + | ((vector_data[pos + 4] >> 2) & 0x3F) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x03) << 14) + | ((vector_data[pos + 5] & 0xFF) << 6) + | ((vector_data[pos + 6] >> 2) & 0x3F) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 0) & 0x3 + pos += 7 + + vector = (to_signed16(x), to_signed16(y), to_signed16(z), rng) + if i < primary_count: + primary_vectors.append(vector) + else: + secondary_vectors.append(vector) + + return (primary_vectors, secondary_vectors) diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py new file mode 100644 index 000000000..b2cdeb8b1 --- /dev/null +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -0,0 +1,105 @@ +"""Data classes for storing and processing MAG Level 1A data.""" +from dataclasses import dataclass +from math import floor + +MAX_FINE_TIME = 65535 # maximum 16 bit unsigned int + + +@dataclass +class TimeTuple: + """ + Class for storing fine time/coarse time for MAG data. + + Course time is mission SCLK in seconds. Fine time is 16bit unsigned sub-second + counter. + """ + + coarse_time: int + fine_time: int + + def __add__(self, seconds: int): + """ + Add a number of seconds to the time tuple. + + Parameters + ---------- + seconds : int + Number of seconds to add + + Returns + ------- + time : TimeTuple + New time tuple with the current time tuple + seconds. + """ + # Add whole seconds to coarse time + coarse = self.coarse_time + floor(seconds) + # fine time is 1/65535th of a second + fine = self.fine_time + round((seconds % 1) * MAX_FINE_TIME) + + # If fine is larger than the max, move the excess into coarse time. + if fine > MAX_FINE_TIME: + coarse = coarse + floor(fine / MAX_FINE_TIME) + fine = fine % MAX_FINE_TIME + + return TimeTuple(coarse, fine) + + +@dataclass +class Vector: + """ + Data class for storing MAG vector data. + + Attributes + ---------- + timestamp : TimeTuple + Time of the vector sample + vectors : tuple[int, int, int, int] + Vector sample, containing x,y,z,range + """ + + # TODO: This timestamp should be in J2000/datetime64 + timestamp: TimeTuple + vectors: tuple[int, int, int, int] + + def __init__(self, vectors, previous_time, time_step): + self.vectors = vectors + self.timestamp = previous_time + time_step + + +@dataclass +class MagL1a: + """ + Data class for MAG Level 1A data. + + Attributes + ---------- + is_mago : bool + True if the data is from MagO, False if data is from MagI + active : bool + True if the sensor is active + start_time : TimeTuple + The coarse and fine time for the sensor + vectors_per_second : int + Number of vectors per second + expected_vector_count : int + Expected number of vectors (vectors_per_second * seconds_of_data) + seconds_of_data : int + Number of seconds of data + SHCOARSE : int + Mission elapsed time + vector_samples : list[Vector] + List of magnetic vector samples, starting at start_time + + """ + + is_mago: bool + active: bool + start_time: TimeTuple + vectors_per_second: int + expected_vector_count: int + seconds_of_data: int + SHCOARSE: int + vector_samples: list + + # def __post_init__(self): + # TODO: Assign timestamps to vectors, convert list to Vector type diff --git a/imap_processing/tests/mag/mag_l1a_test_output.csv b/imap_processing/tests/mag/mag_l1a_test_output.csv new file mode 100644 index 000000000..9cecd8273 --- /dev/null +++ b/imap_processing/tests/mag/mag_l1a_test_output.csv @@ -0,0 +1,97 @@ +sequence,x_pri,y_pri,z_pri,rng_pri,x_sec,y_sec,z_sec,rng_sec,pri_coarse,pri_fine,sec_coarse,sec_fine +5,13324,24467,20105,3,13324,24467,20104,3,439067318,64618,439067318,64612 +5,26623,-16643,-25350,3,26623,-16644,-25351,3,439067318,64618,439067318,64612 +5,26626,-16630,-25324,3,26626,-16631,-25325,3,439067318,64618,439067318,64612 +5,26629,-16617,-25298,3,26629,-16617,-25298,3,439067318,64618,439067318,64612 +5,26633,-16604,-25271,3,26633,-16604,-25272,3,439067318,64618,439067318,64612 +5,26636,-16591,-25245,3,26636,-16591,-25246,3,439067318,64618,439067318,64612 +5,26639,-16577,-25218,3,26639,-16578,-25219,3,439067318,64618,439067318,64612 +5,26643,-16564,-25192,3,26642,-16565,-25193,3,439067318,64618,439067318,64612 +5,26646,-16551,-25166,3,26646,-16552,-25167,3,439067318,64618,439067318,64612 +5,26649,-16538,-25139,3,26649,-16538,-25140,3,439067318,64618,439067318,64612 +5,26652,-16525,-25113,3,26652,-16525,-25114,3,439067318,64618,439067318,64612 +5,26656,-16512,-25087,3,26656,-16512,-25087,3,439067318,64618,439067318,64612 +5,26659,-16498,-25060,3,26659,-16499,-25061,3,439067318,64618,439067318,64612 +5,26662,-16485,-25034,3,26662,-16486,-25035,3,439067318,64618,439067318,64612 +5,26666,-16472,-25007,3,26666,-16472,-25008,3,439067318,64618,439067318,64612 +5,26669,-16459,-24981,3,26669,-16459,-24982,3,439067318,64618,439067318,64612 +6,26672,-16446,-24955,3,26672,-16446,-24956,3,439067307,64605,439067307,64600 +6,26676,-16432,-24928,3,26675,-16433,-24929,3,439067307,64605,439067307,64600 +6,26679,-16419,-24902,3,26679,-16420,-24903,3,439067307,64605,439067307,64600 +6,26682,-16406,-24876,3,26682,-16407,-24877,3,439067307,64605,439067307,64600 +6,26685,-16393,-24849,3,26685,-16393,-24850,3,439067307,64605,439067307,64600 +6,26689,-16380,-24823,3,26689,-16380,-24824,3,439067307,64605,439067307,64600 +6,26692,-16367,-24797,3,26692,-16367,-24797,3,439067307,64605,439067307,64600 +6,26695,-16353,-24770,3,26695,-16354,-24771,3,439067307,64605,439067307,64600 +6,26699,-16340,-24744,3,26698,-16341,-24745,3,439067307,64605,439067307,64600 +6,26702,-16327,-24717,3,26702,-16327,-24718,3,439067307,64605,439067307,64600 +6,26705,-16314,-24691,3,26705,-16314,-24692,3,439067307,64605,439067307,64600 +6,26708,-16301,-24665,3,26708,-16301,-24666,3,439067307,64605,439067307,64600 +6,26712,-16287,-24638,3,26712,-16288,-24639,3,439067307,64605,439067307,64600 +6,26715,-16274,-24612,3,26715,-16275,-24613,3,439067307,64605,439067307,64600 +6,26718,-16261,-24586,3,26718,-16261,-24586,3,439067307,64605,439067307,64600 +6,26722,-16248,-24559,3,26722,-16248,-24560,3,439067307,64605,439067307,64600 +7,26725,-16235,-24533,3,26725,-16235,-24534,3,439067315,64605,439067315,64600 +7,26728,-16222,-24507,3,26728,-16222,-24507,3,439067315,64605,439067315,64600 +7,26732,-16208,-24480,3,26731,-16209,-24481,3,439067315,64605,439067315,64600 +7,26735,-16195,-24454,3,26735,-16196,-24455,3,439067315,64605,439067315,64600 +7,26738,-16182,-24427,3,26738,-16182,-24428,3,439067315,64605,439067315,64600 +7,26741,-16169,-24401,3,26741,-16169,-24402,3,439067315,64605,439067315,64600 +7,26745,-16156,-24375,3,26745,-16156,-24376,3,439067315,64605,439067315,64600 +7,26748,-16142,-24348,3,26748,-16143,-24349,3,439067315,64605,439067315,64600 +7,26751,-16129,-24322,3,26751,-16130,-24323,3,439067315,64605,439067315,64600 +7,26755,-16116,-24296,3,26755,-16116,-24296,3,439067315,64605,439067315,64600 +7,26758,-16103,-24269,3,26758,-16103,-24270,3,439067315,64605,439067315,64600 +7,26761,-16090,-24243,3,26761,-16090,-24244,3,439067315,64605,439067315,64600 +7,26765,-16076,-24216,3,26764,-16077,-24217,3,439067315,64605,439067315,64600 +7,26768,-16063,-24190,3,26768,-16064,-24191,3,439067315,64605,439067315,64600 +7,26771,-16050,-24164,3,26771,-16051,-24165,3,439067315,64605,439067315,64600 +7,26774,-16037,-24137,3,26774,-16037,-24138,3,439067315,64605,439067315,64600 +8,26778,-16024,-24111,3,26778,-16024,-24112,3,439067323,64606,439067323,64601 +8,26781,-16011,-24085,3,26781,-16011,-24086,3,439067323,64606,439067323,64601 +8,26784,-15997,-24058,3,26784,-15998,-24059,3,439067323,64606,439067323,64601 +8,26788,-15984,-24032,3,26787,-15985,-24033,3,439067323,64606,439067323,64601 +8,26791,-15971,-24006,3,26791,-15971,-24006,3,439067323,64606,439067323,64601 +8,26794,-15958,-23979,3,26794,-15958,-23980,3,439067323,64606,439067323,64601 +8,26797,-15945,-23953,3,26797,-15945,-23954,3,439067323,64606,439067323,64601 +8,26801,-15931,-23926,3,26801,-15932,-23927,3,439067323,64606,439067323,64601 +8,26804,-15918,-23900,3,26804,-15919,-23901,3,439067323,64606,439067323,64601 +8,26807,-15905,-23874,3,26807,-15906,-23875,3,439067323,64606,439067323,64601 +8,26811,-15892,-23847,3,26811,-15892,-23848,3,439067323,64606,439067323,64601 +8,26814,-15879,-23821,3,26814,-15879,-23822,3,439067323,64606,439067323,64601 +8,26817,-15866,-23795,3,26817,-15866,-23795,3,439067323,64606,439067323,64601 +8,26821,-15852,-23768,3,26820,-15853,-23769,3,439067323,64606,439067323,64601 +8,26824,-15839,-23742,3,26824,-15840,-23743,3,439067323,64606,439067323,64601 +8,26827,-15826,-23715,3,26827,-15826,-23716,3,439067323,64606,439067323,64601 +9,26830,-15813,-23689,3,26830,-15813,-23690,3,439067331,64604,439067331,64601 +9,26834,-15800,-23663,3,26834,-15800,-23664,3,439067331,64604,439067331,64601 +9,26837,-15786,-23636,3,26837,-15787,-23637,3,439067331,64604,439067331,64601 +9,26840,-15773,-23610,3,26840,-15774,-23611,3,439067331,64604,439067331,64601 +9,26844,-15760,-23584,3,26843,-15761,-23585,3,439067331,64604,439067331,64601 +9,26847,-15747,-23557,3,26847,-15747,-23558,3,439067331,64604,439067331,64601 +9,26850,-15734,-23531,3,26850,-15734,-23532,3,439067331,64604,439067331,64601 +9,26853,-15721,-23505,3,26853,-15721,-23505,3,439067331,64604,439067331,64601 +9,26857,-15707,-23478,3,26857,-15708,-23479,3,439067331,64604,439067331,64601 +9,26860,-15694,-23452,3,26860,-15695,-23453,3,439067331,64604,439067331,64601 +9,26863,-15681,-23425,3,26863,-15681,-23426,3,439067331,64604,439067331,64601 +9,26867,-15668,-23399,3,26867,-15668,-23400,3,439067331,64604,439067331,64601 +9,26870,-15655,-23373,3,26870,-15655,-23374,3,439067331,64604,439067331,64601 +9,26873,-15641,-23346,3,26873,-15642,-23347,3,439067331,64604,439067331,64601 +9,26877,-15628,-23320,3,26876,-15629,-23321,3,439067331,64604,439067331,64601 +9,26880,-15615,-23294,3,26880,-15616,-23294,3,439067331,64604,439067331,64601 +10,26883,-15602,-23267,3,26883,-15602,-23268,3,439067339,64603,439067339,64601 +10,26886,-15589,-23241,3,26886,-15589,-23242,3,439067339,64603,439067339,64601 +10,26890,-15576,-23215,3,26890,-15576,-23215,3,439067339,64603,439067339,64601 +10,26893,-15562,-23188,3,26893,-15563,-23189,3,439067339,64603,439067339,64601 +10,26896,-15549,-23162,3,26896,-15550,-23163,3,439067339,64603,439067339,64601 +10,26900,-15536,-23135,3,26900,-15536,-23136,3,439067339,64603,439067339,64601 +10,26903,-15523,-23109,3,26903,-15523,-23110,3,439067339,64603,439067339,64601 +10,26906,-15510,-23083,3,26906,-15510,-23084,3,439067339,64603,439067339,64601 +10,26910,-15496,-23056,3,26909,-15497,-23057,3,439067339,64603,439067339,64601 +10,26913,-15483,-23030,3,26913,-15484,-23031,3,439067339,64603,439067339,64601 +10,26916,-15470,-23004,3,26916,-15470,-23004,3,439067339,64603,439067339,64601 +10,26919,-15457,-22977,3,26919,-15457,-22978,3,439067339,64603,439067339,64601 +10,26923,-15444,-22951,3,26923,-15444,-22952,3,439067339,64603,439067339,64601 +10,26926,-15430,-22924,3,26926,-15431,-22925,3,439067339,64603,439067339,64601 +10,26929,-15417,-22898,3,26929,-15418,-22899,3,439067339,64603,439067339,64601 +10,26933,-15404,-22872,3,26932,-15405,-22873,3,439067339,64603,439067339,64601 diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py new file mode 100644 index 000000000..965d26b88 --- /dev/null +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pandas as pd + +from imap_processing.mag.l0.decom_mag import decom_packets +from imap_processing.mag.l1a.mag_l1a import process_vector_data +from imap_processing.mag.l1a.mag_l1a_data import MAX_FINE_TIME, TimeTuple + + +def test_process_vector_data(): + current_directory = Path(__file__).parent + test_file = current_directory / "mag_l1_test_data.pkts" + l0 = decom_packets(str(test_file)) + mag_l0 = l0[0] + + total_primary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.PRI_VECSEC + total_secondary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.SEC_VECSEC + + test_vectors = l0[0].VECTORS + + # 36 bytes + (primary_vectors, secondary_vectors) = process_vector_data( + test_vectors, total_primary_vectors, total_secondary_vectors + ) + + validation_data = pd.read_csv(current_directory / "mag_l1a_test_output.csv") + + print(validation_data.index) + for index in range(total_primary_vectors): + # print(validation_data.iloc[index]) + assert primary_vectors[index][0] == validation_data.iloc[index]["x_pri"] + assert primary_vectors[index][1] == validation_data.iloc[index]["y_pri"] + assert primary_vectors[index][2] == validation_data.iloc[index]["z_pri"] + assert primary_vectors[index][3] == validation_data.iloc[index]["rng_pri"] + + assert secondary_vectors[index][0] == validation_data.iloc[index]["x_sec"] + assert secondary_vectors[index][1] == validation_data.iloc[index]["y_sec"] + assert secondary_vectors[index][2] == validation_data.iloc[index]["z_sec"] + assert secondary_vectors[index][3] == validation_data.iloc[index]["rng_sec"] + + +def test_time_tuple(): + test_time_tuple = TimeTuple(439067318, 64618) + + test_add = test_time_tuple + 2 + + assert test_add == TimeTuple(439067320, 64618) + + # 1 / MAX_FINE_TIME + test_add = test_time_tuple + 1 / MAX_FINE_TIME + + assert test_add == TimeTuple(439067318, 64619) + + test_add = test_time_tuple + (1000 / MAX_FINE_TIME) + + assert test_add == TimeTuple(439067319, 83) From b26a4ddf460a175593297a75544df36a47f776fb Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Thu, 28 Mar 2024 14:34:07 -0600 Subject: [PATCH 06/12] Updating vectors to use proper timestamps --- imap_processing/mag/l1a/mag_l1a_data.py | 32 ++++++++++++++++++----- imap_processing/tests/mag/test_mag_l1a.py | 26 +++++++++++++++++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index b2cdeb8b1..b265d9317 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -17,7 +17,7 @@ class TimeTuple: coarse_time: int fine_time: int - def __add__(self, seconds: int): + def __add__(self, seconds: float): """ Add a number of seconds to the time tuple. @@ -61,9 +61,9 @@ class Vector: timestamp: TimeTuple vectors: tuple[int, int, int, int] - def __init__(self, vectors, previous_time, time_step): + def __init__(self, vectors, time: TimeTuple): self.vectors = vectors - self.timestamp = previous_time + time_step + self.timestamp = time @dataclass @@ -87,7 +87,7 @@ class MagL1a: Number of seconds of data SHCOARSE : int Mission elapsed time - vector_samples : list[Vector] + vectors : list[Vector] List of magnetic vector samples, starting at start_time """ @@ -99,7 +99,25 @@ class MagL1a: expected_vector_count: int seconds_of_data: int SHCOARSE: int - vector_samples: list + vectors: list - # def __post_init__(self): - # TODO: Assign timestamps to vectors, convert list to Vector type + def __post_init__(self): + """ + Convert the vector list to a vector list with timestamps associated. + + The first vector starts at start_time, then each subsequent vector time is + computed by adding 1/vectors_per_second to the previous vector's time. + + This replaces self.vectors with a list of Vector objects. + """ + sample_time_interval = 1 / self.vectors_per_second + previous_time = self.start_time + for index, vector in enumerate(self.vectors): + if index == 0: + new_vector = Vector(vector, self.start_time) + else: + new_vector = Vector(vector, previous_time + sample_time_interval) + + previous_time = new_vector.timestamp + + self.vectors[index] = new_vector diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index 965d26b88..97a9b9249 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -4,7 +4,12 @@ from imap_processing.mag.l0.decom_mag import decom_packets from imap_processing.mag.l1a.mag_l1a import process_vector_data -from imap_processing.mag.l1a.mag_l1a_data import MAX_FINE_TIME, TimeTuple +from imap_processing.mag.l1a.mag_l1a_data import ( + MAX_FINE_TIME, + MagL1a, + TimeTuple, + Vector, +) def test_process_vector_data(): @@ -54,3 +59,22 @@ def test_time_tuple(): test_add = test_time_tuple + (1000 / MAX_FINE_TIME) assert test_add == TimeTuple(439067319, 83) + + +def test_vector_time(): + # TODO finish writing this + test_data = MagL1a( + True, + True, + TimeTuple(10000, 0), + 2, # 2 vectors per second + 4, + 2, + 0, + [(1, 2, 3, 4), (1, 2, 3, 4), (2, 2, 2, 3), (3, 3, 3, 4)], + ) + + assert test_data.vectors[0] == Vector((1, 2, 3, 4), TimeTuple(10000, 0)) + assert test_data.vectors[1] == Vector((1, 2, 3, 4), TimeTuple(10000, 32768)) + assert test_data.vectors[2] == Vector((2, 2, 2, 3), TimeTuple(10001, 1)) + assert test_data.vectors[3] == Vector((3, 3, 3, 4), TimeTuple(10001, 32769)) From 9fe2638967f7561dff355d5e3493d3a388f49867 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 1 Apr 2024 08:45:02 -0600 Subject: [PATCH 07/12] Finishing MAG L1A processing and tests --- imap_processing/mag/l0/decom_mag.py | 67 ++------- imap_processing/mag/l1a/mag_l1a.py | 159 +++----------------- imap_processing/mag/l1a/mag_l1a_data.py | 134 ++++++++++++++++- imap_processing/tests/mag/test_mag_decom.py | 22 ++- imap_processing/tests/mag/test_mag_l1a.py | 51 ++++++- 5 files changed, 227 insertions(+), 206 deletions(-) diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py index 49aade53e..ce0687893 100644 --- a/imap_processing/mag/l0/decom_mag.py +++ b/imap_processing/mag/l0/decom_mag.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) -def decom_packets(packet_file_path: str | Path) -> list[MagL0]: +def decom_packets(packet_file_path: str | Path) -> (list[MagL0], list[MagL0]): """Decom MAG data packets using MAG packet definition. Parameters @@ -31,9 +31,9 @@ def decom_packets(packet_file_path: str | Path) -> list[MagL0]: Returns ------- - data : list[MagL0] - A list of MAG L0 data classes, including both burst and normal packets. (the - packet type is defined in each instance of L0.) + data : list[MagL0], list[MagL0] + 2 lists of MAG L0 data classes, one containing normal mode packets, one + containing burst mode packets. [norm, burst] """ # Define paths xtce_document = Path( @@ -43,7 +43,8 @@ def decom_packets(packet_file_path: str | Path) -> list[MagL0]: packet_definition = xtcedef.XtcePacketDefinition(xtce_document) mag_parser = parser.PacketParser(packet_definition) - data_list = [] + norm_data = [] + burst_data = [] with open(packet_file_path, "rb") as binary_data: mag_packets = mag_parser.generator(binary_data) @@ -57,59 +58,17 @@ def decom_packets(packet_file_path: str | Path) -> list[MagL0]: else item.raw_value for item in packet.data.values() ] - data_list.append(MagL0(CcsdsData(packet.header), *values)) - - return data_list - - -def export_to_xarray(l0_data: list[MagL0]): - """Generate xarray files for "raw" MAG CDF files from MagL0 data. - - Mag outputs "RAW" CDF files just after decomming. These have the immediately - post-decom data, with raw binary data for the vectors instead of vector values. - - Parameters - ---------- - l0_data: list[MagL0] - A list of MagL0 datapoints - - Returns - ------- - norm_data : xr.Dataset - xarray dataset for generating burst data CDFs - burst_data : xr.Dataset - xarray dataset for generating burst data CDFs - """ - # TODO split by mago and magi using primary sensor - norm_data = [] - burst_data = [] + if apid == Mode.NORMAL: + norm_data.append(MagL0(CcsdsData(packet.header), *values)) + else: + burst_data.append(MagL0(CcsdsData(packet.header), *values)) - for packet in l0_data: - if packet.ccsds_header.PKT_APID == Mode.NORMAL: - norm_data.append(packet) - if packet.ccsds_header.PKT_APID == Mode.BURST: - burst_data.append(packet) - - norm_dataset = None - burst_dataset = None - - if len(norm_data) > 0: - norm_dataset = generate_dataset( - norm_data, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output() - ) - if len(burst_data) > 0: - burst_dataset = generate_dataset( - burst_data, mag_cdf_attrs.mag_l1a_burst_raw_attrs.output() - ) - - return norm_dataset, burst_dataset + return norm_data, burst_data def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset: """ - Generate a CDF dataset from the sorted L0 MAG data. - - Used to create 2 similar datasets, for norm and burst data. + Generate a CDF dataset from the sorted raw L0 MAG data. Parameters ---------- @@ -124,6 +83,8 @@ def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset: dataset : xr.Dataset xarray dataset with proper CDF attributes and shape. """ + # TODO: Correct CDF attributes from email + vector_data = np.zeros((len(l0_data), len(l0_data[0].VECTORS))) shcoarse_data = np.zeros(len(l0_data), dtype="datetime64[ns]") diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index cdec1321b..7c8594cc4 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -2,9 +2,8 @@ import logging -import numpy as np - from imap_processing.cdf.utils import write_cdf +from imap_processing.mag import mag_cdf_attrs from imap_processing.mag.l0 import decom_mag from imap_processing.mag.l0.mag_l0_data import MagL0 from imap_processing.mag.l1a.mag_l1a_data import MagL1a, TimeTuple @@ -21,17 +20,23 @@ def mag_l1a(packet_filepath): packet_filepath : Packet files for processing """ - mag_l0 = decom_mag.decom_packets(packet_filepath) - - mag_norm_raw, mag_burst_raw = decom_mag.export_to_xarray(mag_l0) + norm_data, burst_data = decom_mag.decom_packets(packet_filepath) - if mag_norm_raw is not None: + if norm_data is not None: + mag_norm_raw = decom_mag.generate_dataset( + norm_data, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output() + ) file = write_cdf(mag_norm_raw) - logger.info(f"Created CDF file at {file}") + logger.info(f"Created RAW CDF file at {file}") - if mag_burst_raw is not None: + mago_norm, magi_norm = process_packets(norm_data) + + if burst_data is not None: + mag_burst_raw = decom_mag.generate_dataset( + burst_data, mag_cdf_attrs.mag_l1a_burst_raw_attrs.output() + ) file = write_cdf(mag_burst_raw) - logger.info(f"Created CDF file at {file}") + logger.info(f"Created RAW CDF file at {file}") def process_packets(mag_l0_list: list[MagL0]): @@ -45,8 +50,8 @@ def process_packets(mag_l0_list: list[MagL0]): Returns ------- - (list[MagL1a], list[MagL1a]) - Tuple containing MagI and MagO L1A data classes + mago, magi: (list[MagL1a], list[MagL1a]) + Tuple containing MagO and MagI L1A data classes """ magi = [] @@ -70,11 +75,11 @@ def process_packets(mag_l0_list: list[MagL0]): total_primary_vectors = seconds_per_packet * mag_l0.PRI_VECSEC total_secondary_vectors = seconds_per_packet * mag_l0.SEC_VECSEC - primary_vectors, secondary_vectors = process_vector_data( + primary_vectors, secondary_vectors = MagL1a.process_vector_data( mag_l0.VECTORS, total_primary_vectors, total_secondary_vectors ) - # Primary sensor is MAGO + # Primary sensor is MAGO (most common expected case) if mag_l0.PRI_SENS == 0: mago_l1a = MagL1a( True, @@ -98,7 +103,7 @@ def process_packets(mag_l0_list: list[MagL0]): secondary_vectors, ) # Primary sensor is MAGI - if mag_l0.PRI_SENS == 1: + else: magi_l1a = MagL1a( False, bool(mag_l0.MAGI_ACT), @@ -121,129 +126,7 @@ def process_packets(mag_l0_list: list[MagL0]): secondary_vectors, ) - magi.append(magi_l1a) mago.append(mago_l1a) + magi.append(magi_l1a) - return magi, mago - - -def process_vector_data( - vector_data: np.ndarray, primary_count: int, secondary_count: int -) -> (list[tuple], list[tuple]): - """ - Given raw packet data, process into Vectors. - - Vectors are grouped into primary sensor and secondary sensor, and returned as a - tuple (primary sensor vectors, secondary sensor vectors) - - Written by MAG instrument team - - Parameters - ---------- - vector_data : np.ndarray - Raw vector data, in bytes. Contains both primary and secondary vector data - (first primary, then secondary) - primary_count : int - Count of the number of primary vectors - secondary_count : int - Count of the number of secondary vectors - - Returns - ------- - (primary, secondary) - Two arrays, each containing tuples of (x, y, z, sample_range) for each vector - sample. - """ - - # TODO: error handling - def to_signed16(n): - n = n & 0xFFFF - return n | (-(n & 0x8000)) - - pos = 0 - primary_vectors = [] - secondary_vectors = [] - - for i in range(primary_count + secondary_count): # 0..63 say - x, y, z, rng = 0, 0, 0, 0 - if i % 4 == 0: # start at bit 0, take 8 bits + 8bits - # pos = 0, 25, 50... - x = ( - ((vector_data[pos + 0] & 0xFF) << 8) - | ((vector_data[pos + 1] & 0xFF) << 0) - ) & 0xFFFF - y = ( - ((vector_data[pos + 2] & 0xFF) << 8) - | ((vector_data[pos + 3] & 0xFF) << 0) - ) & 0xFFFF - z = ( - ((vector_data[pos + 4] & 0xFF) << 8) - | ((vector_data[pos + 5] & 0xFF) << 0) - ) & 0xFFFF - rng = (vector_data[pos + 6] >> 6) & 0x3 - pos += 6 - elif i % 4 == 1: # start at bit 2, take 6 bits, 8 bit, 2 bits per vector - # pos = 6, 31... - x = ( - ((vector_data[pos + 0] & 0x3F) << 10) - | ((vector_data[pos + 1] & 0xFF) << 2) - | ((vector_data[pos + 2] >> 6) & 0x03) - ) & 0xFFFF - y = ( - ((vector_data[pos + 2] & 0x3F) << 10) - | ((vector_data[pos + 3] & 0xFF) << 2) - | ((vector_data[pos + 4] >> 6) & 0x03) - ) & 0xFFFF - z = ( - ((vector_data[pos + 4] & 0x3F) << 10) - | ((vector_data[pos + 5] & 0xFF) << 2) - | ((vector_data[pos + 6] >> 6) & 0x03) - ) & 0xFFFF - rng = (vector_data[pos + 6] >> 4) & 0x3 - pos += 6 - elif i % 4 == 2: # start at bit 4, take 4 bits, 8 bits, 4 bits per vector - # pos = 12, 37... - x = ( - ((vector_data[pos + 0] & 0x0F) << 12) - | ((vector_data[pos + 1] & 0xFF) << 4) - | ((vector_data[pos + 2] >> 4) & 0x0F) - ) & 0xFFFF - y = ( - ((vector_data[pos + 2] & 0x0F) << 12) - | ((vector_data[pos + 3] & 0xFF) << 4) - | ((vector_data[pos + 4] >> 4) & 0x0F) - ) & 0xFFFF - z = ( - ((vector_data[pos + 4] & 0x0F) << 12) - | ((vector_data[pos + 5] & 0xFF) << 4) - | ((vector_data[pos + 6] >> 4) & 0x0F) - ) & 0xFFFF - rng = (vector_data[pos + 6] >> 2) & 0x3 - pos += 6 - elif i % 4 == 3: # start at bit 6, take 2 bits, 8 bits, 6 bits per vector - # pos = 18, 43... - x = ( - ((vector_data[pos + 0] & 0x03) << 14) - | ((vector_data[pos + 1] & 0xFF) << 6) - | ((vector_data[pos + 2] >> 2) & 0x3F) - ) & 0xFFFF - y = ( - ((vector_data[pos + 2] & 0x03) << 14) - | ((vector_data[pos + 3] & 0xFF) << 6) - | ((vector_data[pos + 4] >> 2) & 0x3F) - ) & 0xFFFF - z = ( - ((vector_data[pos + 4] & 0x03) << 14) - | ((vector_data[pos + 5] & 0xFF) << 6) - | ((vector_data[pos + 6] >> 2) & 0x3F) - ) & 0xFFFF - rng = (vector_data[pos + 6] >> 0) & 0x3 - pos += 7 - - vector = (to_signed16(x), to_signed16(y), to_signed16(z), rng) - if i < primary_count: - primary_vectors.append(vector) - else: - secondary_vectors.append(vector) - - return (primary_vectors, secondary_vectors) + return mago, magi diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index b265d9317..cbf164e26 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -2,6 +2,8 @@ from dataclasses import dataclass from math import floor +import numpy as np + MAX_FINE_TIME = 65535 # maximum 16 bit unsigned int @@ -59,11 +61,17 @@ class Vector: # TODO: This timestamp should be in J2000/datetime64 timestamp: TimeTuple - vectors: tuple[int, int, int, int] + x: int + y: int + z: int + rng: int def __init__(self, vectors, time: TimeTuple): - self.vectors = vectors self.timestamp = time + self.x = vectors[0] + self.y = vectors[1] + self.z = vectors[2] + self.rng = vectors[3] @dataclass @@ -121,3 +129,125 @@ def __post_init__(self): previous_time = new_vector.timestamp self.vectors[index] = new_vector + + @staticmethod + def process_vector_data( + vector_data: np.ndarray, primary_count: int, secondary_count: int + ) -> (list[tuple], list[tuple]): + """ + Given raw packet data, process into Vectors. + + Vectors are grouped into primary sensor and secondary sensor, and returned as a + tuple (primary sensor vectors, secondary sensor vectors) + + Written by MAG instrument team + + Parameters + ---------- + vector_data : np.ndarray + Raw vector data, in bytes. Contains both primary and secondary vector data + (first primary, then secondary) + primary_count : int + Count of the number of primary vectors + secondary_count : int + Count of the number of secondary vectors + + Returns + ------- + (primary, secondary) + Two arrays, each containing tuples of (x, y, z, sample_range) for each + vector sample. + """ + + # TODO: error handling + def to_signed16(n): + n = n & 0xFFFF + return n | (-(n & 0x8000)) + + pos = 0 + primary_vectors = [] + secondary_vectors = [] + + for i in range(primary_count + secondary_count): # 0..63 say + x, y, z, rng = 0, 0, 0, 0 + if i % 4 == 0: # start at bit 0, take 8 bits + 8bits + # pos = 0, 25, 50... + x = ( + ((vector_data[pos + 0] & 0xFF) << 8) + | ((vector_data[pos + 1] & 0xFF) << 0) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0xFF) << 8) + | ((vector_data[pos + 3] & 0xFF) << 0) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0xFF) << 8) + | ((vector_data[pos + 5] & 0xFF) << 0) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 6) & 0x3 + pos += 6 + elif i % 4 == 1: # start at bit 2, take 6 bits, 8 bit, 2 bits per vector + # pos = 6, 31... + x = ( + ((vector_data[pos + 0] & 0x3F) << 10) + | ((vector_data[pos + 1] & 0xFF) << 2) + | ((vector_data[pos + 2] >> 6) & 0x03) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x3F) << 10) + | ((vector_data[pos + 3] & 0xFF) << 2) + | ((vector_data[pos + 4] >> 6) & 0x03) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x3F) << 10) + | ((vector_data[pos + 5] & 0xFF) << 2) + | ((vector_data[pos + 6] >> 6) & 0x03) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 4) & 0x3 + pos += 6 + elif i % 4 == 2: # start at bit 4, take 4 bits, 8 bits, 4 bits per vector + # pos = 12, 37... + x = ( + ((vector_data[pos + 0] & 0x0F) << 12) + | ((vector_data[pos + 1] & 0xFF) << 4) + | ((vector_data[pos + 2] >> 4) & 0x0F) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x0F) << 12) + | ((vector_data[pos + 3] & 0xFF) << 4) + | ((vector_data[pos + 4] >> 4) & 0x0F) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x0F) << 12) + | ((vector_data[pos + 5] & 0xFF) << 4) + | ((vector_data[pos + 6] >> 4) & 0x0F) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 2) & 0x3 + pos += 6 + elif i % 4 == 3: # start at bit 6, take 2 bits, 8 bits, 6 bits per vector + # pos = 18, 43... + x = ( + ((vector_data[pos + 0] & 0x03) << 14) + | ((vector_data[pos + 1] & 0xFF) << 6) + | ((vector_data[pos + 2] >> 2) & 0x3F) + ) & 0xFFFF + y = ( + ((vector_data[pos + 2] & 0x03) << 14) + | ((vector_data[pos + 3] & 0xFF) << 6) + | ((vector_data[pos + 4] >> 2) & 0x3F) + ) & 0xFFFF + z = ( + ((vector_data[pos + 4] & 0x03) << 14) + | ((vector_data[pos + 5] & 0xFF) << 6) + | ((vector_data[pos + 6] >> 2) & 0x3F) + ) & 0xFFFF + rng = (vector_data[pos + 6] >> 0) & 0x3 + pos += 7 + + vector = (to_signed16(x), to_signed16(y), to_signed16(z), rng) + if i < primary_count: + primary_vectors.append(vector) + else: + secondary_vectors.append(vector) + + return (primary_vectors, secondary_vectors) diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index 9840f0c22..785fb0b1a 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -5,13 +5,16 @@ from imap_processing.cdf import global_attrs from imap_processing.cdf.utils import write_cdf -from imap_processing.mag.l0.decom_mag import decom_packets, export_to_xarray +from imap_processing.mag import mag_cdf_attrs +from imap_processing.mag.l0.decom_mag import decom_packets, generate_dataset def test_mag_decom(): current_directory = Path(__file__).parent burst_test_file = current_directory / "mag_l0_test_data.pkts" - l0 = decom_packets(burst_test_file) + norm, burst = decom_packets(burst_test_file) + + l0 = burst + norm expected_output = pd.read_csv(current_directory / "mag_l0_test_output.csv") for index, test in enumerate(l0): @@ -46,9 +49,13 @@ def test_mag_decom(): def test_mag_raw_xarray(): current_directory = Path(__file__).parent burst_test_file = current_directory / "mag_l0_test_data.pkts" - l0 = decom_packets(str(burst_test_file)) + l0_norm, l0_burst = decom_packets(str(burst_test_file)) + + norm_data = generate_dataset(l0_norm, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output()) + burst_data = generate_dataset( + l0_burst, mag_cdf_attrs.mag_l1a_burst_raw_attrs.output() + ) - norm_data, burst_data = export_to_xarray(l0) required_attrs = list( global_attrs.GlobalInstrumentAttrs("", "", "").output().keys() ) @@ -69,9 +76,12 @@ def test_mag_raw_xarray(): def test_mag_raw_cdf_generation(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l0_test_data.pkts" - l0 = decom_packets(str(test_file)) + l0_norm, l0_burst = decom_packets(str(test_file)) - norm_data, burst_data = export_to_xarray(l0) + norm_data = generate_dataset(l0_norm, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output()) + burst_data = generate_dataset( + l0_burst, mag_cdf_attrs.mag_l1a_burst_raw_attrs.output() + ) output = write_cdf(norm_data) assert output.exists() diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index 97a9b9249..f0fd6c594 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -3,7 +3,7 @@ import pandas as pd from imap_processing.mag.l0.decom_mag import decom_packets -from imap_processing.mag.l1a.mag_l1a import process_vector_data +from imap_processing.mag.l1a.mag_l1a import process_packets from imap_processing.mag.l1a.mag_l1a_data import ( MAX_FINE_TIME, MagL1a, @@ -12,27 +12,65 @@ ) +def test_compare_validation_data(): + current_directory = Path(__file__).parent + test_file = current_directory / "mag_l1_test_data.pkts" + # Test file contains only normal packets + l0, _ = decom_packets(str(test_file)) + + l1_mago, l1_magi = process_packets(l0) + + assert len(l1_mago) == 6 + assert len(l1_magi) == 6 + + validation_data = pd.read_csv(current_directory / "mag_l1a_test_output.csv") + + vector_index = 0 + + # Validation data does not have differing timestamps + for index in validation_data.index: + # Sequence in validation data starts at 5 + # Mago is primary, Magi is secondary in test data + l1_pri = l1_mago[validation_data["sequence"][index] - 5] + l1_sec = l1_magi[validation_data["sequence"][index] - 5] + + assert l1_pri.vectors[vector_index].x == validation_data["x_pri"][index] + assert l1_pri.vectors[vector_index].y == validation_data["y_pri"][index] + assert l1_pri.vectors[vector_index].z == validation_data["z_pri"][index] + assert l1_pri.vectors[vector_index].rng == validation_data["rng_pri"][index] + + assert l1_sec.vectors[vector_index].x == validation_data["x_sec"][index] + assert l1_sec.vectors[vector_index].y == validation_data["y_sec"][index] + assert l1_sec.vectors[vector_index].z == validation_data["z_sec"][index] + assert l1_sec.vectors[vector_index].rng == validation_data["rng_sec"][index] + + vector_index = ( + 0 if vector_index == l1_pri.expected_vector_count - 1 else vector_index + 1 + ) + + def test_process_vector_data(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l1_test_data.pkts" l0 = decom_packets(str(test_file)) - mag_l0 = l0[0] + + mag_l0 = l0[0][0] + + # TODO rewrite this test with reverse-calculated unsigned 16 bit ints total_primary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.PRI_VECSEC total_secondary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.SEC_VECSEC - test_vectors = l0[0].VECTORS + test_vectors = mag_l0.VECTORS # 36 bytes - (primary_vectors, secondary_vectors) = process_vector_data( + (primary_vectors, secondary_vectors) = MagL1a.process_vector_data( test_vectors, total_primary_vectors, total_secondary_vectors ) validation_data = pd.read_csv(current_directory / "mag_l1a_test_output.csv") - print(validation_data.index) for index in range(total_primary_vectors): - # print(validation_data.iloc[index]) assert primary_vectors[index][0] == validation_data.iloc[index]["x_pri"] assert primary_vectors[index][1] == validation_data.iloc[index]["y_pri"] assert primary_vectors[index][2] == validation_data.iloc[index]["z_pri"] @@ -62,7 +100,6 @@ def test_time_tuple(): def test_vector_time(): - # TODO finish writing this test_data = MagL1a( True, True, From 2b5e3cc15f93999062ae9b21c11e79cbc0d2b538 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 1 Apr 2024 10:46:02 -0600 Subject: [PATCH 08/12] Switching return to dictionaries instead of tuples --- imap_processing/mag/l0/decom_mag.py | 10 +++++----- imap_processing/mag/l1a/mag_l1a.py | 16 +++++++++------- imap_processing/tests/mag/test_mag_decom.py | 12 ++++++++---- imap_processing/tests/mag/test_mag_l1a.py | 8 +++++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py index ce0687893..342c3cded 100644 --- a/imap_processing/mag/l0/decom_mag.py +++ b/imap_processing/mag/l0/decom_mag.py @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) -def decom_packets(packet_file_path: str | Path) -> (list[MagL0], list[MagL0]): +def decom_packets(packet_file_path: str | Path) -> dict[str, list[MagL0]]: """Decom MAG data packets using MAG packet definition. Parameters @@ -31,9 +31,9 @@ def decom_packets(packet_file_path: str | Path) -> (list[MagL0], list[MagL0]): Returns ------- - data : list[MagL0], list[MagL0] - 2 lists of MAG L0 data classes, one containing normal mode packets, one - containing burst mode packets. [norm, burst] + data_dict : dict[str, list[MagL0]] + A dict with 2 keys pointing to lists of MAG L0 data classes. "norm" corresponds + to normal mode packets, "burst" corresponds to burst mode packets. """ # Define paths xtce_document = Path( @@ -63,7 +63,7 @@ def decom_packets(packet_file_path: str | Path) -> (list[MagL0], list[MagL0]): else: burst_data.append(MagL0(CcsdsData(packet.header), *values)) - return norm_data, burst_data + return {"norm": norm_data, "burst": burst_data} def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset: diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index 7c8594cc4..319fb7def 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -20,7 +20,10 @@ def mag_l1a(packet_filepath): packet_filepath : Packet files for processing """ - norm_data, burst_data = decom_mag.decom_packets(packet_filepath) + packets = decom_mag.decom_packets(packet_filepath) + + norm_data = packets["norm"] + burst_data = packets["burst"] if norm_data is not None: mag_norm_raw = decom_mag.generate_dataset( @@ -29,8 +32,6 @@ def mag_l1a(packet_filepath): file = write_cdf(mag_norm_raw) logger.info(f"Created RAW CDF file at {file}") - mago_norm, magi_norm = process_packets(norm_data) - if burst_data is not None: mag_burst_raw = decom_mag.generate_dataset( burst_data, mag_cdf_attrs.mag_l1a_burst_raw_attrs.output() @@ -39,7 +40,7 @@ def mag_l1a(packet_filepath): logger.info(f"Created RAW CDF file at {file}") -def process_packets(mag_l0_list: list[MagL0]): +def process_packets(mag_l0_list: list[MagL0]) -> dict[str, list[MagL1a]]: """ Given a list of MagL0 packets, process them into MagO and MagI L1A data classes. @@ -50,8 +51,9 @@ def process_packets(mag_l0_list: list[MagL0]): Returns ------- - mago, magi: (list[MagL1a], list[MagL1a]) - Tuple containing MagO and MagI L1A data classes + packet_dict: dict[str, list[MagL1a]] + Dictionary containing two keys: "mago" which points to a list of mago MagL1A + objects, and "magi" which points to a list of magi MagL1A objects. """ magi = [] @@ -129,4 +131,4 @@ def process_packets(mag_l0_list: list[MagL0]): mago.append(mago_l1a) magi.append(magi_l1a) - return mago, magi + return {"mago": mago, "magi": magi} diff --git a/imap_processing/tests/mag/test_mag_decom.py b/imap_processing/tests/mag/test_mag_decom.py index 785fb0b1a..f02c13e6a 100644 --- a/imap_processing/tests/mag/test_mag_decom.py +++ b/imap_processing/tests/mag/test_mag_decom.py @@ -12,9 +12,9 @@ def test_mag_decom(): current_directory = Path(__file__).parent burst_test_file = current_directory / "mag_l0_test_data.pkts" - norm, burst = decom_packets(burst_test_file) + packets = decom_packets(str(burst_test_file)) - l0 = burst + norm + l0 = packets["burst"] + packets["norm"] expected_output = pd.read_csv(current_directory / "mag_l0_test_output.csv") for index, test in enumerate(l0): @@ -49,7 +49,9 @@ def test_mag_decom(): def test_mag_raw_xarray(): current_directory = Path(__file__).parent burst_test_file = current_directory / "mag_l0_test_data.pkts" - l0_norm, l0_burst = decom_packets(str(burst_test_file)) + packets = decom_packets(str(burst_test_file)) + l0_norm = packets["norm"] + l0_burst = packets["burst"] norm_data = generate_dataset(l0_norm, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output()) burst_data = generate_dataset( @@ -76,7 +78,9 @@ def test_mag_raw_xarray(): def test_mag_raw_cdf_generation(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l0_test_data.pkts" - l0_norm, l0_burst = decom_packets(str(test_file)) + packets = decom_packets(str(test_file)) + l0_norm = packets["norm"] + l0_burst = packets["burst"] norm_data = generate_dataset(l0_norm, mag_cdf_attrs.mag_l1a_norm_raw_attrs.output()) burst_data = generate_dataset( diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index f0fd6c594..98f83b380 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -16,9 +16,11 @@ def test_compare_validation_data(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l1_test_data.pkts" # Test file contains only normal packets - l0, _ = decom_packets(str(test_file)) + l0 = decom_packets(str(test_file)) - l1_mago, l1_magi = process_packets(l0) + l1 = process_packets(l0["norm"]) + l1_mago = l1["mago"] + l1_magi = l1["magi"] assert len(l1_mago) == 6 assert len(l1_magi) == 6 @@ -54,7 +56,7 @@ def test_process_vector_data(): test_file = current_directory / "mag_l1_test_data.pkts" l0 = decom_packets(str(test_file)) - mag_l0 = l0[0][0] + mag_l0 = l0["norm"][0] # TODO rewrite this test with reverse-calculated unsigned 16 bit ints From d64c9743db7619987925a2caa60b0bdb1cb279c8 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 1 Apr 2024 15:45:57 -0600 Subject: [PATCH 09/12] Adding an additional test --- imap_processing/mag/l1a/mag_l1a_data.py | 6 ++++ imap_processing/tests/mag/test_mag_l1a.py | 40 ++++++++++------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index cbf164e26..93ca5401d 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -168,6 +168,12 @@ def to_signed16(n): primary_vectors = [] secondary_vectors = [] + # Since the vectors are stored as 50 bit chunks but accessed via hex (4 bit + # chunks) there is some shifting required for processing the bytes. + # However, from a bit processing perspective, the first 48 bits of each 50 bit + # chunk corresponds to 3 16 bit signed integers. The last 2 bits are the sensor + # range. + for i in range(primary_count + secondary_count): # 0..63 say x, y, z, rng = 0, 0, 0, 0 if i % 4 == 0: # start at bit 0, take 8 bits + 8bits diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index 98f83b380..300f80b4b 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -1,5 +1,6 @@ from pathlib import Path +import numpy as np import pandas as pd from imap_processing.mag.l0.decom_mag import decom_packets @@ -52,36 +53,31 @@ def test_compare_validation_data(): def test_process_vector_data(): - current_directory = Path(__file__).parent - test_file = current_directory / "mag_l1_test_data.pkts" - l0 = decom_packets(str(test_file)) - - mag_l0 = l0["norm"][0] + expected_vector_data = [[1001, 1002, -3001, 3], [2001, -2002, -3333, 1]] - # TODO rewrite this test with reverse-calculated unsigned 16 bit ints + # 100 bits, created by hand by appending all bits from expected_vector_data into one + # hex string, with range being 2 bits (so the second half is offset from the hex + # values) + hex_string = "03E903EAF447C1F47E0BBCBED0" + input_data = np.frombuffer(bytes.fromhex(hex_string), dtype=np.dtype(">b")) - total_primary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.PRI_VECSEC - total_secondary_vectors = (mag_l0.PUS_SSUBTYPE + 1) * mag_l0.SEC_VECSEC - - test_vectors = mag_l0.VECTORS + total_primary_vectors = 1 + total_secondary_vectors = 1 # 36 bytes (primary_vectors, secondary_vectors) = MagL1a.process_vector_data( - test_vectors, total_primary_vectors, total_secondary_vectors + input_data, total_primary_vectors, total_secondary_vectors ) - validation_data = pd.read_csv(current_directory / "mag_l1a_test_output.csv") - - for index in range(total_primary_vectors): - assert primary_vectors[index][0] == validation_data.iloc[index]["x_pri"] - assert primary_vectors[index][1] == validation_data.iloc[index]["y_pri"] - assert primary_vectors[index][2] == validation_data.iloc[index]["z_pri"] - assert primary_vectors[index][3] == validation_data.iloc[index]["rng_pri"] + assert primary_vectors[0][0] == expected_vector_data[0][0] + assert primary_vectors[0][1] == expected_vector_data[0][1] + assert primary_vectors[0][2] == expected_vector_data[0][2] + assert primary_vectors[0][3] == expected_vector_data[0][3] - assert secondary_vectors[index][0] == validation_data.iloc[index]["x_sec"] - assert secondary_vectors[index][1] == validation_data.iloc[index]["y_sec"] - assert secondary_vectors[index][2] == validation_data.iloc[index]["z_sec"] - assert secondary_vectors[index][3] == validation_data.iloc[index]["rng_sec"] + assert secondary_vectors[0][0] == expected_vector_data[1][0] + assert secondary_vectors[0][1] == expected_vector_data[1][1] + assert secondary_vectors[0][2] == expected_vector_data[1][2] + assert secondary_vectors[0][3] == expected_vector_data[1][3] def test_time_tuple(): From b2153b2de76a0b85659cf57344f5ebf1f600e220 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Tue, 2 Apr 2024 09:03:32 -0600 Subject: [PATCH 10/12] Updating from PR comments --- imap_processing/mag/l0/mag_l0_data.py | 18 ++++++++---------- imap_processing/mag/l1a/mag_l1a_data.py | 14 ++++---------- imap_processing/tests/mag/test_mag_l1a.py | 2 +- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/imap_processing/mag/l0/mag_l0_data.py b/imap_processing/mag/l0/mag_l0_data.py index 63f7c6db3..5e9f5d34c 100644 --- a/imap_processing/mag/l0/mag_l0_data.py +++ b/imap_processing/mag/l0/mag_l0_data.py @@ -1,4 +1,5 @@ """Dataclasses for Level 0 MAG data.""" +from __future__ import annotations from dataclasses import dataclass from enum import IntEnum @@ -92,7 +93,7 @@ class MagL0: PRI_FNTM: int SEC_COARSETM: int SEC_FNTM: int - VECTORS: np.ndarray + VECTORS: np.ndarray | str def __post_init__(self): """Convert Vectors attribute from string to bytearray if needed. @@ -100,15 +101,12 @@ def __post_init__(self): Also convert encoded "VECSEC" (vectors per second) into proper vectors per second values """ - if isinstance(self.VECTORS, str): - # Convert string output from space_packet_parser to bytearray - self.VECTORS = np.frombuffer( - int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"), - dtype=np.dtype(">b"), - ) - - if isinstance(self.VECTORS, bytearray): - self.VECTORS = np.array(self.VECTORS, dtype=np.dtype(">b")) + # Convert string output from space_packet_parser to numpy array of + # big-endian bytes + self.VECTORS = np.frombuffer( + int(self.VECTORS, 2).to_bytes(len(self.VECTORS) // 8, "big"), + dtype=np.dtype(">b"), + ) # Remove buffer from end of vectors if len(self.VECTORS) % 2: diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index 93ca5401d..3ee2addaf 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -25,7 +25,7 @@ def __add__(self, seconds: float): Parameters ---------- - seconds : int + seconds : float Number of seconds to add Returns @@ -119,16 +119,10 @@ def __post_init__(self): This replaces self.vectors with a list of Vector objects. """ sample_time_interval = 1 / self.vectors_per_second - previous_time = self.start_time + current_time = self.start_time for index, vector in enumerate(self.vectors): - if index == 0: - new_vector = Vector(vector, self.start_time) - else: - new_vector = Vector(vector, previous_time + sample_time_interval) - - previous_time = new_vector.timestamp - - self.vectors[index] = new_vector + self.vectors[index] = Vector(vector, current_time) + current_time = self.vectors[index].timestamp + sample_time_interval @staticmethod def process_vector_data( diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index 300f80b4b..42826c26a 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -17,7 +17,7 @@ def test_compare_validation_data(): current_directory = Path(__file__).parent test_file = current_directory / "mag_l1_test_data.pkts" # Test file contains only normal packets - l0 = decom_packets(str(test_file)) + l0 = decom_packets(test_file) l1 = process_packets(l0["norm"]) l1_mago = l1["mago"] From 2f7a67e316896bf3ef8338aa0901cabd3e094068 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Apr 2024 20:01:46 +0000 Subject: [PATCH 11/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- imap_processing/mag/l0/mag_l0_data.py | 1 + imap_processing/mag/l1a/mag_l1a_data.py | 1 + 2 files changed, 2 insertions(+) diff --git a/imap_processing/mag/l0/mag_l0_data.py b/imap_processing/mag/l0/mag_l0_data.py index 5e9f5d34c..95f597f99 100644 --- a/imap_processing/mag/l0/mag_l0_data.py +++ b/imap_processing/mag/l0/mag_l0_data.py @@ -1,4 +1,5 @@ """Dataclasses for Level 0 MAG data.""" + from __future__ import annotations from dataclasses import dataclass diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index 3ee2addaf..4bdbb0ef8 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -1,4 +1,5 @@ """Data classes for storing and processing MAG Level 1A data.""" + from dataclasses import dataclass from math import floor From e43e9b93be9e3bfec326cc25c5e5d6958f2ed905 Mon Sep 17 00:00:00 2001 From: Maxine Hartnett Date: Mon, 8 Apr 2024 10:25:34 -0600 Subject: [PATCH 12/12] Updating comments based on PR --- imap_processing/mag/l0/mag_l0_data.py | 9 ++++++--- imap_processing/mag/l1a/mag_l1a.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/imap_processing/mag/l0/mag_l0_data.py b/imap_processing/mag/l0/mag_l0_data.py index 5e9f5d34c..f9c41b8b7 100644 --- a/imap_processing/mag/l0/mag_l0_data.py +++ b/imap_processing/mag/l0/mag_l0_data.py @@ -1,4 +1,5 @@ """Dataclasses for Level 0 MAG data.""" + from __future__ import annotations from dataclasses import dataclass @@ -69,9 +70,10 @@ class MagL0: Secondary Coarse Time for first vector, seconds SEC_FNTM: int Secondary Fine Time for first vector, subseconds - VECTORS: bin + VECTORS: np.ndarray | str MAG Science Vectors - divide based on PRI_VECSEC and PUS_SSUBTYPE for vector - counts + counts. There is a post init call to convert a string into a numpy array - + the only place it is a string is in the class initialization. """ ccsds_header: CcsdsData @@ -108,7 +110,8 @@ def __post_init__(self): dtype=np.dtype(">b"), ) - # Remove buffer from end of vectors + # Remove buffer from end of vectors. Vector data needs to be in 50 bit chunks, + # and may have an extra byte at the end from CCSDS padding. if len(self.VECTORS) % 2: self.VECTORS = self.VECTORS[:-1] diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index 319fb7def..c45b4477c 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -69,7 +69,7 @@ def process_packets(mag_l0_list: list[MagL0]) -> dict[str, list[MagL1a]]: # seconds of data in this packet is the SUBTYPE plus 1 seconds_per_packet = mag_l0.PUS_SSUBTYPE + 1 - # now we know the number of secs of data in the packet, and the data rates of + # now we know the number of seconds of data in the packet, and the data rates of # each sensor, we can calculate how much data is in this packet and where the # byte boundaries are.