From 3b0691aca55e9d8c172225a7153bd4bf2db2b0a1 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Mon, 4 Mar 2024 09:16:09 -0700 Subject: [PATCH] adding l1a cdfs for all apids --- .../tests/ultra/unit/test_ultra_l1a.py | 191 ++++++++++++++++-- imap_processing/ultra/l0/decom_ultra.py | 7 +- imap_processing/ultra/l1a/ultra_l1a.py | 119 +++++++++-- 3 files changed, 281 insertions(+), 36 deletions(-) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1a.py b/imap_processing/tests/ultra/unit/test_ultra_l1a.py index 4d17898a1..b93d6b86f 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1a.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1a.py @@ -5,8 +5,13 @@ from imap_processing.ultra import ultra_cdf_attrs from imap_processing.ultra.l0.decom_ultra import decom_ultra_apids -from imap_processing.ultra.l0.ultra_utils import ULTRA_AUX -from imap_processing.ultra.l1a.ultra_l1a import ultra_l1a, xarray_aux +from imap_processing.ultra.l0.ultra_utils import ( + ULTRA_AUX, + ULTRA_EVENTS, + ULTRA_RATES, + ULTRA_TOF, +) +from imap_processing.ultra.l1a.ultra_l1a import ultra_l1a, xarray @pytest.fixture() @@ -16,11 +21,34 @@ def decom_ultra_aux(ccsds_path, xtce_path): return data_packet_list +@pytest.fixture() +def decom_ultra_rates(ccsds_path, xtce_path): + """Data for decom_ultra_rates""" + data_packet_list = decom_ultra_apids(ccsds_path, xtce_path, ULTRA_RATES.apid[0]) + return data_packet_list + + +@pytest.fixture() +def decom_ultra_events(ccsds_path_events, xtce_path): + """Data for decom_ultra_events""" + data_packet_list = decom_ultra_apids( + ccsds_path_events, xtce_path, ULTRA_EVENTS.apid[0] + ) + return data_packet_list + + +@pytest.fixture() +def decom_ultra_tof(ccsds_path_tof, xtce_path): + """Data for decom_ultra_tof""" + data_packet_list = decom_ultra_apids(ccsds_path_tof, xtce_path, ULTRA_TOF.apid[0]) + return data_packet_list + + def test_xarray_aux(decom_ultra_aux, aux_test_path): - """This function checks that an xarray was + """This function checks that a xarray was successfully created from the decom_ultra_aux data.""" - dataset = xarray_aux(decom_ultra_aux) + dataset = xarray(decom_ultra_aux, ULTRA_AUX.apid[0]) # Spot check string data and attributes spin_period_valid_list = dataset.variables["SPINPERIODVALID"].values.tolist() @@ -59,15 +87,152 @@ def test_xarray_aux(decom_ultra_aux, aux_test_path): assert shcoarse_attr == expected_shcoarse_attr -def test_cdf_aux(ccsds_path, xtce_path, tmp_path, decom_ultra_aux): - """Tests that CDF file is created and contains same attributes as xarray_aux.""" +def test_xarray_rates(decom_ultra_rates, rates_test_path): + """This function checks that a xarray was + successfully created from the decom_ultra_rates data.""" + + dataset = xarray(decom_ultra_rates, ULTRA_RATES.apid[0]) + + # Spot check metadata data and attributes + specific_epoch_data = dataset.sel(Epoch="2022-05-30T22:52:00.184000")["START_RF"] + startrf_list = specific_epoch_data.values.tolist() + startrf_attr = dataset.variables["START_RF"].attrs + + expected_startrf_attr = dataclasses.replace( + ultra_cdf_attrs.ultra_metadata_attrs, + catdesc="START_RF", + fieldname="START_RF", + label_axis="START_RF", + ).output() + + assert startrf_list == decom_ultra_rates["START_RF"][0] + assert startrf_attr == expected_startrf_attr + + +def test_xarray_tof(decom_ultra_tof, tof_test_path): + """This function checks that a xarray was + successfully created from the decom_ultra_tof data.""" + + dataset = xarray(decom_ultra_tof, ULTRA_TOF.apid[0]) + + # Spot check metadata data and attributes + specific_epoch_data = dataset.sel(Epoch="2024-01-24T11:39:21.184000")["PACKETDATA"] + packetdata_list = specific_epoch_data.values.tolist() + packetdata_attr = dataset.variables["PACKETDATA"].attrs + + expected_packetdata_attr = dataclasses.replace( + ultra_cdf_attrs.ultra_metadata_attrs, + catdesc="PACKETDATA", + fieldname="PACKETDATA", + label_axis="PACKETDATA", + depend_1="Row", + depend_2="Column", + units="PIXELS", + ).output() + + for i in range(len(packetdata_list)): + assert (packetdata_list[i] == decom_ultra_tof["PACKETDATA"][i]).all() + + assert packetdata_attr == expected_packetdata_attr + + +def test_xarray_events(decom_ultra_events, events_test_path): + """This function checks that a xarray was + successfully created from the decom_ultra_events data.""" + + dataset = xarray(decom_ultra_events, ULTRA_EVENTS.apid[0]) + + # Spot check metadata data and attributes + specific_epoch_data = dataset.sel(Epoch="2023-08-21T16:14:01.184000")["COIN_TYPE"] + cointype_list = specific_epoch_data.values.tolist() + cointype_attr = dataset.variables["COIN_TYPE"].attrs + + expected_cointype_attr = dataclasses.replace( + ultra_cdf_attrs.ultra_metadata_attrs, + catdesc="COIN_TYPE", + fieldname="COIN_TYPE", + label_axis="COIN_TYPE", + ).output() + + assert cointype_list == decom_ultra_events["COIN_TYPE"][0:2] + assert cointype_attr == expected_cointype_attr + + +def test_cdf_aux( + ccsds_path, + xtce_path, + tmp_path, + decom_ultra_aux, +): + """Tests that CDF file is created and contains same attributes as xarray.""" + # TODO: change test filename with new naming convention + test_data_path_aux = tmp_path / "ultra_l1a_aux_20210101-20210102_v01-01.cdf" + assert not test_data_path_aux.exists() + + ultra_l1a(ccsds_path, xtce_path, test_data_path_aux, ULTRA_AUX.apid[0]) + assert test_data_path_aux.exists() + + dataset_aux = xarray(decom_ultra_aux, ULTRA_AUX.apid[0]) + input_xarray_aux = cdf_to_xarray(test_data_path_aux) + + assert input_xarray_aux.attrs.keys() == dataset_aux.attrs.keys() + + +def test_cdf_rates( + ccsds_path, + xtce_path, + tmp_path, + decom_ultra_rates, +): + """Tests that CDF file is created and contains same attributes as xarray.""" + # TODO: change test filename with new naming convention + test_data_path_rates = tmp_path / "ultra_l1a_rates_20210101-20210102_v01-01.cdf" + assert not test_data_path_rates.exists() + + ultra_l1a(ccsds_path, xtce_path, test_data_path_rates, ULTRA_RATES.apid[0]) + assert test_data_path_rates.exists() + + dataset_rates = xarray(decom_ultra_rates, ULTRA_RATES.apid[0]) + input_xarray_rates = cdf_to_xarray(test_data_path_rates) + + assert input_xarray_rates.attrs.keys() == dataset_rates.attrs.keys() + + +def test_cdf_tof( + ccsds_path_tof, + xtce_path, + tmp_path, + decom_ultra_tof, +): + """Tests that CDF file is created and contains same attributes as xarray.""" # TODO: change test filename with new naming convention - test_data_path = tmp_path / "ultra_l1a_aux_20210101-20210102_v01-01.cdf" + test_data_path_tof = tmp_path / "ultra_l1a_tof_20210101-20210102_v01-01.cdf" + assert not test_data_path_tof.exists() + + ultra_l1a(ccsds_path_tof, xtce_path, test_data_path_tof, ULTRA_TOF.apid[0]) + assert test_data_path_tof.exists() + + dataset_tof = xarray(decom_ultra_tof, ULTRA_TOF.apid[0]) + input_xarray_tof = cdf_to_xarray(test_data_path_tof) + + assert input_xarray_tof.attrs.keys() == dataset_tof.attrs.keys() + + +def test_cdf_events( + ccsds_path_events, + xtce_path, + tmp_path, + decom_ultra_events, +): + """Tests that CDF file is created and contains same attributes as xarray.""" + # TODO: change test filename with new naming convention + test_data_path_events = tmp_path / "ultra_l1a_events_20210101-20210102_v01-01.cdf" + assert not test_data_path_events.exists() + + ultra_l1a(ccsds_path_events, xtce_path, test_data_path_events, ULTRA_EVENTS.apid[0]) + assert test_data_path_events.exists() - assert not test_data_path.exists() - ultra_l1a(ccsds_path, xtce_path, test_data_path) - assert test_data_path.exists() + dataset_events = xarray(decom_ultra_events, ULTRA_EVENTS.apid[0]) + input_xarray_events = cdf_to_xarray(test_data_path_events) - dataset = xarray_aux(decom_ultra_aux) - input_xarray = cdf_to_xarray(test_data_path) - assert input_xarray.attrs.keys() == dataset.attrs.keys() + assert input_xarray_events.attrs.keys() == dataset_events.attrs.keys() diff --git a/imap_processing/ultra/l0/decom_ultra.py b/imap_processing/ultra/l0/decom_ultra.py index 51d25f642..de1d0ba8e 100644 --- a/imap_processing/ultra/l0/decom_ultra.py +++ b/imap_processing/ultra/l0/decom_ultra.py @@ -2,6 +2,7 @@ import logging from collections import defaultdict +from pathlib import Path from imap_processing import decom from imap_processing.ccsds.ccsds_data import CcsdsData @@ -50,15 +51,15 @@ def append_params( append_ccsds_fields(decom_data, ccsds_data) -def decom_ultra_apids(packet_file: str, xtce: str, apid: int): +def decom_ultra_apids(packet_file: Path, xtce: Path, apid: int): """ Unpack and decode Ultra packets using CCSDS format and XTCE packet definitions. Parameters ---------- - packet_file : str + packet_file : Path Path to the CCSDS data packet file. - xtce : str + xtce : Path Path to the XTCE packet definition file. apid : int The APID to process. diff --git a/imap_processing/ultra/l1a/ultra_l1a.py b/imap_processing/ultra/l1a/ultra_l1a.py index 554810bae..796074b78 100644 --- a/imap_processing/ultra/l1a/ultra_l1a.py +++ b/imap_processing/ultra/l1a/ultra_l1a.py @@ -1,25 +1,28 @@ -"""Contains code to perform ULTRA L1a science processing.""" +"""Contains code to perform ULTRA L1a cdf generation.""" import dataclasses import logging from pathlib import Path +import numpy as np import xarray as xr from imap_processing.cdf.global_attrs import ConstantCoordinates from imap_processing.cdf.utils import calc_start_time, write_cdf from imap_processing.ultra import ultra_cdf_attrs -from imap_processing.ultra.l0.decom_ultra import ULTRA_AUX, decom_ultra_apids +from imap_processing.ultra.l0.decom_ultra import ULTRA_TOF, decom_ultra_apids logger = logging.getLogger(__name__) -def xarray_aux(decom_ultra_aux: dict): - """Create xarray for auxiliary packet. +def initiate_data_arrays(decom_ultra: dict, apid: int): + """Initiate xarray data arrays. Parameters ---------- - decom_ultra_aux : dict + decom_ultra : dict Parsed data. + apid : int + Packet APID. Returns ------- @@ -28,7 +31,7 @@ def xarray_aux(decom_ultra_aux: dict): """ # Converted time time_converted = [] - for time in decom_ultra_aux["SHCOARSE"]: + for time in decom_ultra["SHCOARSE"]: time_converted.append(calc_start_time(time)) epoch_time = xr.DataArray( @@ -38,13 +41,70 @@ def xarray_aux(decom_ultra_aux: dict): attrs=ConstantCoordinates.EPOCH, ) - dataset = xr.Dataset( - coords={"Epoch": epoch_time}, - attrs=ultra_cdf_attrs.ultra_l1a_attrs.output(), - ) + if apid != ULTRA_TOF.apid[0]: + dataset = xr.Dataset( + coords={"Epoch": epoch_time}, + attrs=ultra_cdf_attrs.ultra_l1a_attrs.output(), + ) + else: + row = xr.DataArray( + # Number of pixel rows + np.arange(54), + name="Row", + dims=["Row"], + attrs=dataclasses.replace( + ultra_cdf_attrs.ultra_support_attrs, + catdesc="ROW", # TODO: short and long descriptions + fieldname="ROW", + var_type="ignore_data", + ).output(), + ) - for key, value in decom_ultra_aux.items(): - if key in [ + column = xr.DataArray( + # Number of pixel columns + np.arange(180), + name="Column", + dims=["Column"], + attrs=dataclasses.replace( + ultra_cdf_attrs.ultra_support_attrs, + catdesc="COLUMN", # TODO: short and long descriptions + fieldname="COLUMN", + var_type="ignore_data", + ).output(), + ) + + dataset = xr.Dataset( + coords={"Epoch": epoch_time, "Row": row, "Column": column}, + attrs=ultra_cdf_attrs.ultra_l1a_attrs.output(), + ) + + return dataset + + +def xarray(decom_ultra: dict, apid: int): + """Create xarray for packet. + + Parameters + ---------- + decom_ultra : dict + Parsed data. + apid : int + Packet APID. + + Returns + ------- + dataset : xarray.Dataset + Data in xarray format. + """ + dataset = initiate_data_arrays(decom_ultra, apid) + + for key, value in decom_ultra.items(): + # EVENT DATA and FASTDATA_00 have been broken down further + # (see ultra_utils.py) and are therefore not needed. + if key in {"EVENTDATA", "FASTDATA_00"}: + continue + # Packet headers require support attributes + elif key in [ "VERSION", "TYPE", "SEC_HDR_FLG", @@ -58,6 +118,8 @@ def xarray_aux(decom_ultra_aux: dict): catdesc=key, # TODO: short and long descriptions fieldname=key, ).output() + dims = ["Epoch"] + # AUX enums require string attibutes elif key in [ "SPINPERIODVALID", "SPINPHASEVALID", @@ -73,6 +135,21 @@ def xarray_aux(decom_ultra_aux: dict): catdesc=key, # TODO: short and long descriptions fieldname=key, ).output() + dims = ["Epoch"] + # TOF packetdata has multiple dimensions + elif key == "PACKETDATA": + attrs = dataclasses.replace( + ultra_cdf_attrs.ultra_metadata_attrs, + catdesc=key, # TODO: short and long descriptions + fieldname=key, + label_axis=key, + depend_1="Row", + depend_2="Column", + units="PIXELS", + ).output() + dims = ["Epoch", "Row", "Column"] + # Use metadata with a single dimension for + # all other data products else: attrs = dataclasses.replace( ultra_cdf_attrs.ultra_metadata_attrs, @@ -80,20 +157,21 @@ def xarray_aux(decom_ultra_aux: dict): fieldname=key, label_axis=key, ).output() + dims = ["Epoch"] dataset[key] = xr.DataArray( value, name=key, - dims=["Epoch"], + dims=dims, attrs=attrs, ) return dataset -def ultra_l1a(packet_file: Path, xtce: Path, output_filepath: Path): +def ultra_l1a(packet_file: Path, xtce: Path, output_filepath: Path, apid: int): """ - Process ULTRA L0 data into L1A CDF files at cdf_filepath.. + Process ULTRA L0 data into L1A CDF files at output_filepath. Parameters ---------- @@ -103,10 +181,11 @@ def ultra_l1a(packet_file: Path, xtce: Path, output_filepath: Path): Path to the XTCE packet definition file. output_filepath : Path Full directory and filename for CDF file + apid : int + Packet APID. """ - decom_ultra_aux = decom_ultra_apids(packet_file, xtce, ULTRA_AUX.apid[0]) + decom_ultra = decom_ultra_apids(packet_file, xtce, apid) - if decom_ultra_aux: - dataset = xarray_aux(decom_ultra_aux) - write_cdf(dataset, Path(output_filepath)) - logging.info(f"Created CDF file at {output_filepath}") + dataset = xarray(decom_ultra, apid) + write_cdf(dataset, Path(output_filepath)) + logging.info(f"Created CDF file at {output_filepath}")