forked from IMAP-Science-Operations-Center/imap_processing
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request IMAP-Science-Operations-Center#288 from bourque/in…
…itial-codice-l1a Initial CoDICE L1A Processing Algorithm
Showing
19 changed files
with
824 additions
and
408 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
__version__ = "01" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,116 @@ | ||
"""CDF attrs for CoDICE. | ||
This website provides information about what variables are required and what | ||
their value should be: | ||
https://spdf.gsfc.nasa.gov/istp_guide/istp_guide.html | ||
For further details, see the documentation provided at | ||
https://imap-processing.readthedocs.io/en/latest/development/CDFs/cdf_requirements.html | ||
""" | ||
|
||
|
||
from imap_processing.cdf.defaults import GlobalConstants | ||
from imap_processing.cdf.global_attrs import ( | ||
AttrBase, | ||
GlobalDataLevelAttrs, | ||
GlobalInstrumentAttrs, | ||
ScienceAttrs, | ||
StringAttrs, | ||
) | ||
from imap_processing.codice import __version__ | ||
|
||
descriptor = "CoDICE>Compact Dual Ion Composition Experiment" | ||
codice_description_text = ( | ||
"The Compact Dual Ion Composition Experiment (CoDICE) will measure the " | ||
"distributions and composition of interstellar pickup ions (PUIs), " | ||
"particles that make it through the heliosheath into the heliosphere. " | ||
"CoDICE also collects and characterizes solar wind ions including the " | ||
"mass and composition of highly energized particles (called suprathermal) " | ||
"from the Sun. CoDICE combines an electrostatic analyzer(ESA) with a " | ||
"Time-Of-Flight versus Energy (TOF / E) subsystem to simultaneously " | ||
"measure the velocity, arrival direction, ionic charge state, and mass of " | ||
"specific species of ions in the LISM. CoDICE also has a path for higher " | ||
"energy particles to skip the ESA but still get measured by the common " | ||
"TOF / E system. These measurements are critical in determining the Local " | ||
"Interstellar Medium (LISM) composition and flow properties, the origin of " | ||
"the enigmatic suprathermal tails on the solar wind distributions and " | ||
"advance understanding of the acceleration of particles in the heliosphere." | ||
) | ||
|
||
codice_base = GlobalInstrumentAttrs( | ||
version=__version__, descriptor=descriptor, text=codice_description_text | ||
) | ||
|
||
codice_l1a_global_attrs = GlobalDataLevelAttrs( | ||
data_type="L1A->Level-1A", | ||
logical_source="imap_codice_l1a", | ||
logical_source_desc="IMAP Mission CoDICE Instrument Level-1A Data", | ||
instrument_base=codice_base, | ||
) | ||
|
||
codice_l1b_global_attrs = GlobalDataLevelAttrs( | ||
data_type="L1B->Level-1B", | ||
logical_source="imap_cpdice_l1b", | ||
logical_source_desc="IMAP Mission CoDICE Instrument Level-1B Data", | ||
instrument_base=codice_base, | ||
) | ||
|
||
int_base = AttrBase( | ||
validmin=0, | ||
validmax=GlobalConstants.INT_MAXVAL, | ||
format="I12", | ||
var_type="support_data", | ||
display_type="no_plot", | ||
) | ||
|
||
float_base = AttrBase( | ||
validmin=0, | ||
validmax=GlobalConstants.INT_MAXVAL, | ||
format="I12", | ||
var_type="support_data", | ||
display_type="no_plot", | ||
) | ||
|
||
string_base = StringAttrs( | ||
depend_0="Epoch", | ||
) | ||
|
||
codice_metadata_attrs = ScienceAttrs( | ||
validmin=0, | ||
validmax=GlobalConstants.INT_MAXVAL, | ||
display_type="no_plot", | ||
depend_0="Epoch", | ||
format="I12", | ||
units="dN", | ||
var_type="support_data", | ||
variable_purpose="PRIMARY", | ||
) | ||
|
||
# TODO: ask CoDICE team about valid min and max values of these data | ||
l1a_science_attrs = ScienceAttrs( | ||
validmin=0, | ||
validmax=GlobalConstants.INT_MAXVAL, | ||
display_type="spectrogram", | ||
depend_0="Epoch", | ||
depend_1="Energy", | ||
depend_2="Counts", | ||
format="I12", | ||
units="dN", | ||
var_type="data", | ||
variable_purpose="PRIMARY", | ||
) | ||
|
||
l1b_science_attrs = ScienceAttrs( | ||
validmin=0, | ||
validmax=GlobalConstants.INT_MAXVAL, | ||
display_type="spectrogram", | ||
depend_0="Epoch", | ||
depend_1="Energy", | ||
depend_2="Angle", | ||
depend_3="Rates", | ||
format="I12", | ||
units="dN", | ||
var_type="data", | ||
variable_purpose="PRIMARY", | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
"""Perform CoDICE L0 processing. | ||
This module contains a function to decommutate CoDICE CCSDS packets using | ||
XTCE packet definitions. | ||
For more information on this process and the latest versions of the packet | ||
definitions, see https://lasp.colorado.edu/galaxy/display/IMAP/CoDICE. | ||
Use | ||
--- | ||
from imap_processing.codice.codice_l0 import decom_packets | ||
packet_file = '/path/to/raw_ccsds_20230822_122700Z_idle.bin' | ||
xtce_document = '/path/to/P_COD_NHK.xml' | ||
packet_list = decom_packets(packet_file, xtce_document) | ||
""" | ||
|
||
from imap_processing import decom, imap_module_directory | ||
|
||
|
||
def decom_packets(packet_file: str) -> list: | ||
"""Decom CoDICE data packets using CoDICE packet definition. | ||
Parameters | ||
---------- | ||
packet_file : str | ||
Path to data packet path with filename. | ||
Returns | ||
------- | ||
list : list | ||
all the unpacked data. | ||
""" | ||
xtce_document = f"{imap_module_directory}/codice/packet_definitions/P_COD_NHK.xml" | ||
return decom.decom_packets(packet_file, xtce_document) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
"""Perform CoDICE l1a processing. | ||
This module contains functions to process decommutated CoDICE packets and create | ||
L1a data products. | ||
Use | ||
--- | ||
from imap_processing.codice.codice_l0 import decom_packets | ||
from imap_processing.codice.codice_l1a import codice_l1a | ||
packets = decom_packets(packet_file, xtce_document) | ||
cdf_filename = codice_l1a(packets) | ||
""" | ||
|
||
import logging | ||
|
||
import space_packet_parser | ||
|
||
from imap_processing.cdf.utils import write_cdf | ||
from imap_processing.codice.utils import CODICEAPID, create_dataset | ||
from imap_processing.utils import group_by_apid, sort_by_time | ||
|
||
|
||
def codice_l1a( | ||
packets: list[space_packet_parser.parser.Packet], cdf_directory: str | ||
) -> str: | ||
"""Process CoDICE l0 data to create l1a data products. | ||
Parameters | ||
---------- | ||
packets : list[space_packet_parser.parser.Packet] | ||
Decom data list that contains all APIDs | ||
cdf_directory : str | ||
The directory in which to write the output CDF file. | ||
Returns | ||
------- | ||
cdf_filename : str | ||
The path to the CDF file that was created | ||
""" | ||
# Group data by APID and sort by time | ||
grouped_data = group_by_apid(packets) | ||
|
||
for apid in grouped_data.keys(): | ||
if apid == CODICEAPID.COD_NHK: | ||
sorted_packets = sort_by_time(grouped_data[apid], "SHCOARSE") | ||
data = create_dataset(packets=sorted_packets) | ||
else: | ||
logging.debug(f"{apid} is currently not supported") | ||
|
||
# Write data to CDF | ||
cdf_filename = write_cdf( | ||
data, | ||
mode="", | ||
description="hk", | ||
directory=cdf_directory, | ||
) | ||
|
||
return cdf_filename |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
"""Tests the L1a processing for decommutated CoDICE data""" | ||
|
||
from pathlib import Path | ||
|
||
import pytest | ||
import space_packet_parser | ||
|
||
from imap_processing import imap_module_directory | ||
from imap_processing.codice.codice_l0 import decom_packets | ||
from imap_processing.codice.codice_l1a import codice_l1a | ||
|
||
|
||
@pytest.fixture(scope="session") | ||
def l0_test_data() -> list: | ||
"""Decom some packets to be used for testing | ||
Returns | ||
------- | ||
packets : list[space_packet_parser.parser.Packet] | ||
A list of decommutated packets for testing | ||
""" | ||
|
||
packet_file = Path( | ||
f"{imap_module_directory}/codice/tests/data/" | ||
f"raw_ccsds_20230822_122700Z_idle.bin" | ||
) | ||
packets = decom_packets(packet_file) | ||
|
||
return packets | ||
|
||
|
||
def test_codice_l1a( | ||
l0_test_data: list[space_packet_parser.parser.Packet], tmp_path: Path | ||
) -> str: | ||
"""Tests the ``codice_l1a`` function and ensured that a proper CDF file | ||
was created | ||
Parameters | ||
---------- | ||
l0_test_data : list[space_packet_parser.parser.Packet] | ||
A list of packets to process | ||
tmp_path : pathlib.PosixPath | ||
pytest fixture used to provide a temporary directory during testing | ||
""" | ||
|
||
cdf_filename = codice_l1a(l0_test_data, tmp_path) | ||
|
||
assert Path(cdf_filename).name == "imap_codice_l1a_hk_20100101_v01.cdf" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,142 @@ | ||
"""Various classes and functions used throughout CoDICE processing. | ||
This module contains utility classes and functions that are used by various | ||
other CoDICE processing modules. | ||
""" | ||
|
||
import collections | ||
import dataclasses | ||
from enum import IntEnum | ||
|
||
import space_packet_parser | ||
import xarray as xr | ||
|
||
from imap_processing.cdf.global_attrs import ConstantCoordinates | ||
from imap_processing.codice import cdf_attrs | ||
|
||
|
||
class CODICEAPID(IntEnum): | ||
"""Create ENUM for CoDICE APIDs. | ||
Parameters | ||
---------- | ||
IntEnum : IntEnum | ||
""" | ||
|
||
COD_AUT = 1120 | ||
COD_NHK = 1136 | ||
COD_EVTMSG = 1137 | ||
COD_MEMDMP = 1138 | ||
COD_SHK = 1139 | ||
COD_RTS = 1141 | ||
COD_DIAG_SNSR_HV = 1145 | ||
COD_DIAG_OPTC_HV = 1146 | ||
COD_DIAG_APDFPGA = 1147 | ||
COD_DIAG_SSDFPGA = 1148 | ||
COD_DIAG_FSW = 1149 | ||
COD_DIAG_SYSVARS = 1150 | ||
COD_LO_IAL = 1152 | ||
COD_LO_PHA = 1153 | ||
COD_LO_INSTRUMENT_COUNTERS = 1154 | ||
COD_LO_PRIORITY_COUNTS = 1155 | ||
COD_LO_SW_SPECIES_COUNTS = 1156 | ||
COD_LO_NSW_SPECIES_COUNTS = 1157 | ||
COD_LO_SW_ANGULAR_COUNTS = 1158 | ||
COD_LO_NSW_ANGULAR_COUNTS = 1159 | ||
COD_HI_IAL = 1168 | ||
COD_HI_PHA = 1169 | ||
COD_HI_INSTRUMENTCOUNTERS = 1170 | ||
COD_HI_OMNI_SPECIES_COUNTS = 1172 | ||
COD_HI_SECT_SPECIES_COUNTS = 1173 | ||
COD_CSTOL_CONFIG = 2457 | ||
|
||
|
||
class CoDICECompression(IntEnum): | ||
"""Create ENUM for CoDICE compression algorithms. | ||
Parameters | ||
---------- | ||
IntEnum : IntEnum | ||
""" | ||
|
||
NO_COMPRESSION = 1 | ||
LOSSY_A = 2 | ||
LOSSY_B = 3 | ||
LOSSLESS = 4 | ||
LOSSY_A_LOSSLESS = 5 | ||
LOSSY_B_LOSSLESS = 6 | ||
|
||
|
||
def add_metadata_to_array( | ||
packet: space_packet_parser.parser.Packet, metadata_arrays: dict | ||
) -> dict: | ||
"""Add metadata to the metadata_arrays. | ||
Parameters | ||
---------- | ||
packet : space_packet_parser.parser.Packet | ||
CODICE data packet | ||
metadata_arrays : dict | ||
Metadata arrays | ||
Returns | ||
------- | ||
metadata_arrays : dict | ||
Updated metadata arrays with values | ||
""" | ||
for key, value in packet.header.items(): | ||
metadata_arrays.setdefault(key, []).append(value.raw_value) | ||
|
||
for key, value in packet.data.items(): | ||
metadata_arrays.setdefault(key, []).append(value.raw_value) | ||
|
||
return metadata_arrays | ||
|
||
|
||
def create_dataset(packets: list[space_packet_parser.parser.Packet]) -> xr.Dataset: | ||
"""Create dataset for each metadata field. | ||
Parameters | ||
---------- | ||
packets : list[space_packet_parser.parser.Packet] | ||
The list of packets to process | ||
Returns | ||
------- | ||
xarray.Dataset | ||
xarray dataset containing the metadata | ||
""" | ||
metadata_arrays = collections.defaultdict(list) | ||
|
||
for packet in packets: | ||
add_metadata_to_array(packet, metadata_arrays) | ||
|
||
epoch_time = xr.DataArray( | ||
metadata_arrays["SHCOARSE"], | ||
name="Epoch", | ||
dims=["Epoch"], | ||
attrs=ConstantCoordinates.EPOCH, | ||
) | ||
|
||
dataset = xr.Dataset( | ||
coords={"Epoch": epoch_time}, | ||
attrs=cdf_attrs.codice_l1a_global_attrs.output(), | ||
) | ||
|
||
for key, value in metadata_arrays.items(): | ||
if key == "SHCOARSE": | ||
continue | ||
else: | ||
dataset[key] = xr.DataArray( | ||
value, | ||
dims=["Epoch"], | ||
attrs=dataclasses.replace( | ||
cdf_attrs.codice_metadata_attrs, | ||
catdesc=key, | ||
fieldname=key, | ||
label_axis=key, | ||
depend_0="Epoch", | ||
).output(), | ||
) | ||
|
||
return dataset |
Empty file.
This file was deleted.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.