L1 cdf creation (IMAP-Science-Operations-Center#179)

* Extracting and adding new data to the IDEX L1

* Adding some of the values to the XML, rather than shifting bits in the code
(work in progress, need to do a few more variables)

* Finishing up having space packet parser decom

* Getting rid of unecessary bit masksi8kt9gloh

* making changes based on comments

* Formatting the file after the upstream sync

* Adding L1 CDF creation

* Forgot to commit poetry changes

* Making changes based on Greg's comments,
as well as issues 1-8 from SPDF

* Fixing import statements

* Changing version into a string

* Adding more descriptions, units, and labels in the attributes

* Fixing the way strings are inserted into the CDF

* Fixing issues with white spice in the text of attrs

* Files are finally ISTP compliant!

* Fixing a couple things for ISTP

* Had the contents of the 32 bit things reversed

* Fixing the packet definition for the CCSDS file

* More fixes to ensure ISTP compliance

* Updating the file name

* touching finishes on the L1 CDF

* Adding fixture for temp directory in tests

* Fixing one of the ruff errors by adding a namedtuple

* Adding new "write_cdf" function that writed cdfs based on attributes

* Getting rid of a few more magic numbers

* Updating the packet definition

imap_processing/cdf_utils.py

@@ -0,0 +1,102 @@
+import os
+
+import numpy as np
+import xarray as xr
+from cdflib.xarray import xarray_to_cdf
+
+# Recommended FILLVAL for all integers
+INT_FILLVAL = np.iinfo(np.int64).min
+# Recommended FILLVALL for all floats
+DOUBLE_FILLVAL = np.float64(-1.0e31)
+# Recommended min/max Epoch based on MMS approved values
+MIN_EPOCH = -315575942816000000
+MAX_EPOCH = 946728069183000000
+
+global_base = {
+    "Project": "STP>Solar-Terrestrial Physics",
+    "Source_name": "IMAP>Interstellar Mapping and Acceleration Probe",
+    "Discipline": "Solar Physics>Heliospheric Physics",
+    "PI_name": "Dr. David J. McComas",
+    "PI_affiliation": [
+        "Princeton Plasma Physics Laboratory",
+        "100 Stellarator Road, Princeton, NJ 08540",
+    ],
+    "Instrument_type": "Particles (space)",
+    "Mission_group": "IMAP>Interstellar Mapping and Acceleration Probe",
+}
+
+epoch_attrs = {
+    "CATDESC": "Default time",
+    "FIELDNAM": "Epoch",
+    "FILLVAL": INT_FILLVAL,
+    "FORMAT": "a2",
+    "LABLAXIS": "Epoch",
+    "UNITS": "ns",
+    "VALIDMIN": MIN_EPOCH,
+    "VALIDMAX": MAX_EPOCH,
+    "VAR_TYPE": "support_data",
+    "SCALETYP": "linear",
+    "MONOTON": "INCREASE",
+    "TIME_BASE": "J2000",
+    "TIME_SCALE": "Terrestrial Time",
+    "REFERENCE_POSITION": "Rotating Earth Geoid",
+}
+
+
+def write_cdf(data: xr.Dataset, description: str = "", directory: str = ""):
+    """Write the contents of "data" to a CDF file using cdflib.xarray_to_cdf.
+
+    This function determines the file name to use from the global attributes,
+    fills in the the final attributes, and converts the whole dataset to a CDF.
+    The date in the file name is determined by the time of the first Epoch in the
+    xarray Dataset.  The first 3 file name fields (mission, instrument, level) are
+    determined by the "Logical_source" attribute.  The version is determiend from
+    "Data_version".
+
+    Parameters
+    ----------
+        data (xarray.Dataset): The dataset object to convert to a CDF
+        description (str): The description to insert into the file name after the
+                            orbit, before the SPICE field.  No underscores allowed.
+        directory (str): The directory to write the file to
+
+    Returns
+    -------
+        str
+            The name of the file created
+    """
+    # Determine the start date of the data in the file,
+    # based on the time of the first dust impact
+    file_start_date = data["Epoch"][0].data
+    date_string = np.datetime_as_string(file_start_date, unit="D").replace("-", "")
+
+    # Determine the optional "description" field
+    description = (
+        description
+        if (description.startswith("_") or not description)
+        else f"_{description}"
+    )
+
+    # Determine the file name based on the attributes in the xarray
+    filename = (
+        data.attrs["Logical_source"]
+        + "_"
+        + date_string
+        + description
+        + f"_v{data.attrs['Data_version']}.cdf"
+    )
+    filename_and_path = os.path.join(directory, filename)
+
+    # Insert the final attribute:
+    # The Logical_file_id is always the name of the file without the extension
+    data.attrs["Logical_file_id"] = filename.split(".")[0]
+
+    # Convert the xarray object to a CDF
+    xarray_to_cdf(
+        data,
+        filename_and_path,
+        datetime64_to_cdftt2000=True,
+        terminate_on_warning=True,
+    )  # Terminate if not ISTP compliant
+
+    return filename_and_path

imap_processing/idex/__init__.py

@@ -0,0 +1,2 @@
+# Set IDEX software version here
+__version__ = "01"

imap_processing/idex/idex_cdf_attrs.py

@@ -0,0 +1,173 @@
+from imap_processing import cdf_utils
+from imap_processing.idex import __version__
+
+# Valid min/maxes
+
+# Data is in a 12 bit unsigned INT
+DATA_MIN = 0  # It could go down to 0 in theory
+DATA_MAX = 4096  # It cannot exceed 4096 (2^12)
+
+# Samples span 130 microseconds at the most, and values are allowed to be negative
+SAMPLE_RATE_MIN = -130  # All might be negative
+SAMPLE_RATE_MAX = 130  # All might be positive
+
+# Global Attributes
+idex_global_base = {
+    "Data_type": "L1>Level-1",
+    "Data_version": __version__,
+    "Descriptor": "IDEX>Interstellar Dust Experiment",
+    "TEXT": (
+        "The Interstellar Dust Experiment (IDEX) is a time-of-flight (TOF) "
+        "dust impact ionization mass spectrometer on the IMAP mission that "
+        "provides the elemental composition, speed, and mass distributions "
+        "of interstellar dust and interplanetary dust particles. Each record "
+        "contains the data from a single dust impact. See "
+        "https://imap.princeton.edu/instruments/idex for more details."
+    ),
+    "Logical_file_id": "FILL ME IN AT FILE CREATION",
+} | cdf_utils.global_base
+
+idex_l1_global_attrs = {
+    "Data_type": "L1>Level-1",
+    "Logical_source": "imap_idex_l1",
+    "Logical_source_description": "IMAP Mission IDEX Instrument Level-1 Data.",
+} | idex_global_base
+
+idex_l2_global_attrs = {
+    "Data_type": "L2>Level-2",
+    "Logical_source": "imap_idex_l2",
+    "Logical_source_description": "IMAP Mission IDEX Instrument Level-2 Data",
+} | idex_global_base
+
+# L1 variables base dictionaries
+# (these need to be filled in by the variable dictionaries below)
+l1_data_base = {
+    "DEPEND_0": "Epoch",
+    "DISPLAY_TYPE": "spectrogram",
+    "FILLVAL": cdf_utils.INT_FILLVAL,
+    "FORMAT": "I12",
+    "UNITS": "dN",
+    "VALIDMIN": DATA_MIN,
+    "VALIDMAX": DATA_MAX,
+    "VAR_TYPE": "data",
+    "SCALETYP": "linear",
+    # "VARIABLE_PURPOSE" tells CDAWeb which variables are worth plotting
+    "VARIABLE_PURPOSE": "PRIMARY",
+}
+
+l1_tof_base = {"DEPEND_1": "Time_High_SR"} | l1_data_base
+
+l1_target_base = {"DEPEND_1": "Time_Low_SR"} | l1_data_base
+
+sample_rate_base = {
+    "DEPEND_0": "Epoch",
+    "FILLVAL": cdf_utils.DOUBLE_FILLVAL,
+    "FORMAT": "F12.5",
+    "LABLAXIS": "Time",
+    "UNITS": "microseconds",
+    "VALIDMIN": SAMPLE_RATE_MIN,
+    "VALIDMAX": SAMPLE_RATE_MAX,
+    "VAR_TYPE": "support_data",
+    "SCALETYP": "linear",
+    "VAR_NOTES": (
+        "The number of microseconds since the event.  "
+        "0 is the start of data collection, negative "
+        "numbers represent data collected prior to a dust event"
+    ),
+}
+
+trigger_base = {
+    "DEPEND_0": "Epoch",
+    "FILLVAL": cdf_utils.INT_FILLVAL,
+    "FORMAT": "I12",
+    "VALIDMIN": 0,  # All values are positive integers or 0 by design
+    "VAR_TYPE": "data",
+    "DISPLAY_TYPE": "no_plot",
+}
+
+# L1 Attribute Dictionaries
+low_sr_attrs = {
+    "CATDESC": "Low sample rate time steps for a dust event.",
+    "FIELDNAM": "Low Sample Rate Time",
+    "VAR_NOTES": (
+        "The low sample rate in microseconds. "
+        "Steps are approximately 1/4.025 microseconds in duration. "
+        "Used by the Ion_Grid, Target_Low, and Target_High variables."
+    ),
+} | sample_rate_base
+
+high_sr_attrs = {
+    "CATDESC": "High sample rate time steps for a dust event.",
+    "FIELDNAM": "High Sample Rate Time",
+    "VAR_NOTES": (
+        "The high sample rate in microseconds. "
+        "Steps are approximately 1/260 microseconds in duration. "
+        "Used by the TOF_High, TOF_Mid, and TOF_Low variables."
+    ),
+} | sample_rate_base
+
+tof_high_attrs = {
+    "CATDESC": "Time of flight waveform on the high-gain channel",
+    "FIELDNAM": "High Gain Time of Flight",
+    "LABLAXIS": "TOF High Ampl.",
+    "VAR_NOTES": (
+        "High gain channel of the time-of-flight signal. "
+        "Sampled at 260 Megasamples per second, with a 10-bit resolution. "
+        "Data is used to quantify dust composition."
+    ),
+} | l1_tof_base
+
+tof_mid_attrs = {
+    "CATDESC": "Time of flight waveform on the mid-gain channel",
+    "FIELDNAM": "Mid Gain Time of Flight",
+    "LABLAXIS": "TOF Mid Ampl.",
+    "VAR_NOTES": (
+        "Mid gain channel of the time-of-flight signal. "
+        "Sampled at 260 Megasamples per second, with a 10-bit resolution. "
+        "Data is used to quantify dust composition."
+    ),
+} | l1_tof_base
+
+tof_low_attrs = {
+    "CATDESC": "Time of flight waveform on the low-gain channel",
+    "FIELDNAM": "Low Gain Time of Flight",
+    "LABLAXIS": "TOF Low Ampl.",
+    "VAR_NOTES": (
+        "Low gain channel of the time-of-flight signal. "
+        "Sampled at 260 Megasamples per second, with a 10-bit resolution. "
+        "Data is used to quantify dust composition."
+    ),
+} | l1_tof_base
+
+target_low_attrs = {
+    "CATDESC": "Target low charge sensitive amplifier waveform",
+    "FIELDNAM": "Low Target Signal",
+    "LABLAXIS": "Low Target Ampl.",
+    "VAR_NOTES": (
+        "Low gain channel of IDEX's target signal. "
+        "Sampled at 3.75 Msps with 12-bit resolution. "
+        "Data is used to quantify dust charge. "
+    ),
+} | l1_target_base
+
+target_high_attrs = {
+    "CATDESC": "Ion grid charge sensitive amplifier waveform",
+    "FIELDNAM": "High Target Signal",
+    "LABLAXIS": "High Target Ampl.",
+    "VAR_NOTES": (
+        "High gain channel of IDEX's target signal. "
+        "Sampled at 3.75 Msps with 12-bit resolution. "
+        "Data is used to quantify dust charge."
+    ),
+} | l1_target_base
+
+ion_grid_attrs = {
+    "CATDESC": "Ion grid charge sensitive amplifier waveform data",
+    "FIELDNAM": "Ion Grid Signal",
+    "LABLAXIS": "Ion Grid Ampl.",
+    "VAR_NOTES": (
+        "This is the ion grid signal from IDEX. "
+        "Sampled at 3.75 Msps with 12-bit resolution. "
+        "Data is used to quantify dust charge."
+    ),
+} | l1_target_base