MNT: Update file naming convention via imap-data-access upgrade

docs/source/code-documentation/tools/xarray-to-cdf.rst

@@ -126,16 +126,16 @@ The following python code is the **minimum** code you'd need to store this data
       # The following attributes are specific to JIM.
       "Instrument_type": "Particles (space)",
       "Data_type": "L1_Mode_Description>Level-1 Mode Description",
-      "Data_version": "01",
+      "Data_version": "001",
       "Descriptor": "JIM>Just an Ion Monitor",
       "TEXT": (
          "JIM is a fictitious instrument that counts ions at 3 different energies on "
          "the IMAP mission. This is where a detailed description of the instrument "
          "goes, as well as the type of data in the file.  For example, if a (mode) "
          "or (descriptor) exist they can be described here."
       ),
-      "Logical_file_id": "imap_jim_l1_mode_description_20250101_v01",
-      "Logical_source": "imap_jim_l1_mode_description",
+      "Logical_file_id": "imap_jim_l1_mode-description_20250101_v001",
+      "Logical_source": "imap_jim_l1_mode-description",
       "Logical_source_description": "IMAP Mission JIM Instrument Level-1 (mode) (description) Data.",
    }
 

imap_processing/cdf/global_attrs.py

@@ -149,7 +149,7 @@ class GlobalDataLevelAttrs:
     data_type : str
         The level of data, ex "L1>Level-1"
     logical_source : str
-        The source of the data, ex "imap_idex_l1"
+        The source of the data, ex "imap_idex_l1_sci"
     logical_source_desc : str
         The description of the data, ex "IMAP Mission IDEX Instrument Level-1 Data."
     instrument_base : GlobalInstrumentAttrs

imap_processing/cdf/utils.py

@@ -2,7 +2,9 @@
 
 import logging
 from pathlib import Path
+from typing import Optional
 
+import imap_data_access
 import numpy as np
 import xarray as xr
 from cdflib.xarray import xarray_to_cdf
@@ -39,7 +41,7 @@ def calc_start_time(shcoarse_time: int):
     return launch_time + time_delta
 
 
-def write_cdf(data: xr.Dataset, filepath: Path):
+def write_cdf(dataset: xr.Dataset, directory: Optional[Path] = None):
     """Write the contents of "data" to a CDF file using cdflib.xarray_to_cdf.
 
     This function determines the file name to use from the global attributes,
@@ -51,7 +53,7 @@ def write_cdf(data: xr.Dataset, filepath: Path):
 
     Parameters
     ----------
-        data : xarray.Dataset
+        dataset : xarray.Dataset
             The dataset object to convert to a CDF
         filepath: Path
             The output path, including filename, to write the CDF to.
@@ -61,20 +63,39 @@ def write_cdf(data: xr.Dataset, filepath: Path):
         pathlib.Path
             Path to the file created
     """
-    if not filepath.parent.exists():
-        logger.info("The directory does not exist, creating directory %s", filepath)
-        filepath.parent.mkdir(parents=True)
-
+    # Use the directory if provided, otherwise use the default
+    directory = directory or imap_data_access.config["DATA_DIR"]
+
+    # Create the filename from the global attributes
+    # Logical_source looks like "imap_swe_l2_counts-1min"
+    instrument, data_level, descriptor = dataset.attrs["Logical_source"].split("_")[1:]
+    start_time = np.datetime_as_string(dataset["Epoch"].values[0], unit="D").replace(
+        "-", ""
+    )
+    version = f"v{int(dataset.attrs['Data_version']):03d}"  # vXXX
+    repointing = dataset.attrs.get("Repointing", None)
+    science_file = imap_data_access.ScienceFilePath.generate_from_inputs(
+        instrument=instrument,
+        data_level=data_level,
+        descriptor=descriptor,
+        start_time=start_time,
+        version=version,
+        repointing=repointing,
+    )
+    file_path = directory / science_file.construct_path()
+    if not file_path.parent.exists():
+        logger.info("The directory does not exist, creating directory %s", directory)
+        file_path.parent.mkdir(parents=True)
     # Insert the final attribute:
     # The Logical_file_id is always the name of the file without the extension
-    data.attrs["Logical_file_id"] = filepath.stem
+    dataset.attrs["Logical_file_id"] = file_path.stem
 
     # Convert the xarray object to a CDF
     xarray_to_cdf(
-        data,
-        str(filepath),
+        dataset,
+        str(file_path),
         datetime64_to_cdftt2000=True,
         terminate_on_warning=True,
     )  # Terminate if not ISTP compliant
 
-    return filepath
+    return file_path

imap_processing/cli.py

@@ -53,9 +53,8 @@ def _parse_args():
             'instrument': 'mag',
             'data_level': 'l0',
             'descriptor': 'sci',
-            'version': 'v00-01',
-            'start_date': '20231212',
-            'end_date': '20231212'
+            'version': 'v001',
+            'start_date': '20231212'
         }]"
     --upload-to-sdc
 
@@ -70,15 +69,13 @@ def _parse_args():
         '"imap_cli --instrument "mag" '
         '--data-level "l1a"'
         ' --start-date "20231212"'
-        '--end-date "20231212"'
-        '--version "v00-01"'
+        '--version "v001"'
         '--dependency "['
         '   {"instrument": "mag",'
         '   "data_level": "l0"',
         '   "descriptor": "sci"',
-        '   "version": "v00-01"',
+        '   "version": "v001"',
         '   "start_date": "20231212"',
-        '   "end_date": "20231212"',
         '}]" --upload-to-sdc"',
     )
     instrument_help = (
@@ -94,9 +91,8 @@ def _parse_args():
         "Example: '[{'instrument': 'mag',"
         "'data_level': 'l0',"
         "'descriptor': 'sci',"
-        "'version': 'v00-01',"
-        "'start_date': '20231212',"
-        "'end_date': '20231212'}]"
+        "'version': 'v001',"
+        "'start_date': '20231212'}]"
     )
 
     parser = argparse.ArgumentParser(prog="imap_cli", description=description)
@@ -123,7 +119,7 @@ def _parse_args():
         "--version",
         type=str,
         required=True,
-        help="Version of the data. Format: vxx-xx",
+        help="Version of the data. Format: vXXX",
     )
     parser.add_argument(
         "--dependency",
@@ -154,7 +150,7 @@ def _validate_args(args):
     if args.instrument not in imap_data_access.VALID_INSTRUMENTS:
         raise ValueError(
             f"{args.instrument} is not in the supported instrument list: "
-            f"{imap_processing.INSTRUMENTS}"
+            f"{imap_data_access.VALID_INSTRUMENTS}"
         )
     if args.data_level not in imap_processing.PROCESSING_LEVELS[args.instrument]:
         raise ValueError(
@@ -178,15 +174,14 @@ class ProcessInstrument(ABC):
             'data_level': 'l0',
             'descriptor': 'sci',
             'version': 'v00-01',
-            'start_date': '20231212',
-            'end_date': '20231212'
+            'start_date': '20231212'
         }]"
     start_date : str
         The start date for the output data. Format: YYYYMMDD
     end_date : str
         The end date for the output data. Format: YYYYMMDD
     version : str
-        The version of the data. Format: vxx-xx
+        The version of the data. Format: vXXX
     upload_to_sdc : bool
         A flag indicating whether to upload the output file to the SDC.
     """
@@ -317,10 +312,10 @@ def process(self):
                     f"{file_paths}. Expected only one dependency."
                 )
             filename_norm = imap_data_access.ScienceFilePath.generate_from_inputs(
-                "mag", "l1a", "raw-norm", self.start_date, self.end_date, self.version
+                "mag", "l1a", "raw-norm", self.start_date, self.version
             ).construct_path()
             filename_burst = imap_data_access.ScienceFilePath.generate_from_inputs(
-                "mag", "l1a", "raw-burst", self.start_date, self.end_date, self.version
+                "mag", "l1a", "raw-burst", self.start_date, self.version
             ).construct_path()
             mag_l1a(file_paths[0], filename_norm, filename_burst)
 
@@ -349,7 +344,7 @@ class Swe(ProcessInstrument):
     def process(self):
         """Perform SWE specific processing."""
         # self.file_path example:
-        # imap/swe/l1a/2023/09/imap_swe_l1a_sci_20230927_20230927_v01-00.cdf
+        # imap/swe/l1a/2023/09/imap_swe_l1a_sci_20230927_v001.cdf
         dependencies = self.download_dependencies()
         print(f"Processing SWE {self.data_level}")
 
@@ -358,20 +353,7 @@ def process(self):
         if self.data_level == "l1a":
             processed_data = swe_l1a(Path(dependencies[0]))
             for data in processed_data:
-                # write data to cdf
-                file = imap_data_access.ScienceFilePath.generate_from_inputs(
-                    "swe",
-                    "l1a",
-                    data["descriptor"],
-                    self.start_date,
-                    self.end_date,
-                    self.version,
-                )
-
-                cdf_file_path = write_cdf(
-                    data=data["data"], filepath=file.construct_path()
-                )
-
+                cdf_file_path = write_cdf(data)
                 print(f"processed file path: {cdf_file_path}")
 
                 if self.upload_to_sdc:
@@ -385,7 +367,7 @@ def process(self):
             # TODO: Update this descriptor
             descriptor = "test"
             file = imap_data_access.ScienceFilePath.generate_from_inputs(
-                "swe", "l1b", descriptor, self.start_date, self.end_date, self.version
+                "swe", "l1b", descriptor, self.start_date, self.version
             )
 
             cdf_file_path = write_cdf(

imap_processing/codice/cdf_attrs.py

@@ -44,14 +44,14 @@
 
 codice_l1a_global_attrs = GlobalDataLevelAttrs(
     data_type="L1A->Level-1A",
-    logical_source="imap_codice_l1a",
+    logical_source="imap_codice_l1a_sci",
     logical_source_desc="IMAP Mission CoDICE Instrument Level-1A Data",
     instrument_base=codice_base,
 )
 
 codice_l1b_global_attrs = GlobalDataLevelAttrs(
     data_type="L1B->Level-1B",
-    logical_source="imap_cpdice_l1b",
+    logical_source="imap_codice_l1b_sci",
     logical_source_desc="IMAP Mission CoDICE Instrument Level-1B Data",
     instrument_base=codice_base,
 )

imap_processing/codice/codice_l1a.py

@@ -14,7 +14,6 @@
 
 import logging
 
-import imap_data_access
 import space_packet_parser
 
 from imap_processing.cdf.utils import write_cdf
@@ -47,10 +46,7 @@ def codice_l1a(packets: list[space_packet_parser.parser.Packet]) -> str:
         else:
             logger.debug(f"{apid} is currently not supported")
 
-    file = imap_data_access.ScienceFilePath.generate_from_inputs(
-        "codice", "l1a", "hk", "20210101", "20210102", "v01-01"
-    )
     # Write data to CDF
-    cdf_filename = write_cdf(data, file.construct_path())
+    cdf_filename = write_cdf(data)
 
     return cdf_filename

imap_processing/idex/idex_cdf_attrs.py

@@ -28,13 +28,16 @@
 
 idex_l1_global_attrs = GlobalDataLevelAttrs(
     "L1>Level-1",
-    "imap_idex_l1",
+    "imap_idex_l1_sci",
     "IMAP Mission IDEX Instrument Level-1 Data.",
     idex_base,
 )
 
 idex_l2_global_attrs = GlobalDataLevelAttrs(
-    "L2>Level-2", "imap_idex_l2", "IMAP Mission IDEX Instrument Level-2 Data", idex_base
+    "L2>Level-2",
+    "imap_idex_l2_sci",
+    "IMAP Mission IDEX Instrument Level-2 Data",
+    idex_base,
 )
 
 l1_data_base = ScienceAttrs(

imap_processing/idex/idex_packet_parser.py

@@ -473,11 +473,12 @@ class PacketParser:
 
     Examples
     --------
-        >>> # Print out the data in a L0 file
-        >>> from imap_processing.idex.idex_packet_parser import PacketParser
-        >>> l0_file = "imap_processing/tests/idex/imap_idex_l0_20230725_v01-00.pkts"
-        >>> l1_data = PacketParser(l0_file)
-        >>> l1_data.write_l1_cdf()
+    .. code-block:: python
+        # Print out the data in a L0 file
+        from imap_processing.idex.idex_packet_parser import PacketParser
+        l0_file = "imap_processing/tests/idex/imap_idex_l0_sci_20230725_v001.pkts"
+        l1_data = PacketParser(l0_file)
+        l1_data.write_l1_cdf()
 
     """
 

imap_processing/mag/mag_cdf_attrs.py

@@ -29,21 +29,24 @@
     # TODO: data type should include "norm" and "burst" L1A-norm>Level-1A-normal-rate
     "L1A>Level-1A",
     # Should also include data type
-    logical_source="imap_mag_l1a",
+    # TODO: replace "sci" with descriptor "norm" / "burst"
+    logical_source="imap_mag_l1a_sci",
     logical_source_desc="IMAP Mission MAG Instrument Level-1A Data.",
     instrument_base=mag_base,
 )
 
 mag_l1b_attrs = GlobalDataLevelAttrs(
     "L1A>Level-1B",
-    logical_source="imap_mag_l1b",
+    # TODO: replace "sci" with descriptor "norm" / "burst"
+    logical_source="imap_mag_l1b_sci",
     logical_source_desc="IMAP Mission MAG Instrument Level-1B Data.",
     instrument_base=mag_base,
 )
 
 mag_l1c_attrs = GlobalDataLevelAttrs(
     "L1A>Level-1C",
-    logical_source="imap_mag_l1c",
+    # TODO: replace "sci" with descriptor "norm" / "burst"
+    logical_source="imap_mag_l1c_sci",
     logical_source_desc="IMAP Mission MAG Instrument Level-1C Data.",
     instrument_base=mag_base,
 )

imap_processing/swe/__init__.py

@@ -1 +1 @@
-__version__ = "01-00"
+__version__ = "001"

imap_processing/swe/l1a/swe_l1a.py

@@ -51,9 +51,12 @@ def swe_l1a(file_path):
             sorted_packets = sort_by_time(grouped_data[apid], "SHCOARSE")
             data = create_dataset(packets=sorted_packets)
 
+        # TODO: add this mode and descriptor into the global attributes directly
         # write data to CDF
         mode = f"{data['APP_MODE'].data[0]}-" if apid == SWEAPID.SWE_APP_HK else ""
         descriptor = f"{mode}{filename_descriptors.get(apid)}"
+        # Update the global descriptor
+        data.attrs["descriptor"] = descriptor
 
-        processed_data.append({"data": data, "descriptor": descriptor})
+        processed_data.append(data)
     return processed_data

imap_processing/swe/l1b/swe_l1b.py

@@ -46,8 +46,12 @@ def swe_l1b(l1a_dataset: xr.Dataset):
         if data is None:
             print("No data to write to CDF")
             return
+        # TODO: replace "sci" with proper descriptor, or add to global attributes
+        data.attrs["descriptor"] = "sci"
     else:
         data = eu_data
         # Update global attributes to l1b global attributes
         data.attrs.update(swe_cdf_attrs.swe_l1b_global_attrs.output())
+        # TODO: replace "sci" with proper descriptor, or add to global attributes
+        data.attrs["descriptor"] = "sci"
     return data

imap_processing/swe/swe_cdf_attrs.py

@@ -37,7 +37,7 @@
             )
         ],
         "Logical_file_id": "FILL ME IN AT FILE CREATION",
-        "Logical_source": "imap_swe_l1a",
+        "Logical_source": "imap_swe_l1a_sci",
         "Logical_source_description": ["IMAP Mission SWE Instrument Level-1 Data"],
 
 
@@ -117,14 +117,16 @@
 
 swe_l1a_global_attrs = GlobalDataLevelAttrs(
     data_type="L1A->Level-1A",
-    logical_source="imap_swe_l1a",
+    # TODO: replace "sci" with proper descriptor
+    logical_source="imap_swe_l1a_sci",
     logical_source_desc="IMAP Mission SWE Instrument Level-1A Data",
     instrument_base=swe_base,
 )
 
 swe_l1b_global_attrs = GlobalDataLevelAttrs(
     data_type="L1B->Level-1B",
-    logical_source="imap_swe_l1b",
+    # TODO: replace "sci" with proper descriptor
+    logical_source="imap_swe_l1b_sci",
     logical_source_desc="IMAP Mission SWE Instrument Level-1B Data",
     instrument_base=swe_base,
 )