Merge pull request IMAP-Science-Operations-Center#303 from greglucas/…

…data-dir

Make a default data directory

docs/Makefile

@@ -14,7 +14,7 @@ help:
 
 clean:
 	-rm -rf $(BUILDDIR)/*
-	-rm -rf source/reference/generated
+	-rm -rf source/code-documentation/generated
 
 .PHONY: help Makefile
 

docs/source/code-documentation/index.rst

@@ -22,9 +22,31 @@ Instruments
    hi
    swapi
 
-
 Utility functions can be found in modules within the top package level.
 
+Processing
+----------
+
+To process an instrument, a command line utility is installed with the
+package.  The command line utility is called ``imap_cli`` and
+takes the instrument and level as arguments.  For example, to process
+the CODICE instrument at level 1, the command would be
+
+.. code:: text
+
+    imap_cli --instrument codice --level 1
+
+This will write output files to the default location, which is
+the current working directory + "/imap-data". To change the data
+directory, use the ``--data-dir`` option, or the environment
+variable ``IMAP_DATA_DIR``. For example to use a temporary directory
+
+.. code:: text
+
+      imap_cli --instrument codice --level 1 --data-dir /tmp/imap-data
+      # or equivalently with an environment variable
+      IMAP_DATA_DIR=/tmp/imap-data imap_cli --instrument codice --level 1
+
 Tools
 -----
 

imap_processing/__init__.py

@@ -13,12 +13,25 @@
 # packet definitions directory path.
 #
 # This directory is used by the imap_processing package to find the packet definitions.
+import os
 from pathlib import Path
 
+# NOTE: Use a config dictionary, so it is a mutable global object,
+#       otherwise updating previous imports from other modules
+#       wouldn't have been updated globally (for example if referencing a string).
+config = {"DATA_DIR": Path(os.getenv("IMAP_DATA_DIR") or Path.cwd() / "imap-data")}
+"""imap_processing configuration dictionary.
+
+DATA_DIR : This is where the file data is stored and organized by instrument and level.
+    The default location is in the current working directory, but can be
+    set on the command line using the --data-dir option, or through
+    the environment variable IMAP_DATA_DIR.
+"""
+
 # Eg. imap_module_directory = /usr/local/lib/python3.11/site-packages/imap_processing
 imap_module_directory = Path(__file__).parent
 
-instruments = [
+INSTRUMENTS = [
     "codice",
     "glows",
     "hi",
@@ -31,7 +44,7 @@
     "ultra",
 ]
 
-processing_levels = {
+PROCESSING_LEVELS = {
     "codice": ["l0", "l1a", "l1b", "l2"],
     "glows": ["l0", "l1a", "l1b", "l2"],
     "hi": ["l0", "l1a", "l1b", "l1c", "l2"],

imap_processing/cdf/utils.py

@@ -1,11 +1,15 @@
 """Various utility functions to support creation of CDF files."""
 
-import os
+import logging
+from pathlib import Path
+from typing import Optional
 
 import numpy as np
 import xarray as xr
 from cdflib.xarray import xarray_to_cdf
 
+import imap_processing
+
 
 def calc_start_time(shcoarse_time: int):
     """Calculate the datetime64 from the CCSDS secondary header information.
@@ -36,7 +40,9 @@ def calc_start_time(shcoarse_time: int):
 
 
 def write_cdf(
-    data: xr.Dataset, description: str = "", mode: str = "", directory: str = ""
+    data: xr.Dataset,
+    description: str = "",
+    directory: Optional[Path] = None,
 ):
     """Write the contents of "data" to a CDF file using cdflib.xarray_to_cdf.
 
@@ -49,16 +55,19 @@ def write_cdf(
 
     Parameters
     ----------
-        data (xarray.Dataset): The dataset object to convert to a CDF
-        description (str): The description to insert into the file name after the
+        data : xarray.Dataset
+            The dataset object to convert to a CDF
+        description : str, optional
+            The description to insert into the file name after the
                             orbit, before the SPICE field.  No underscores allowed.
-        mode (str): Instrument mode
-        directory (str): The directory to write the file to
+        directory : pathlib.Path
+            The directory to write the file to. The default is obtained
+            from the global imap_processing.config["DATA_DIR"].
 
     Returns
     -------
-        str
-            The name of the file created
+        pathlib.Path
+            Path to the file created
     """
     # Determine the start date of the data in the file,
     # based on the time of the first dust impact
@@ -81,24 +90,37 @@ def write_cdf(
         if (description.startswith("_") or not description)
         else f"_{description}"
     )
-    mode = mode if (mode.startswith("_") or not mode) else f"_{mode}"
 
     # Determine the file name based on the attributes in the xarray
     # Set file name based on this convention:
-    # imap_<instrument>_<datalevel>_<mode>_<descriptor>_<startdate>_
+    # imap_<instrument>_<datalevel>_<descriptor>_<startdate>_
     # <version>.cdf
     # data.attrs["Logical_source"] has the mission, instrument, and level
     # like this:
     #   imap_idex_l1
     filename = (
         data.attrs["Logical_source"]
-        + mode
         + description
         + "_"
         + date_string
         + f"_v{data.attrs['Data_version']}.cdf"
     )
-    filename_and_path = os.path.join(directory, filename)
+
+    if directory is None:
+        # Storage directory
+        # mission/instrument/data_level/year/month/filename
+        # /<directory | DATA_DIR>/<instrument>/<data_level>/<year>/<month>
+        _, instrument, data_level = data.attrs["Logical_source"].split("_")
+        directory = imap_processing.config["DATA_DIR"] / instrument / data_level
+        directory /= date_string[:4]
+        directory /= date_string[4:6]
+    filename_and_path = Path(directory)
+    if not filename_and_path.exists():
+        logging.info(
+            "The directory does not exist, creating directory %s", filename_and_path
+        )
+        filename_and_path.mkdir(parents=True)
+    filename_and_path /= filename
 
     # Insert the final attribute:
     # The Logical_file_id is always the name of the file without the extension

imap_processing/cli.py

@@ -13,8 +13,9 @@
 import argparse
 import sys
 from abc import ABC, abstractmethod
+from pathlib import Path
 
-from imap_processing import instruments, processing_levels
+import imap_processing
 
 
 def _parse_args():
@@ -28,15 +29,25 @@ def _parse_args():
     description = (
         "This command line program invokes the processing pipeline "
         "for a specific instrument and data level. Example usage: "
-        '"python run_processing swe l1a".'
+        '"python run_processing --instrument swe --level l1a".'
+    )
+    data_dir_help = (
+        "Directory to use for reading and writing IMAP data. "
+        "The default is an 'imap-data/' folder in the "
+        "current working directory. This can also be "
+        "set using the IMAP_DATA_DIR environment variable."
+    )
+    instrument_help = (
+        "The instrument to process. Acceptable values are: "
+        f"{imap_processing.INSTRUMENTS}"
     )
-
-    instrument_help = f"The instrument to process. Acceptable values are: {instruments}"
     level_help = (
-        f"The data level to process. Acceptable values are: {processing_levels}"
+        "The data level to process. Acceptable values are: "
+        f"{imap_processing.PROCESSING_LEVELS}"
     )
 
     parser = argparse.ArgumentParser(prog="imap_cli", description=description)
+    parser.add_argument("--data-dir", type=str, required=False, help=data_dir_help)
     parser.add_argument("--instrument", type=str, required=True, help=instrument_help)
     parser.add_argument("--level", type=str, required=True, help=level_help)
     args = parser.parse_args()
@@ -52,15 +63,23 @@ def _validate_args(args):
     args : argparse.Namespace
         An object containing the parsed arguments and their values
     """
-    if args.instrument not in instruments:
+    if args.instrument not in imap_processing.INSTRUMENTS:
         raise ValueError(
-            f"{args.instrument} is not in the supported instrument list: {instruments}"
+            f"{args.instrument} is not in the supported instrument list: "
+            f"{imap_processing.INSTRUMENTS}"
         )
-    if args.level not in processing_levels[args.instrument]:
+    if args.level not in imap_processing.PROCESSING_LEVELS[args.instrument]:
         raise ValueError(
             f"{args.level} is not a supported data level for the {args.instrument}"
-            f" instrument, valid levels are: {processing_levels[args.instrument]}"
+            " instrument, valid levels are: "
+            f"{imap_processing.PROCESSING_LEVELS[args.instrument]}"
         )
+    if args.data_dir:
+        data_path = Path(args.data_dir)
+        if not data_path.exists():
+            raise ValueError(f"Data directory {args.data_dir} does not exist")
+        # Set the data directory to the user-supplied value
+        imap_processing.config["DATA_DIR"] = data_path
 
 
 class ProcessInstrument(ABC):
@@ -162,7 +181,13 @@ def process(self):
 
 
 def main():
-    """Create CLI entrypoint."""
+    """Run the processing for a specific instrument & data level.
+
+    Set up the command line arguments, parse them, and then invoke the
+    appropriate instrument processing function.
+    """
+    # NOTE: This is to allow the cli script to be installed and reference
+    #       this function for an entrypoint.
     args = _parse_args()
 
     _validate_args(args)

imap_processing/codice/codice_l1a.py

@@ -21,17 +21,13 @@
 from imap_processing.utils import group_by_apid, sort_by_time
 
 
-def codice_l1a(
-    packets: list[space_packet_parser.parser.Packet], cdf_directory: str
-) -> str:
+def codice_l1a(packets: list[space_packet_parser.parser.Packet]) -> str:
     """Process CoDICE l0 data to create l1a data products.
 
     Parameters
     ----------
     packets : list[space_packet_parser.parser.Packet]
         Decom data list that contains all APIDs
-    cdf_directory : str
-        The directory in which to write the output CDF file.
 
     Returns
     -------
@@ -51,9 +47,7 @@ def codice_l1a(
     # Write data to CDF
     cdf_filename = write_cdf(
         data,
-        mode="",
         description="hk",
-        directory=cdf_directory,
     )
 
     return cdf_filename

imap_processing/swe/l1a/swe_l1a.py

@@ -12,7 +12,7 @@
 from imap_processing.utils import group_by_apid, sort_by_time
 
 
-def swe_l1a(packets, cdf_filepath):
+def swe_l1a(packets):
     """Process SWE l0 data into l1a data.
 
     Receive all L0 data file. Based on appId, it
@@ -23,13 +23,11 @@ def swe_l1a(packets, cdf_filepath):
     ----------
     packets: list
         Decom data list that contains all appIds
-    cdf_filepath: str
-        Folder path of where to write CDF file
 
     Returns
     -------
-    str
-        Path name of where CDF file was created.
+    pathlib.Path
+        Path to where the CDF file was created.
         This is used to upload file from local to s3.
         TODO: test this later.
     """
@@ -51,9 +49,8 @@ def swe_l1a(packets, cdf_filepath):
             data = create_dataset(packets=sorted_packets)
 
         # write data to CDF
+        mode = f"{data['APP_MODE'].data[0]}-" if apid == SWEAPID.SWE_APP_HK else ""
         return write_cdf(
             data,
-            mode=f"{data['APP_MODE'].data[0]}" if apid == SWEAPID.SWE_APP_HK else "",
-            description=filename_descriptors.get(apid),
-            directory=cdf_filepath,
+            description=f"{mode}{filename_descriptors.get(apid)}",
         )

imap_processing/swe/l1b/swe_l1b.py

@@ -10,19 +10,17 @@
 from imap_processing.utils import convert_raw_to_eu
 
 
-def swe_l1b(l1a_dataset: xr.Dataset, cdf_filepath: str):
+def swe_l1b(l1a_dataset: xr.Dataset):
     """Process data to L1B.
 
     Parameters
     ----------
     l1a_dataset : xarray.Dataset
         l1a data input
-    cdf_filepath: str
-        Folder path of where to write CDF file
 
     Returns
     -------
-    str
+    pathlib.Path
         Path to the L1B file.
 
     Raises
@@ -50,10 +48,8 @@ def swe_l1b(l1a_dataset: xr.Dataset, cdf_filepath: str):
         data = eu_data
         # Update global attributes to l1b global attributes
         data.attrs.update(swe_cdf_attrs.swe_l1b_global_attrs.output())
-
+    mode = f"{data['APP_MODE'].data[0]}-" if apid == SWEAPID.SWE_APP_HK else ""
     return write_cdf(
         data,
-        mode=f"{data['APP_MODE'].data[0]}" if apid == SWEAPID.SWE_APP_HK else "",
-        description=filename_descriptors.get(apid),
-        directory=cdf_filepath,
+        description=f"{mode}{filename_descriptors.get(apid)}",
     )