From 757eccf5e753da08ee2d9ac3e312a308c6a304f4 Mon Sep 17 00:00:00 2001 From: Greg Lucas Date: Thu, 27 Jun 2024 11:11:55 -0600 Subject: [PATCH] ENH/MNT: Allow array-like time conversion (#653) * ENH/MNT: Allow array-like time conversion We can convert entire arrays at once to datetime64 from integer MET rather than having to loop through in each individual calling location. This is faster and easier to read. * ENH/MNT: Change from datetime formats to integer epochs We have decided to move to epoch being defined as integer nanoseconds since the J2000 EPOCH. This means that the epoch variable will be of datatype int64 and the new conversion utility will convert MET in seconds to j2000ns for users with a default reference epoch of 2010-01-01 per APL's documentation. --- .github/workflows/test.yml | 2 +- imap_processing/__init__.py | 5 -- .../cdf/config/imap_hi_variable_attrs.yaml | 2 +- imap_processing/cdf/utils.py | 70 ++++++++++--------- imap_processing/cli.py | 4 +- imap_processing/codice/codice_l1a.py | 32 ++++----- imap_processing/codice/utils.py | 12 ++-- imap_processing/glows/l1a/glows_l1a.py | 10 +-- imap_processing/hi/l1a/histogram.py | 6 +- .../hi/l1a/science_direct_event.py | 24 +------ imap_processing/hit/l1a/hit_l1a.py | 4 +- imap_processing/hit/l1b/hit_l1b.py | 2 +- imap_processing/idex/idex_packet_parser.py | 9 ++- imap_processing/lo/l1a/lo_l1a.py | 4 +- imap_processing/lo/l1a/lo_l1a_write_cdfs.py | 7 +- imap_processing/lo/l1b/lo_l1b.py | 4 +- imap_processing/lo/l1c/lo_l1c.py | 4 +- imap_processing/mag/l0/decom_mag.py | 4 +- imap_processing/mag/l1a/mag_l1a.py | 18 +++-- imap_processing/mag/l1a/mag_l1a_data.py | 8 +-- imap_processing/swapi/l1/swapi_l1.py | 4 +- imap_processing/swe/l1a/swe_science.py | 6 +- imap_processing/tests/cdf/test_utils.py | 49 ++++++++----- imap_processing/tests/mag/test_mag_l1a.py | 6 +- imap_processing/tests/swapi/test_swapi_l1.py | 8 +-- .../tests/ultra/unit/test_ultra_l1a.py | 22 ++++-- imap_processing/ultra/l1a/ultra_l1a.py | 21 +++--- imap_processing/utils.py | 4 +- 28 files changed, 168 insertions(+), 183 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c0d602796..1aee0dff1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-2019, ubuntu-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] python-version: ['3.9', '3.10', '3.11', '3.12'] defaults: run: diff --git a/imap_processing/__init__.py b/imap_processing/__init__.py index 49d0a3742..839a1c649 100644 --- a/imap_processing/__init__.py +++ b/imap_processing/__init__.py @@ -14,8 +14,6 @@ # This directory is used by the imap_processing package to find the packet definitions. from pathlib import Path -import numpy as np - from imap_processing._version import __version__, __version_tuple__ # noqa: F401 # Eg. imap_module_directory = /usr/local/lib/python3.11/site-packages/imap_processing @@ -34,6 +32,3 @@ "swe": ["l0", "l1a", "l1b", "l2"], "ultra": ["l0", "l1a", "l1b", "l1c", "l2"], } - -# Reference start time (launch time or epoch) -launch_time = np.datetime64("2010-01-01T00:01:06.184", "ns") diff --git a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml index 4d091176e..e6aace7a5 100644 --- a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml @@ -70,7 +70,7 @@ default_epoch: &default_epoch TIME_BASE: J2000 TIME_SCALE: Terrestrial Time REFERENCE_POSITION: Rotating Earth Geoid - dtype: datetime64[ns] + dtype: int64 # ------- L1A DE Section ------- hi_de_ccsds_met: diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index c0a2e434f..517cccce7 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -16,39 +16,49 @@ logger = logging.getLogger(__name__) -def calc_start_time( - shcoarse_time: float, - launch_time: Optional[np.datetime64] = imap_processing.launch_time, -) -> np.datetime64: - """ - Calculate the datetime64 from the CCSDS secondary header information. +# Reference start time (launch time or epoch) +# DEFAULT_EPOCH = np.datetime64("2010-01-01T00:01:06.184", "ns") +IMAP_EPOCH = np.datetime64("2010-01-01T00:00:00", "ns") +J2000_EPOCH = np.datetime64("2000-01-01T11:58:55.816", "ns") + - Since all instrument has SHCOARSE or MET seconds, we need convert it to - UTC. Took this from IDEX code. +def met_to_j2000ns( + met: np.typing.ArrayLike, + reference_epoch: Optional[np.datetime64] = IMAP_EPOCH, +) -> np.typing.ArrayLike: + """ + Convert mission elapsed time (MET) to nanoseconds from J2000. Parameters ---------- - shcoarse_time : float - Number of seconds since epoch (nominally the launch time). - launch_time : np.datetime64 - The time of launch to use as the baseline. + met : array_like + Number of seconds since epoch according to the spacecraft clock. + reference_epoch : np.datetime64 + The time of reference for the mission elapsed time. The standard + reference time for IMAP is January 1, 2010 00:00:00 UTC. Per APL's + IMAP Timekeeping System Design document. Returns ------- - np.timedelta64 - The time of the event. + array_like or scalar, int64 + The mission elapsed time converted to nanoseconds since the J2000 epoch. Notes ----- - TODO - move this into imap-data-access? How should it be used? - This conversion is temporary for now, and will need SPICE in the future. - Nick Dutton mentioned that s/c clock start epoch is - jan-1-2010-00:01:06.184 ET - We will use this for now. + This conversion is temporary for now, and will need SPICE in the future to + account for spacecraft clock drift. """ - # Get the datetime of Jan 1 2010 as the start date - time_delta = np.timedelta64(int(shcoarse_time * 1e9), "ns") - return launch_time + time_delta + # Mission elapsed time is in seconds, convert to nanoseconds + # NOTE: We need to multiply the incoming met by 1e9 first because we could have + # float input and we want to keep ns precision in those floats + # NOTE: We need int64 here when running on 32bit systems as plain int will default + # to 32bit and overflow due to the nanosecond multiplication + time_array = (np.asarray(met, dtype=float) * 1e9).astype(np.int64) + # Calculate the time difference between our reference system and J2000 + j2000_offset = ( + (reference_epoch - J2000_EPOCH).astype("timedelta64[ns]").astype(np.int64) + ) + return j2000_offset + time_array def load_cdf( @@ -64,19 +74,14 @@ def load_cdf( remove_xarray_attrs : bool Whether to remove the xarray attributes that get injected by the cdf_to_xarray function from the output xarray.Dataset. Default is True. - **kwargs : {dict} optional - Keyword arguments for ``cdf_to_xarray``. This function overrides the - ``cdf_to_xarray`` default keyword value `to_datetime=False` with - ``to_datetime=True`. + **kwargs : dict, optional + Keyword arguments for ``cdf_to_xarray``. Returns ------- dataset : xarray.Dataset The ``xarray`` dataset for the CDF file. """ - # TODO: remove this when cdflib is updated to version >1.3.0 - if "to_datetime" not in kwargs: - kwargs["to_datetime"] = True dataset = cdf_to_xarray(file_path, kwargs) # cdf_to_xarray converts single-value attributes to lists @@ -121,9 +126,9 @@ def write_cdf(dataset: xr.Dataset): # Create the filename from the global attributes # Logical_source looks like "imap_swe_l2_counts-1min" instrument, data_level, descriptor = dataset.attrs["Logical_source"].split("_")[1:] - start_time = np.datetime_as_string(dataset["epoch"].values[0], unit="D").replace( - "-", "" - ) + # Convert J2000 epoch referenced data to datetime64 + dt64 = J2000_EPOCH + dataset["epoch"].values[0].astype("timedelta64[ns]") + start_time = np.datetime_as_string(dt64, unit="D").replace("-", "") # Will now accept vXXX or XXX formats, as batch starter sends versions as vXXX. r = re.compile(r"v\d{3}") @@ -159,7 +164,6 @@ def write_cdf(dataset: xr.Dataset): xarray_to_cdf( dataset, str(file_path), - datetime64_to_cdftt2000=True, terminate_on_warning=True, ) # Terminate if not ISTP compliant diff --git a/imap_processing/cli.py b/imap_processing/cli.py index c1c6f4708..2eb30f031 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -618,7 +618,7 @@ def do_processing(self, dependencies): elif self.data_level == "l1b": data_dict = {} for dependency in dependencies: - dataset = load_cdf(dependency, to_datetime=True) + dataset = load_cdf(dependency) data_dict[dataset.attrs["Logical_source"]] = dataset dataset = lo_l1b.lo_l1b(data_dict, self.version) return [dataset] @@ -626,7 +626,7 @@ def do_processing(self, dependencies): elif self.data_level == "l1c": data_dict = {} for dependency in dependencies: - dataset = load_cdf(dependency, to_datetime=True) + dataset = load_cdf(dependency) data_dict[dataset.attrs["Logical_source"]] = dataset dataset = lo_l1c.lo_l1c(data_dict, self.version) return [dataset] diff --git a/imap_processing/codice/codice_l1a.py b/imap_processing/codice/codice_l1a.py index 4122fa330..f106c52ee 100644 --- a/imap_processing/codice/codice_l1a.py +++ b/imap_processing/codice/codice_l1a.py @@ -22,7 +22,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import IMAP_EPOCH, met_to_j2000ns from imap_processing.codice import constants from imap_processing.codice.codice_l0 import decom_packets from imap_processing.codice.utils import CODICEAPID, create_hskp_dataset @@ -78,9 +78,7 @@ def __init__(self, table_id: int, plan_id: int, plan_step: int, view_id: int): self.plan_step = plan_step self.view_id = view_id - def create_science_dataset( - self, start_time: np.datetime64, data_version: str - ) -> xr.Dataset: + def create_science_dataset(self, met: np.int64, data_version: str) -> xr.Dataset: """ Create an ``xarray`` dataset for the unpacked science data. @@ -88,8 +86,8 @@ def create_science_dataset( Parameters ---------- - start_time : numpy.datetime64 - The start time of the packet, used to determine epoch data variable. + met : numpy.int64 + The mission elapsed time of the packet, used to determine epoch data. data_version : str Version of the data product being created. @@ -106,10 +104,7 @@ def create_science_dataset( # Define coordinates epoch = xr.DataArray( - [ - start_time, - start_time + np.timedelta64(1, "s"), - ], # TODO: Fix after SIT-3 (see note below) + met_to_j2000ns(met), # TODO: Fix after SIT-3 (see note below) name="epoch", dims=["epoch"], attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), @@ -380,11 +375,8 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: packets = sort_by_time(grouped_data[apid], "SHCOARSE") # Determine the start time of the packet - start_time = calc_start_time( - packets[0].data["ACQ_START_SECONDS"].raw_value, - launch_time=np.datetime64("2010-01-01T00:01:06.184", "ns"), - ) - + met = packets[0].data["ACQ_START_SECONDS"].raw_value + met = [met, met + 1] # TODO: Remove after SIT-3 # Extract the data science_values = packets[0].data["DATA"].raw_value @@ -397,7 +389,7 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: pipeline.get_acquisition_times() pipeline.get_data_products(apid) pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(start_time, data_version) + dataset = pipeline.create_science_dataset(met, data_version) # TODO: Temporary workaround in order to create hi data products in absence # of simulated data. This is essentially the same process as is for @@ -417,15 +409,15 @@ def process_codice_l1a(file_path: Path | str, data_version: str) -> xr.Dataset: apid = CODICEAPID.COD_HI_SECT_SPECIES_COUNTS table_id, plan_id, plan_step, view_id = (1, 0, 0, 6) - start_time = np.datetime64( - "2024-04-29T00:00:00", "ns" - ) # Using this to match the other data products + met0 = (np.datetime64("2024-04-29T00:00") - IMAP_EPOCH).astype("timedelta64[s]") + met0 = met0.astype(np.int64) + met = [met0, met0 + 1] # Using this to match the other data products science_values = "" # Currently don't have simulated data for this pipeline = CoDICEL1aPipeline(table_id, plan_id, plan_step, view_id) pipeline.get_data_products(apid) pipeline.unpack_science_data(science_values) - dataset = pipeline.create_science_dataset(start_time, data_version) + dataset = pipeline.create_science_dataset(met, data_version) # Write dataset to CDF logger.info(f"\nFinal data product:\n{dataset}\n") diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index ca9890402..44a8c6433 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -13,7 +13,7 @@ from imap_processing.cdf.global_attrs import ConstantCoordinates from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns class CODICEAPID(IntEnum): @@ -129,12 +129,10 @@ def create_hskp_dataset(packets, data_version: str) -> xr.Dataset: # TODO: Is there a way to get the attrs from the YAML-based method? epoch = xr.DataArray( - [ - calc_start_time( - item, launch_time=np.datetime64("2010-01-01T00:01:06.184", "ns") - ) - for item in metadata_arrays["SHCOARSE"] - ], + met_to_j2000ns( + metadata_arrays["SHCOARSE"], + reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns"), + ), name="epoch", dims=["epoch"], attrs=ConstantCoordinates.EPOCH, diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index e9a6d7413..508ccd007 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -8,7 +8,7 @@ import xarray as xr from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import J2000_EPOCH, met_to_j2000ns from imap_processing.glows import __version__, glows_cdf_attrs from imap_processing.glows.l0.decom_glows import decom_packets from imap_processing.glows.l0.glows_l0_data import DirectEventL0 @@ -46,7 +46,7 @@ def glows_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]: hist_l1a = HistogramL1A(hist) # Split by IMAP start time # TODO: Should this be MET? - hist_day = calc_start_time(hist.SEC).astype("datetime64[D]") + hist_day = (J2000_EPOCH + met_to_j2000ns(hist.SEC)).astype("datetime64[D]") hists_by_day[hist_day].append(hist_l1a) # Generate CDF files for each day @@ -85,7 +85,7 @@ def process_de_l0( de_by_day = dict() for de in de_l0: - de_day = calc_start_time(de.MET).astype("datetime64[D]") + de_day = (J2000_EPOCH + met_to_j2000ns(de.MET)).astype("datetime64[D]") if de_day not in de_by_day: de_by_day[de_day] = [DirectEventL1A(de)] elif de.SEQ != 0: @@ -163,7 +163,7 @@ def generate_de_dataset( for index, de in enumerate(de_l1a_list): # Set the timestamp to the first timestamp of the direct event list - epoch_time = calc_start_time(de.l0.MET).astype("datetime64[ns]") + epoch_time = met_to_j2000ns(de.l0.MET).astype("datetime64[ns]") # determine if the length of the direct_events numpy array is long enough, # and extend the direct_events length dimension if necessary. @@ -332,7 +332,7 @@ def generate_histogram_dataset( for index, hist in enumerate(hist_l1a_list): # TODO: Should this be MET? - epoch_time = calc_start_time(hist.imap_start_time.to_seconds()) + epoch_time = met_to_j2000ns(hist.imap_start_time.to_seconds()) hist_data[index] = hist.histograms support_data["flags_set_onboard"].append(hist.flags["flags_set_onboard"]) diff --git a/imap_processing/hi/l1a/histogram.py b/imap_processing/hi/l1a/histogram.py index 36e836da4..e33fa09e0 100644 --- a/imap_processing/hi/l1a/histogram.py +++ b/imap_processing/hi/l1a/histogram.py @@ -7,7 +7,7 @@ from space_packet_parser.parser import Packet from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.hi import hi_cdf_attrs # TODO: Verify that these names are OK for counter variables in the CDF @@ -57,9 +57,7 @@ def create_dataset(packets: list[Packet]) -> xr.Dataset: # unpack the packets data into the Dataset for i_epoch, packet in enumerate(packets): - dataset.epoch.data[i_epoch] = calc_start_time( - packet.data["CCSDS_MET"].raw_value - ) + dataset.epoch.data[i_epoch] = met_to_j2000ns(packet.data["CCSDS_MET"].raw_value) dataset.ccsds_met[i_epoch] = packet.data["CCSDS_MET"].raw_value dataset.esa_step[i_epoch] = packet.data["ESA_STEP"].raw_value diff --git a/imap_processing/hi/l1a/science_direct_event.py b/imap_processing/hi/l1a/science_direct_event.py index 19a987b57..0c86a421f 100644 --- a/imap_processing/hi/l1a/science_direct_event.py +++ b/imap_processing/hi/l1a/science_direct_event.py @@ -4,9 +4,10 @@ import xarray as xr from space_packet_parser.parser import Packet -from imap_processing import imap_module_directory, launch_time +from imap_processing import imap_module_directory from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.global_attrs import ConstantCoordinates +from imap_processing.cdf.utils import met_to_j2000ns # TODO: read LOOKED_UP_DURATION_OF_TICK from # instrument status summary later. This value @@ -20,25 +21,6 @@ MICROSECOND_TO_NS = 1e3 -def get_direct_event_time(time_in_ns: int) -> np.datetime64: - """ - Create MET(Mission Elapsed Time) time using input times. - - Parameters - ---------- - time_in_ns : int - Time in nanoseconds. - - Returns - ------- - met_datetime : numpy.datetime64 - Human-readable MET time. - """ - met_datetime = launch_time + np.timedelta64(int(time_in_ns), "ns") - - return met_datetime - - def parse_direct_event(event_data: str) -> dict: """ Parse event data. @@ -267,7 +249,7 @@ def create_dataset(de_data_list: list, packet_met_time: list) -> xr.Dataset: + event["de_tag"] * LOOKED_UP_DURATION_OF_TICK * MICROSECOND_TO_NS ) data_dict["event_met"].append(de_met_in_ns) - data_dict["epoch"].append(get_direct_event_time(de_met_in_ns)) + data_dict["epoch"].append(met_to_j2000ns(de_met_in_ns / 1e9)) data_dict["esa_stepping_num"].append(current_esa_step) # start_bitmask_data is 1, 2, 3 for detector A, B, C # respectively. This is used to identify which detector diff --git a/imap_processing/hit/l1a/hit_l1a.py b/imap_processing/hit/l1a/hit_l1a.py index 485594dfe..cb20c1be7 100644 --- a/imap_processing/hit/l1a/hit_l1a.py +++ b/imap_processing/hit/l1a/hit_l1a.py @@ -166,9 +166,7 @@ def create_datasets(data: dict, skip_keys=None): metadata_arrays[data_key].append(field_value) # Convert integers into datetime64[s] - epoch_converted_times = [ - utils.calc_start_time(time) for time in metadata_arrays["shcoarse"] - ] + epoch_converted_times = utils.met_to_j2000ns(metadata_arrays["shcoarse"]) # Create xarray data arrays for dependencies epoch_time = xr.DataArray( diff --git a/imap_processing/hit/l1b/hit_l1b.py b/imap_processing/hit/l1b/hit_l1b.py index dff772550..ad50d31cd 100644 --- a/imap_processing/hit/l1b/hit_l1b.py +++ b/imap_processing/hit/l1b/hit_l1b.py @@ -79,7 +79,7 @@ def create_hk_dataset(): # Create fake data for now # Convert integers into datetime64[s] - epoch_converted_time = [utils.calc_start_time(time) for time in [0, 1, 2]] + epoch_converted_time = utils.met_to_j2000ns([0, 1, 2]) # Shape for dims n_epoch = 3 diff --git a/imap_processing/idex/idex_packet_parser.py b/imap_processing/idex/idex_packet_parser.py index 5312bb48e..fb521517e 100644 --- a/imap_processing/idex/idex_packet_parser.py +++ b/imap_processing/idex/idex_packet_parser.py @@ -16,6 +16,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.global_attrs import ConstantCoordinates +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.idex import idex_cdf_attrs logger = logging.getLogger(__name__) @@ -648,12 +649,10 @@ def _set_impact_time(self, packet): # Number of microseconds since the last second microseconds_since_last_second = 20 * num_of_20_microsecond_increments # Get the datetime of Jan 1 2012 as the start date - launch_time = np.datetime64("2012-01-01T00:00:00.000000000") + met = seconds_since_launch + microseconds_since_last_second * 1e-6 - self.impact_time = ( - launch_time - + np.timedelta64(seconds_since_launch, "s") - + np.timedelta64(microseconds_since_last_second, "us") + self.impact_time = met_to_j2000ns( + met, reference_epoch=np.datetime64("2012-01-01T00:00:00.000000000") ) def _set_sample_trigger_times(self, packet): diff --git a/imap_processing/lo/l1a/lo_l1a.py b/imap_processing/lo/l1a/lo_l1a.py index 2d7b43e46..18db7274e 100644 --- a/imap_processing/lo/l1a/lo_l1a.py +++ b/imap_processing/lo/l1a/lo_l1a.py @@ -7,7 +7,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.lo.l0.data_classes.science_direct_events import ScienceDirectEvents @@ -95,7 +95,7 @@ def create_datasets(attr_mgr, logical_source, data_fields): """ # Convert each packet's spacecraft time to an absolute epoch time # TODO: replace temp hardcoded values with packet values - epoch_converted_time = [calc_start_time(time) for time in [0, 1, 2]] + epoch_converted_time = met_to_j2000ns([0, 1, 2]) # Create a data array for the poch time # TODO: might need to update the attrs to use new YAML file diff --git a/imap_processing/lo/l1a/lo_l1a_write_cdfs.py b/imap_processing/lo/l1a/lo_l1a_write_cdfs.py index 2f7e3625f..20bcff0cd 100644 --- a/imap_processing/lo/l1a/lo_l1a_write_cdfs.py +++ b/imap_processing/lo/l1a/lo_l1a_write_cdfs.py @@ -4,6 +4,7 @@ import xarray as xr from imap_processing.cdf.global_attrs import ConstantCoordinates +from imap_processing.cdf.utils import J2000_EPOCH from imap_processing.lo.l0.lo_apid import LoAPID from imap_processing.lo.l1a import lo_cdf_attrs from imap_processing.lo.l1a.lo_data_container import LoContainer @@ -58,8 +59,12 @@ def create_lo_scide_dataset(sci_de: list): sci_de_time = xr.DataArray( sci_de_times, dims="epoch", attrs=lo_cdf_attrs.lo_tof_attrs.output() ) + epoch_times = ( + np.array(sci_de_times, dtype="datetime64[s]").astype("datetime64[ns]") + - J2000_EPOCH + ).astype(np.int64) sci_de_epoch = xr.DataArray( - np.array(sci_de_times, dtype="datetime64[s]").astype("datetime64[ns]"), + epoch_times, dims=["epoch"], name="epoch", attrs=ConstantCoordinates.EPOCH, diff --git a/imap_processing/lo/l1b/lo_l1b.py b/imap_processing/lo/l1b/lo_l1b.py index 55c700efa..d260b46fe 100644 --- a/imap_processing/lo/l1b/lo_l1b.py +++ b/imap_processing/lo/l1b/lo_l1b.py @@ -6,7 +6,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns def lo_l1b(dependencies: dict, data_version: str): @@ -80,7 +80,7 @@ def create_datasets(attr_mgr, logical_source, data_fields): # and relative L1A DE time to calculate the absolute DE time, # this epoch conversion will go away and the time in the DE dataclass # can be used direction - epoch_converted_time = [calc_start_time(time) for time in [0, 1, 2]] + epoch_converted_time = met_to_j2000ns([0, 1, 2]) # Create a data array for the epoch time # TODO: might need to update the attrs to use new YAML file diff --git a/imap_processing/lo/l1c/lo_l1c.py b/imap_processing/lo/l1c/lo_l1c.py index 0646ff747..bd415ccc1 100644 --- a/imap_processing/lo/l1c/lo_l1c.py +++ b/imap_processing/lo/l1c/lo_l1c.py @@ -6,7 +6,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns def lo_l1c(dependencies: dict, data_version: str): @@ -81,7 +81,7 @@ def create_datasets(attr_mgr, logical_source, data_fields): # and relative L1A DE time to calculate the absolute DE time, # this epoch conversion will go away and the time in the DE dataclass # can be used direction - epoch_converted_time = [calc_start_time(1)] + epoch_converted_time = [met_to_j2000ns(1)] # Create a data array for the epoch time # TODO: might need to update the attrs to use new YAML file diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py index 721d7fd9c..4e0e726ce 100644 --- a/imap_processing/mag/l0/decom_mag.py +++ b/imap_processing/mag/l0/decom_mag.py @@ -14,7 +14,7 @@ from imap_processing import imap_module_directory from imap_processing.ccsds.ccsds_data import CcsdsData from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.mag import mag_cdf_attrs from imap_processing.mag.l0.mag_l0_data import MagL0, Mode @@ -110,7 +110,7 @@ def generate_dataset(l0_data: list[MagL0], dataset_attrs: dict) -> xr.Dataset: ) vector_data[index, :vector_len] = datapoint.VECTORS - shcoarse_data[index] = calc_start_time(datapoint.SHCOARSE) + shcoarse_data[index] = met_to_j2000ns(datapoint.SHCOARSE) # Add remaining pieces to arrays for key, value in dataclasses.asdict(datapoint).items(): if key not in ("ccsds_header", "VECTORS", "SHCOARSE"): diff --git a/imap_processing/mag/l1a/mag_l1a.py b/imap_processing/mag/l1a/mag_l1a.py index 304e27c91..81d444e2f 100644 --- a/imap_processing/mag/l1a/mag_l1a.py +++ b/imap_processing/mag/l1a/mag_l1a.py @@ -7,7 +7,7 @@ import xarray as xr from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import J2000_EPOCH, met_to_j2000ns from imap_processing.mag import mag_cdf_attrs from imap_processing.mag.l0 import decom_mag from imap_processing.mag.l0.mag_l0_data import MagL0 @@ -139,12 +139,16 @@ def process_packets( primary_start_time = TimeTuple(mag_l0.PRI_COARSETM, mag_l0.PRI_FNTM) secondary_start_time = TimeTuple(mag_l0.SEC_COARSETM, mag_l0.SEC_FNTM) - primary_day = calc_start_time(primary_start_time.to_seconds()).astype( - "datetime64[D]" - ) - secondary_day = calc_start_time(secondary_start_time.to_seconds()).astype( - "datetime64[D]" - ) + primary_day = ( + J2000_EPOCH + + met_to_j2000ns(primary_start_time.to_seconds()).astype("timedelta64[ns]") + ).astype("datetime64[D]") + secondary_day = ( + J2000_EPOCH + + met_to_j2000ns(secondary_start_time.to_seconds()).astype( + "timedelta64[ns]" + ) + ).astype("datetime64[D]") # seconds of data in this packet is the SUBTYPE plus 1 seconds_per_packet = mag_l0.PUS_SSUBTYPE + 1 diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index cb62d5363..13c9f28f4 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -5,7 +5,7 @@ import numpy as np -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns MAX_FINE_TIME = 65535 # maximum 16 bit unsigned int @@ -132,12 +132,12 @@ def calculate_vector_time(vectors, vectors_per_second, start_time) -> np.ndarray ------- vector_objects : numpy.ndarray Vectors with timestamps added in seconds, calculated from - cdf.utils.calc_start_time. - TODO: Move timestamps to J2000. + cdf.utils.met_to_j2000ns. """ + # TODO: Move timestamps to J2000 timedelta = np.timedelta64(int(1 / vectors_per_second * 1e9), "ns") - start_time_ns = calc_start_time(start_time.to_seconds()) + start_time_ns = met_to_j2000ns(start_time.to_seconds()) # Calculate time skips for each vector in ns times = np.reshape( diff --git a/imap_processing/swapi/l1/swapi_l1.py b/imap_processing/swapi/l1/swapi_l1.py index 633fc0a6c..3303d0534 100644 --- a/imap_processing/swapi/l1/swapi_l1.py +++ b/imap_processing/swapi/l1/swapi_l1.py @@ -8,7 +8,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.decom import decom_packets from imap_processing.swapi.swapi_cdf_attrs import ( compression_attrs, @@ -473,7 +473,7 @@ def process_swapi_science(sci_dataset, data_version: str): # epoch time. Should be same dimension as number of good sweeps epoch_time = good_sweep_sci["epoch"].data.reshape(total_full_sweeps, 12)[:, 0] - epoch_converted_time = [calc_start_time(time) for time in epoch_time] + epoch_converted_time = met_to_j2000ns(epoch_time) epoch_time = xr.DataArray( epoch_converted_time, name="epoch", diff --git a/imap_processing/swe/l1a/swe_science.py b/imap_processing/swe/l1a/swe_science.py index 72e4d9ae5..df57b287e 100644 --- a/imap_processing/swe/l1a/swe_science.py +++ b/imap_processing/swe/l1a/swe_science.py @@ -7,7 +7,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.swe.utils.swe_utils import ( add_metadata_to_array, ) @@ -149,9 +149,7 @@ def swe_science(decom_data, data_version): cdf_attrs.add_instrument_variable_attrs("swe", "l1a") cdf_attrs.add_global_attribute("Data_version", data_version) - epoch_converted_time = [ - calc_start_time(sc_time) for sc_time in metadata_arrays["SHCOARSE"] - ] + epoch_converted_time = met_to_j2000ns(metadata_arrays["SHCOARSE"]) epoch_time = xr.DataArray( epoch_converted_time, name="epoch", diff --git a/imap_processing/tests/cdf/test_utils.py b/imap_processing/tests/cdf/test_utils.py index 0a496eaf7..f3f489daa 100644 --- a/imap_processing/tests/cdf/test_utils.py +++ b/imap_processing/tests/cdf/test_utils.py @@ -5,10 +5,15 @@ import pytest import xarray as xr -from imap_processing import launch_time from imap_processing.cdf.global_attrs import ConstantCoordinates from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.cdf.utils import calc_start_time, load_cdf, write_cdf +from imap_processing.cdf.utils import ( + IMAP_EPOCH, + J2000_EPOCH, + load_cdf, + met_to_j2000ns, + write_cdf, +) @pytest.fixture() @@ -29,11 +34,7 @@ def test_dataset(): { "epoch": ( "epoch", - [ - np.datetime64("2010-01-01T00:01:01", "ns"), - np.datetime64("2010-01-01T00:01:02", "ns"), - np.datetime64("2010-01-01T00:01:03", "ns"), - ], + met_to_j2000ns([1, 2, 3]), ) }, attrs=swe_attrs.get_global_attributes("imap_swe_l1a_sci") @@ -47,15 +48,27 @@ def test_dataset(): return dataset -def test_calc_start_time(): - """Tests the ``calc_start_time`` function""" - - assert calc_start_time(0) == launch_time - assert calc_start_time(1) == launch_time + np.timedelta64(1, "s") - different_launch_time = launch_time + np.timedelta64(2, "s") - assert calc_start_time( - 0, launch_time=different_launch_time - ) == launch_time + np.timedelta64(2, "s") +def test_met_to_j2000ns(): + """Tests the ``met_to_j2000ns`` function""" + imap_epoch_offset = (IMAP_EPOCH - J2000_EPOCH).astype(np.int64) + assert met_to_j2000ns(0) == imap_epoch_offset + assert met_to_j2000ns(1) == imap_epoch_offset + 1e9 + # Large input should work (avoid overflow with int32 SHCOARSE inputs) + assert met_to_j2000ns(np.int32(2**30)) == imap_epoch_offset + 2**30 * 1e9 + assert met_to_j2000ns(0).dtype == np.int64 + # Float input should work + assert met_to_j2000ns(0.0) == imap_epoch_offset + assert met_to_j2000ns(1.2) == imap_epoch_offset + 1.2e9 + # Negative input should work + assert met_to_j2000ns(-1) == imap_epoch_offset - 1e9 + # array-like input should work + output = met_to_j2000ns([0, 1]) + np.testing.assert_array_equal(output, [imap_epoch_offset, imap_epoch_offset + 1e9]) + # Different reference epoch should shift the result + different_epoch_time = IMAP_EPOCH + np.timedelta64(2, "ns") + assert ( + met_to_j2000ns(0, reference_epoch=different_epoch_time) == imap_epoch_offset + 2 + ) def test_load_cdf(test_dataset): @@ -68,8 +81,8 @@ def test_load_cdf(test_dataset): dataset = load_cdf(file_path) assert isinstance(dataset, xr.core.dataset.Dataset) - # Test that epoch is converted to datetime64 by default - assert dataset["epoch"].data.dtype == np.dtype("datetime64[ns]") + # Test that epoch is represented as a 64bit integer + assert dataset["epoch"].data.dtype == np.int64 # Test removal of attributes that are added on by cdf_to_xarray and # are specific to xarray plotting xarray_attrs = ["units", "standard_name", "long_name"] diff --git a/imap_processing/tests/mag/test_mag_l1a.py b/imap_processing/tests/mag/test_mag_l1a.py index 85fe2f324..3584bfb02 100644 --- a/imap_processing/tests/mag/test_mag_l1a.py +++ b/imap_processing/tests/mag/test_mag_l1a.py @@ -3,7 +3,7 @@ import numpy as np import pandas as pd -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.mag.l0.decom_mag import decom_packets from imap_processing.mag.l1a.mag_l1a import process_packets from imap_processing.mag.l1a.mag_l1a_data import ( @@ -22,7 +22,7 @@ def test_compare_validation_data(): l1 = process_packets(l0["norm"]) # Should have one day of data expected_day = np.datetime64("2023-11-30") - + print(l1["mago"]) l1_mago = l1["mago"][expected_day] l1_magi = l1["magi"][expected_day] @@ -101,7 +101,7 @@ def test_calculate_vector_time(): test_data = MagL1a.calculate_vector_time(test_vectors, test_vecsec, start_time) - converted_start_time_ns = calc_start_time(start_time.to_seconds()) + converted_start_time_ns = met_to_j2000ns(start_time.to_seconds()) skips_ns = np.timedelta64(int(1 / test_vecsec * 1e9), "ns") expected_data = np.array( diff --git a/imap_processing/tests/swapi/test_swapi_l1.py b/imap_processing/tests/swapi/test_swapi_l1.py index 02b5c7c28..12efc71c3 100644 --- a/imap_processing/tests/swapi/test_swapi_l1.py +++ b/imap_processing/tests/swapi/test_swapi_l1.py @@ -3,7 +3,7 @@ import xarray as xr from imap_processing import imap_module_directory -from imap_processing.cdf.utils import write_cdf +from imap_processing.cdf.utils import met_to_j2000ns, write_cdf from imap_processing.decom import decom_packets from imap_processing.swapi.l1.swapi_l1 import ( SWAPIAPID, @@ -217,11 +217,7 @@ def test_process_swapi_science(decom_test_data): # Test dataset dimensions assert processed_data.sizes == {"epoch": 3, "energy": 72} # Test epoch data is correct - expected_epoch_datetime = [ - np.datetime64("2010-01-01T00:01:54.184000000"), - np.datetime64("2010-01-01T00:02:06.184000000"), - np.datetime64("2010-01-01T00:02:18.184000000"), - ] + expected_epoch_datetime = met_to_j2000ns([48, 60, 72]) np.testing.assert_array_equal(processed_data["epoch"].data, expected_epoch_datetime) expected_count = [ diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1a.py b/imap_processing/tests/ultra/unit/test_ultra_l1a.py index 184f3f457..de50af8cb 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1a.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1a.py @@ -1,9 +1,10 @@ import dataclasses +import numpy as np import pytest from imap_processing import decom -from imap_processing.cdf.utils import load_cdf, write_cdf +from imap_processing.cdf.utils import J2000_EPOCH, load_cdf, write_cdf from imap_processing.ultra import ultra_cdf_attrs from imap_processing.ultra.l0.decom_ultra import process_ultra_apids from imap_processing.ultra.l0.ultra_utils import ( @@ -107,7 +108,10 @@ def test_xarray_rates(decom_test_data): dataset = create_dataset({ULTRA_RATES.apid[0]: decom_ultra_rates}) # Spot check metadata data and attributes - specific_epoch_data = dataset.sel(epoch="2024-02-07T15:28:37.184000")["START_RF"] + j2000_time = ( + np.datetime64("2024-02-07T15:28:37.184000", "ns") - J2000_EPOCH + ).astype(np.int64) + specific_epoch_data = dataset.sel(epoch=j2000_time)["START_RF"] startrf_list = specific_epoch_data.values.tolist() startrf_attr = dataset.variables["START_RF"].attrs @@ -142,9 +146,10 @@ def test_xarray_tof(decom_test_data): dataset = create_dataset({ULTRA_TOF.apid[0]: decom_ultra_tof}) # Spot check metadata data and attributes - specific_epoch_data = dataset.sel(epoch="2024-02-07T15:28:36.184000", sid=0)[ - "PACKETDATA" - ] + j2000_time = ( + np.datetime64("2024-02-07T15:28:36.184000", "ns") - J2000_EPOCH + ).astype(np.int64) + specific_epoch_data = dataset.sel(epoch=j2000_time, sid=0)["PACKETDATA"] packetdata_attr = dataset.variables["PACKETDATA"].attrs expected_packetdata_attr = dataclasses.replace( @@ -189,7 +194,10 @@ def test_xarray_events(decom_test_data, decom_ultra_aux, events_test_path): ) # Spot check metadata data and attributes - specific_epoch_data = dataset.sel(epoch="2024-02-07T15:28:37.184000")["COIN_TYPE"] + j2000_time = ( + np.datetime64("2024-02-07T15:28:37.184000", "ns") - J2000_EPOCH + ).astype(np.int64) + specific_epoch_data = dataset.sel(epoch=j2000_time)["COIN_TYPE"] cointype_list = specific_epoch_data.values.tolist() cointype_attr = dataset.variables["COIN_TYPE"].attrs @@ -200,7 +208,7 @@ def test_xarray_events(decom_test_data, decom_ultra_aux, events_test_path): label_axis="coin_type", ).output() - assert cointype_list == decom_ultra_events["COIN_TYPE"][0:1] + assert cointype_list == decom_ultra_events["COIN_TYPE"][0] assert cointype_attr == expected_cointype_attr diff --git a/imap_processing/ultra/l1a/ultra_l1a.py b/imap_processing/ultra/l1a/ultra_l1a.py index 6cd292fbc..2bb8ab6de 100644 --- a/imap_processing/ultra/l1a/ultra_l1a.py +++ b/imap_processing/ultra/l1a/ultra_l1a.py @@ -13,7 +13,7 @@ from imap_processing import decom, imap_module_directory from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.ultra import ultra_cdf_attrs from imap_processing.ultra.l0.decom_ultra import ( ULTRA_AUX, @@ -43,38 +43,33 @@ def initiate_data_arrays(decom_ultra: dict, apid: int): dataset : xarray.Dataset Data in xarray format. """ - # Converted time - time_converted = [] - if apid in ULTRA_EVENTS.apid: index = ULTRA_EVENTS.apid.index(apid) logical_source = ULTRA_EVENTS.logical_source[index] addition_to_logical_desc = ULTRA_EVENTS.addition_to_logical_desc - for time in decom_ultra["EVENTTIMES"]: - time_converted.append(calc_start_time(time)) + raw_time = decom_ultra["EVENTTIMES"] elif apid in ULTRA_TOF.apid: index = ULTRA_TOF.apid.index(apid) logical_source = ULTRA_TOF.logical_source[index] addition_to_logical_desc = ULTRA_TOF.addition_to_logical_desc - for time in np.unique(decom_ultra["SHCOARSE"]): - time_converted.append(calc_start_time(time)) + raw_time = np.unique(decom_ultra["SHCOARSE"]) elif apid in ULTRA_AUX.apid: index = ULTRA_AUX.apid.index(apid) logical_source = ULTRA_AUX.logical_source[index] addition_to_logical_desc = ULTRA_AUX.addition_to_logical_desc - for time in decom_ultra["SHCOARSE"]: - time_converted.append(calc_start_time(time)) + raw_time = decom_ultra["SHCOARSE"] elif apid in ULTRA_RATES.apid: index = ULTRA_RATES.apid.index(apid) logical_source = ULTRA_RATES.logical_source[index] addition_to_logical_desc = ULTRA_RATES.addition_to_logical_desc - for time in decom_ultra["SHCOARSE"]: - time_converted.append(calc_start_time(time)) + raw_time = decom_ultra["SHCOARSE"] else: raise ValueError(f"APID {apid} not recognized.") epoch_time = xr.DataArray( - time_converted, + met_to_j2000ns( + raw_time, reference_epoch=np.datetime64("2010-01-01T00:01:06.184", "ns") + ), name="epoch", dims=["epoch"], attrs=ConstantCoordinates.EPOCH, diff --git a/imap_processing/utils.py b/imap_processing/utils.py index d08ed2290..23b2c1247 100644 --- a/imap_processing/utils.py +++ b/imap_processing/utils.py @@ -10,7 +10,7 @@ from space_packet_parser.parser import Packet from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time +from imap_processing.cdf.utils import met_to_j2000ns from imap_processing.common_cdf_attrs import metadata_attrs logger = logging.getLogger(__name__) @@ -239,7 +239,7 @@ def update_epoch_to_datetime(dataset: xr.Dataset): Dataset with updated epoch dimension from int to datetime object. """ # convert epoch to datetime - epoch_converted_time = [calc_start_time(time) for time in dataset["epoch"].data] + epoch_converted_time = met_to_j2000ns(dataset["epoch"]) # add attrs back to epoch epoch = xr.DataArray( epoch_converted_time,