diff --git a/imap_processing/cdf/config/imap_glows_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_glows_global_cdf_attrs.yaml new file mode 100644 index 000000000..50896f823 --- /dev/null +++ b/imap_processing/cdf/config/imap_glows_global_cdf_attrs.yaml @@ -0,0 +1,26 @@ +instrument_base: &instrument_base + Descriptor: GLOWS>GLObal Solar Wind Structure + TEXT: > + The GLObal Solar Wind Structure (GLOWS) is a non-imaging single-pixel Lyman-alpha + photometer that will be used to observe the sky distribution of the helioglow to + better understand the evolution of the solar wind structure. + The Lyman-alpha photon counts from these observations can be used to build a more + comprehensive picture of the solar wind structure and how it changes through the + solar cycles. + GLOWS design and assembly is led by the Space Research Center, Warsaw, Poland + (CBK PAN). See https://imap.princeton.edu/instruments/glows for more details. + Instrument_type: Particles (space) + +imap_glows_l1b_hist: + <<: *instrument_base + Data_level: L1B + Data_type: L1B_hist>Level-1B histogram + Logical_source: imap_glows_l1b_hist + Logical_source_description: IMAP Mission GLOWS Instrument Level-1B Histogram Data. + +imap_glows_l1b_de: + <<: *instrument_base + Data_level: L1B + Data_type: L1B_de>Level-1B Direct Events + Logical_source: imap_glows_l1b_de + Logical_source_description: IMAP Mission GLOWS Instrument Level-1B Direct Events Data. diff --git a/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml new file mode 100644 index 000000000..1c532daa7 --- /dev/null +++ b/imap_processing/cdf/config/imap_glows_l1b_variable_attrs.yaml @@ -0,0 +1,336 @@ +int_fillval: &int_fillval -9223372036854775808 +valid_max: &valid_max 3155630469184000000 + +epoch_dim: + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + FILLVAL: *int_fillval + LABLAXIS: epoch + FORMAT: ' ' # Supposedly not required fails in xarray_to_cdf + UNITS: ns + VALIDMIN: -315575942816000000 + VALIDMAX: 3155630469184000000 + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: epoch + DISPLAY_TYPE: time_series + FILLVAL: *int_fillval + FORMAT: I12 + VALIDMIN: *int_fillval + VALIDMAX: 9223372036854775807 + VAR_TYPE: data + +default_coords: &default_coords + FORMAT: F2.4 # Float with 4 digits + VAR_TYPE: data + DISPLAY_TYPE: time_series + +metadata_default: &metadata_default + <<: *default + VAR_TYPE: metadata + LABLAXIS: Metadata + +bin_dim: + <<: *default_coords + VALIDMIN: 0 + VALIDMAX: 70 + CATDESC: Counts of direct events for photon impacts + FIELDNAM: Counts of direct events + LABLAXIS: Counts + +flag_dim: + <<: *default_coords + VALIDMIN: 0 + VALIDMAX: 1 + FILLVAL: -1 + CATDESC: Flags for histogram information + FIELDNAM: Flags for histogram information + LABLAXIS: Flags + UNITS: ' ' + +per_second_dim: + <<: *default_coords + VALIDMIN: 0 + VALIDMAX: 300 + CATDESC: Direct events recorded approximately per second + FIELDNAM: List of direct events + LABLAXIS: Photon Counts per Second + +ecliptic_dim: + <<: *default_coords + # TODO: Update validmin and validmax + VALIDMIN: *int_fillval + VALIDMAX: *valid_max + CATDESC: Cartesian Ecliptic X, Y, Z coordinates for the spacecraft in KM + FIELDNAM: Cartesian Ecliptic X, Y, Z coordinates + LABLAXIS: Ecliptic Coordinates + UNITS: km + FILLVAL: *int_fillval + +histogram_dim: + <<: *default + VALIDMIN: 0 + VALIDMAX: 70 + CATDESC: Histogram of direct events for photon impacts + DEPEND_1: bins + FIELDNAM: Histogram of direct events + FORMAT: F2.4 + +histograms: + <<: *default + DEPEND_1: bins + CATDESC: array of block-accumulated counts as histograms + FIELDNAM: block-accumulated count numbers + LABLAXIS: Bins + UNITS: counts + +flight_software_version: + <<: *metadata_default + CATDESC: IMAP flight software version + FIELDNAM: IMAP flight software version + +seq_count_in_pkts_file: + <<: *metadata_default + CATDESC: sequence count in packets file + FIELDNAM: sequence count in packets file + +flags_set_onboard: + <<: *default + DEPEND_1: flags + DISPLAY_TYPE: no_plot + LABLAXIS: Flags + CATDESC: > + Flags for bad data [missing PPS, missing time, missing spin phase, missing + spin periods, overexposure, nonmonotonic event, data collected at night, + HV test, pulse test, memory error] + FIELDNAM: Flags set onboard for missing or bad data + UNITS: ' ' + +last_spin_id: + <<: *metadata_default + CATDESC: Last spin ID + FIELDNAM: Last spin ID + +is_generated_on_ground: + <<: *metadata_default + CATDESC: Indicates if the histogram data is generated on ground + FIELDNAM: is generated on ground + +number_of_spins_per_block: + <<: *metadata_default + CATDESC: Number of spins per block + FIELDNAM: Number of spins per block + +unique_block_identifier: + <<: *metadata_default + CATDESC: YYYY-MM-DDThh:mm:ss based on IMAP UTC time + FIELDNAM: YYYY-MM-DDThh:mm:ss based on IMAP UTC time + +number_of_bins_per_histogram: + <<: *metadata_default + CATDESC: Number of histogram bins + FIELDNAM: Number of histogram bins + +number_of_events: + <<: *metadata_default + CATDESC: Total number of events/counts in histogram + FIELDNAM: total number of events/counts in histogram + +# TODO: Might not be required for L1B +number_of_de_packets: + <<: *metadata_default + CATDESC: Number of direct event packets + FIELDNAM: Number of direct event packets + +imap_spin_angle_bin_cntr: + <<: *metadata_default + CATDESC: IMAP spin angle psi for bin centers + FIELDNAM: IMAP spin angle psi for bin centers + +histogram_flag_array: + <<: *metadata_default + CATDESC: array of bad-angle flags for histogram bins + FIELDNAM: array of bad-angle flags for histogram bins + UNITS: ' ' + +filter_temperature_average: + <<: *metadata_default + CATDESC: block-averaged value, decoded to Celsius degrees + FIELDNAM: block-averaged value, decoded to Celsius degrees + +filter_temperature_std_dev: + <<: *metadata_default + CATDESC: standard deviation (1 sigma), decoded to Celsius degrees + FIELDNAM: standard deviation (1 sigma), decoded to Celsius degrees + +hv_voltage_average: + <<: *metadata_default + CATDESC: block-averaged value, decoded to volts + FIELDNAM: block-averaged value, decoded to volts + +hv_voltage_std_dev: + <<: *metadata_default + CATDESC: standard deviation (1 sigma), decoded to volts + FIELDNAM: standard deviation (1 sigma), decoded to volts + +spin_period_average: + <<: *metadata_default + CATDESC: block-averaged onboard value, decoded to seconds + FIELDNAM: block-averaged onboard value, decoded to seconds + +spin_period_std_dev: + <<: *metadata_default + CATDESC: standard deviation (1 sigma), decoded to seconds + FIELDNAM: standard deviation (1 sigma), decoded to seconds + +pulse_length_average: + <<: *metadata_default + CATDESC: block-averaged value, decoded to us + FIELDNAM: block-averaged value, decoded to us + +pulse_length_std_dev: + <<: *metadata_default + CATDESC: Pulse length standard deviation (1 sigma) + FIELDNAM: Pulse length standard deviation + +glows_start_time: + <<: *metadata_default + CATDESC: GLOWS clock, subseconds as decimal part of float + FIELDNAM: GLOWS clock, subseconds as decimal part of float + +glows_end_time_offset: + <<: *metadata_default + CATDESC: GLOWS clock, subseconds as decimal part of float + FIELDNAM: GLOWS clock, subseconds as decimal part of float + +imap_start_time: + <<: *metadata_default + CATDESC: IMAP clock, subseconds as decimal part of float + FIELDNAM: IMAP clock, subseconds as decimal part of float + +imap_end_time_offset: + <<: *metadata_default + CATDESC: IMAP clock, subseconds as decimal part of float + FIELDNAM: IMAP clock, subseconds as decimal part of float + +spin_period_ground_average: + <<: *metadata_default + CATDESC: block-averaged value computed on ground + FIELDNAM: block-averaged value computed on ground + +spin_period_ground_std_dev: + <<: *metadata_default + CATDESC: standard deviation (1 sigma) + FIELDNAM: standard deviation (1 sigma) + +position_angle_offset_average: + <<: *metadata_default + CATDESC: block-averaged value in degrees + FIELDNAM: block-averaged value in degrees + +position_angle_offset_std_dev: + <<: *metadata_default + CATDESC: standard deviation (1 sigma) + FIELDNAM: standard deviation (1 sigma) + +spin_axis_orientation_std_dev: + <<: *metadata_default + CATDESC: standard deviation( 1 sigma) for spin axis orientation + FIELDNAM: standard deviation for spin axis orientation + +spin_axis_orientation_average: + <<: *metadata_default + CATDESC: block-averaged spin-axis ecliptic longitude and latitude in degrees + FIELDNAM: block-averaged spin-axis in degrees + +spacecraft_location_average: + <<: *metadata_default + CATDESC: block-averaged Cartesian ecliptic coordinates [X, Y, Z] [km] of IMAP + FIELDNAM: block-averaged Cartesian ecliptic coordinates of IMAP + +spacecraft_location_std_dev: + <<: *metadata_default + CATDESC: Spacecraft location standard deviations (1 sigma) delta X, delta Y, delta Z for [X, Y, Z] + FIELDNAM: standard deviations (1 sigma) for spacecraft location + +spacecraft_velocity_average: + <<: *metadata_default + CATDESC: block-averaged values [VX, VY, VZ] [km/s] of IMAP velocity components (Cartesian ecliptic frame) + FIELDNAM: block-averaged values of IMAP velocity components (Cartesian ecliptic frame) + +spacecraft_velocity_std_dev: + <<: *metadata_default + CATDESC: Spacecraft velocity standard deviations (1 sigma) delta VX, delta VY, delta VZ for [VX, VY, VZ] + FIELDNAM: Spacecraft velocity standard deviations + +flags: + <<: *metadata_default + CATDESC: flags for extra information, per histogram. + FIELDNAM: flags for extra information, per histogram. + +# DE attributes +unique_identifier: + <<: *metadata_default + CATDESC: YYYY-MM-DDThh:mm:ss based on IMAP UTC time + FIELDNAM: YYYY-MM-DDThh:mm:ss based on IMAP UTC time + +imap_time_last_pps: + <<: *metadata_default + CATDESC: IMAP clock, subseconds as decimal part of float for the last PPS + FIELDNAM: IMAP clock, subseconds as decimal part of float + +imap_time_next_pps: + <<: *metadata_default + CATDESC: IMAP clock, subseconds as decimal part of float for the next PPS + FIELDNAM: IMAP clock, subseconds as decimal part of float + +glows_time_last_pps: + <<: *metadata_default + CATDESC: GLOWS clock, subseconds as decimal part of float for the last PPS + FIELDNAM: GLOWS clock, subseconds as decimal part of float + +number_of_completed_spins: + <<: *metadata_default + CATDESC: Number of completed spins + FIELDNAM: Number of completed spins + +filter_temperature: + <<: *metadata_default + CATDESC: Filter temperature in degrees Celsius + FIELDNAM: Filter temperature in degrees Celsius + +hv_voltage: + <<: *metadata_default + CATDESC: HV voltage in volts + FIELDNAM: HV voltage in volts + +spin_period: + <<: *metadata_default + CATDESC: Spin period in seconds + FIELDNAM: Spin period in seconds + +spin_phase_at_next_pps: + <<: *metadata_default + CATDESC: Spin phase at the next PPS in degrees + FIELDNAM: Spin phase at the next PPS in degrees + +direct_event_glows_times: + <<: *metadata_default + DEPEND_1: per_second + CATDESC: > + An array of times for direct events from the GLOWS clock, with subseconds as the + decimal part of the float. + FIELDNAM: Direct event times, GLOWS clock + +direct_event_pulse_lengths: + <<: *metadata_default + DEPEND_1: per_second + CATDESC: An array of pulse lengths for direct events in microseconds + FIELDNAM: Direct event pulse lengths (us) diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index 149df960f..f0f865b6d 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -117,7 +117,6 @@ def write_cdf(dataset: xr.Dataset): # Create the filename from the global attributes # Logical_source looks like "imap_swe_l2_counts-1min" instrument, data_level, descriptor = dataset.attrs["Logical_source"].split("_")[1:] - start_time = np.datetime_as_string(dataset["epoch"].values[0], unit="D").replace( "-", "" ) diff --git a/imap_processing/cli.py b/imap_processing/cli.py index 5713fde1d..8d6ba7f87 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -33,6 +33,8 @@ # In code: # call cdf.utils.write_cdf from imap_processing.codice import codice_l1a +from imap_processing.glows.l1a.glows_l1a import glows_l1a +from imap_processing.glows.l1b.glows_l1b import glows_l1b from imap_processing.hi.l1a import hi_l1a from imap_processing.hi.l1b import hi_l1b from imap_processing.hit.l1a.hit_l1a import hit_l1a @@ -390,6 +392,27 @@ class Glows(ProcessInstrument): def do_processing(self, dependencies): """Perform GLOWS specific processing.""" print(f"Processing GLOWS {self.data_level}") + products = [] + if self.data_level == "l1a": + if len(dependencies) > 1: + raise ValueError( + f"Unexpected dependencies found for GLOWS L1A:" + f"{dependencies}. Expected only one input dependency." + ) + datasets = glows_l1a(dependencies[0], self.version) + products = [write_cdf(dataset) for dataset in datasets] + + if self.data_level == "l1b": + if len(dependencies) < 1: + raise ValueError( + f"Unexpected dependencies found for GLOWS L1B:" + f"{dependencies}. Expected at least one input dependency." + ) + input_dataset = load_cdf(dependencies[0]) + dataset = glows_l1b(input_dataset, self.version) + products = [write_cdf(dataset)] + + return products class Hi(ProcessInstrument): diff --git a/imap_processing/glows/l0/glows_l0_data.py b/imap_processing/glows/l0/glows_l0_data.py index 25579e3a0..4a4038439 100644 --- a/imap_processing/glows/l0/glows_l0_data.py +++ b/imap_processing/glows/l0/glows_l0_data.py @@ -150,10 +150,7 @@ class DirectEventL0(GlowsL0): Methods ------- - within_same_sequence - Compare two DirectEventL0 objects and see if MET and LEN are equal (meaning they - are in the same sequence) - + within_same_sequence(other) """ MET: int diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index 7e90bd57f..7710f2a9d 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -8,18 +8,19 @@ import xarray as xr from imap_processing.cdf.global_attrs import ConstantCoordinates -from imap_processing.cdf.utils import calc_start_time, write_cdf +from imap_processing.cdf.utils import calc_start_time from imap_processing.glows import __version__, glows_cdf_attrs from imap_processing.glows.l0.decom_glows import decom_packets from imap_processing.glows.l0.glows_l0_data import DirectEventL0 from imap_processing.glows.l1a.glows_l1a_data import DirectEventL1A, HistogramL1A -def glows_l1a(packet_filepath: Path, data_version: str) -> list[Path]: +def glows_l1a(packet_filepath: Path, data_version: str) -> list[xr.Dataset]: """ Process packets into GLOWS L1A CDF files. - Outputs CDF files for histogram and direct event GLOWS L1A CDF files. + Outputs Datasets for histogram and direct event GLOWS L1A. This list can be passed + into write_cdf to output CDF files. Parameters ---------- @@ -30,8 +31,8 @@ def glows_l1a(packet_filepath: Path, data_version: str) -> list[Path]: Returns ------- - generated_files: list[Path] - List of the paths of the generated CDF files + generated_files: list[xr.Dataset] + List of the L1A datasets """ # TODO: Data version inside file as well? # Create glows L0 @@ -49,16 +50,16 @@ def glows_l1a(packet_filepath: Path, data_version: str) -> list[Path]: hists_by_day[hist_day].append(hist_l1a) # Generate CDF files for each day - generated_files = [] + output_datasets = [] for hist_l1a_list in hists_by_day.values(): dataset = generate_histogram_dataset(hist_l1a_list, data_version) - generated_files.append(write_cdf(dataset)) + output_datasets.append(dataset) for de_l1a_list in de_by_day.values(): dataset = generate_de_dataset(de_l1a_list, data_version) - generated_files.append(write_cdf(dataset)) + output_datasets.append(dataset) - return generated_files + return output_datasets def process_de_l0( diff --git a/imap_processing/glows/l1b/glows_l1b.py b/imap_processing/glows/l1b/glows_l1b.py index 975df0781..5f28cdd43 100644 --- a/imap_processing/glows/l1b/glows_l1b.py +++ b/imap_processing/glows/l1b/glows_l1b.py @@ -2,24 +2,130 @@ import dataclasses +import numpy as np import xarray as xr +from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.glows.l1b.glows_l1b_data import DirectEventL1B, HistogramL1B -def glows_l1b_histograms(input_dataset: xr.Dataset) -> xr.Dataset: - """Process the GLOWS L1B data and format the histogram attribute outputs.""" - # TODO: add CDF attribute steps +def glows_l1b(input_dataset: xr.Dataset, data_version: str) -> xr.Dataset: + """Process the GLOWS L1B data and format the output datasets.""" + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("glows") + cdf_attrs.add_instrument_variable_attrs("glows", "l1b") + cdf_attrs.add_global_attribute("Data_version", data_version) - # Will call process_histograms and construct the dataset with the return - raise NotImplementedError + data_epoch = xr.DataArray( + input_dataset["epoch"], + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch_dim"), + ) + logical_source = ( + input_dataset.attrs["Logical_source"][0] + if isinstance(input_dataset.attrs["Logical_source"], list) + else input_dataset.attrs["Logical_source"] + ) -def glows_l1b_de(input_dataset: xr.Dataset) -> xr.Dataset: - """Process the GLOWS L1B data and format the direct event attribute outputs.""" - # TODO: generate dataset with CDF attributes - # Will call process_de and construct the dataset with the return - raise NotImplementedError + if "hist" in logical_source: + flag_data = xr.DataArray( + np.arange(17), + name="flags", + dims=["flags"], + attrs=cdf_attrs.get_variable_attributes("flag_dim"), + ) + bad_flag_data = xr.DataArray( + np.arange(4), + name="bad_angle_flags", + dims=["bad_angle_flags"], + attrs=cdf_attrs.get_variable_attributes("flag_dim"), + ) + + # TODO: the four spacecraft location/velocity values should probably each get + # their own dimension/attributes + eclipic_data = xr.DataArray( + np.arange(3), + name="ecliptic", + dims=["ecliptic"], + attrs=cdf_attrs.get_variable_attributes("ecliptic_dim"), + ) + + bin_data = xr.DataArray( + input_dataset["bins"], + name="bins", + dims=["bins"], + attrs=cdf_attrs.get_variable_attributes("bin_dim"), + ) + + output_dataarrays = process_histogram(input_dataset) + # TODO: Is it ok to copy the dimensions from the input dataset? + + output_dataset = xr.Dataset( + coords={ + "epoch": data_epoch, + "bins": bin_data, + "bad_angle_flags": bad_flag_data, + "flags": flag_data, + "ecliptic": eclipic_data, + }, + attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_hist"), + ) + + # Since we know the output_dataarrays are in the same order as the fields in the + # HistogramL1B dataclass, we can use dataclasses.fields to get the field names. + + fields = dataclasses.fields(HistogramL1B) + + for index, dataarray in enumerate(output_dataarrays): + # Dataarray is already an xr.DataArray type, so we can just assign it + output_dataset[fields[index].name] = dataarray + output_dataset[ + fields[index].name + ].attrs = cdf_attrs.get_variable_attributes(fields[index].name) + + elif "de" in logical_source: + output_dataarrays = process_de(input_dataset) + per_second_data = xr.DataArray( + input_dataset["per_second"], + name="per_second", + dims=["per_second"], + attrs=cdf_attrs.get_variable_attributes("per_second_dim"), + ) + + flag_data = xr.DataArray( + np.arange(11), + name="flags", + dims=["flags"], + attrs=cdf_attrs.get_variable_attributes("flag_dim"), + ) + + output_dataset = xr.Dataset( + coords={ + "epoch": data_epoch, + "per_second": per_second_data, + "flags": flag_data, + }, + attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_de"), + ) + fields = dataclasses.fields(DirectEventL1B) + + for index, dataarray in enumerate(output_dataarrays): + # Dataarray is already an xr.DataArray type, so we can just assign it + output_dataset[fields[index].name] = dataarray + output_dataset[ + fields[index].name + ].attrs = cdf_attrs.get_variable_attributes(fields[index].name) + + else: + raise ValueError( + f"Logical_source {input_dataset.attrs['Logical_source']} for input file " + f"does not match histogram " + "('hist') or direct event ('de')." + ) + + return output_dataset def process_de(l1a: xr.Dataset) -> tuple[xr.DataArray]: @@ -50,31 +156,37 @@ def process_de(l1a: xr.Dataset) -> tuple[xr.DataArray]: # is passed into the function (here a lambda.) # We need to specify the other dimensions for input and output so the arrays are - # properly aligned. The input dimensions are in `dims` and the output dimensions are - # in `new_dims`. + # properly aligned. The input dimensions are in `input_dims` and the output + # dimensions are in `output_dims`. # An empty array passes the epoch dimension through - dims = [[] for i in l1a.keys()] - new_dims = [[] for i in range(14)] + input_dims = [[] for i in l1a.keys()] + + output_dimension_mapping = { + "flags": ["flags"], + "direct_event_glows_times": ["per_second"], + "direct_event_pulse_lengths": ["per_second"], + } + + # For each attribute, retrieve the dims from output_dimension_mapping or use an + # empty list. Output_dims should be the same length as the number of attributes in + # the class. + output_dims = [ + output_dimension_mapping.get(field.name, []) + for field in dataclasses.fields(DirectEventL1B) + ] # Set the two direct event dimensions. This is the only multi-dimensional L1A # (input) variable. - dims[0] = ["per_second", "direct_event"] - - # Flags is a constant length. It has a dimension of "flags" - new_dims[-3] = ["flags"] - - # glows_times and pulse_lengths should be dimension of "per_second", the same as - # the input. - new_dims[-2] = ["per_second"] - new_dims[-1] = ["per_second"] + input_dims[0] = ["per_second", "direct_event"] l1b_fields = xr.apply_ufunc( lambda *args: tuple(dataclasses.asdict(DirectEventL1B(*args)).values()), *dataarrays, - input_core_dims=dims, - output_core_dims=new_dims, + input_core_dims=input_dims, + output_core_dims=output_dims, vectorize=True, + keep_attrs=True, ) return l1b_fields @@ -103,39 +215,41 @@ def process_histogram(l1a: xr.Dataset) -> xr.Dataset: """ dataarrays = [l1a[i] for i in l1a.keys()] - dims = [[] for i in l1a.keys()] - # 34 is the number of output attributes in the HistogramL1B class. - new_dims = [[] for i in range(34)] - - # histograms is the only multi dimensional variable, so we need to set its dims to - # pass along all the dims EXCEPT for epoch. (in this case just "bins") - # The rest of the vars are epoch only, so they have an empty list. - dims[0] = ["bins"] - - # This preserves the dimensions for the histogram output - new_dims[0] = ["bins"] - new_dims[22] = ["bins"] # For imap_spin_angle_center - a per-bin value - - # For histogram_flag_array - varies by the new dimension "flags" and by the number - # of bins - new_dims[23] = ["flags", "bins"] - - # For the new arrays added: add their dimensions. These aren't defined anywhere, - # so when the dataset is created I will need to add "ecliptic" as a new dimension. - - # These represent the spacecraft position and std dev, and velocity and std dev. - new_dims[-4] = ["ecliptic"] - new_dims[-3] = ["ecliptic"] - new_dims[-2] = ["ecliptic"] - new_dims[-1] = ["ecliptic"] + input_dims = [[] for i in l1a.keys()] + + # This should include a mapping to every dimension in the output data besides epoch. + # Only non-1D variables need to be in this mapping. + output_dimension_mapping = { + "histograms": ["bins"], + "imap_spin_angle_bin_cntr": ["bins"], + "histogram_flag_array": ["bad_angle_flags", "bins"], + "spacecraft_location_average": ["ecliptic"], + "spacecraft_location_std_dev": ["ecliptic"], + "spacecraft_velocity_average": ["ecliptic"], + "spacecraft_velocity_std_dev": ["ecliptic"], + "flags": ["flags", "bins"], + } + + # For each attribute, retrieve the dims from output_dimension_mapping or use an + # empty list. Output_dims should be the same length as the number of attributes in + # the class. + output_dims = [ + output_dimension_mapping.get(field.name, []) + for field in dataclasses.fields(HistogramL1B) + ] + + # histograms is the only multi dimensional input variable, so we set the non-epoch + # dimension ("bins"). + # The rest of the input vars are epoch only, so they have an empty list. + input_dims[0] = ["bins"] - # TODO: validate the input dims line up with the dims value l1b_fields = xr.apply_ufunc( - lambda *args: tuple(dataclasses.asdict(HistogramL1B(*args)).values()), + lambda *args: HistogramL1B(*args).output_data(), *dataarrays, - input_core_dims=dims, - output_core_dims=new_dims, + input_core_dims=input_dims, + output_core_dims=output_dims, vectorize=True, + keep_attrs=True, ) # This is a tuple of dataarrays and not a dataset yet diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index de2559c73..d689a6734 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -1,6 +1,7 @@ # ruff: noqa: PLR0913 """Module for GLOWS L1B data products.""" +import dataclasses import json from dataclasses import InitVar, dataclass, field from pathlib import Path @@ -220,8 +221,8 @@ class DirectEventL1B: direct_events: InitVar[np.ndarray] seq_count_in_pkts_file: np.double # Passed from L1A - unique_identifier: str = field(init=False) - number_of_de_packets: np.double + # unique_identifier: str = field(init=False) + number_of_de_packets: np.double # TODO Is this required in L1B? imap_time_last_pps: np.double glows_time_last_pps: np.double # Added to the end of glows_time_last_pps as subseconds @@ -277,9 +278,11 @@ def __post_init__( ) # TODO: double check that this time is in unix time and is the correct variable - self.unique_identifier = np.datetime_as_string( - np.datetime64(int(self.imap_time_last_pps), "ns"), "s" - ) + # TODO: This cannot be in the data because it's a string, put it in the + # attributes + # self.unique_identifier = np.datetime_as_string( + # np.datetime64(int(self.imap_time_last_pps), "ns"), "s" + # ) self.glows_time_last_pps = TimeTuple( int(self.glows_time_last_pps), glows_ssclk_last_pps ).to_seconds() @@ -359,6 +362,8 @@ class HistogramL1B: array of block-accumulated count numbers flight_software_version: str seq_count_in_pkts_file: int + last_spin_id: int + The ID of the previous spin flags_set_onboard: int is_generated_on_ground: int number_of_spins_per_block @@ -371,8 +376,6 @@ class HistogramL1B: total number of events/counts in histogram imap_spin_angle_bin_cntr IMAP spin angle ψ for bin centers, see Sec. - - histogram_flag_array - array of bad-angle flags for histogram bins, see Tab. 14 filter_temperature_average block-averaged value, decoded to Celsius degrees using Eq. (47) filter_temperature_std_dev @@ -397,6 +400,9 @@ class HistogramL1B: IMAP clock, subseconds as decimal part of float, see Sec. -.1 imap_end_time_offset IMAP clock, subseconds as decimal part of float, see Sec. -.1 + histogram_flag_array + flags for bad-time information per bin, consisting of [is_close_to_uv_source, + is_inside_excluded_region, is_excluded_by_instr_team, is_suspected_transient] spin_period_ground_average block-averaged value computed on ground, see Sec. -.1 spin_period_ground_std_dev @@ -447,13 +453,11 @@ class HistogramL1B: imap_end_time_offset: np.double # No conversion needed from l1a->l1b glows_start_time: np.double # No conversion needed from l1a->l1b glows_end_time_offset: np.double # No conversion needed from l1a->l1b - unique_block_identifier: str = field( - init=False - ) # Could be datetime TODO: Missing from values in L1A - imap_spin_angle_bin_cntr: np.ndarray = field( - init=False - ) # Same size as bins TODO add dims - histogram_flag_array: np.ndarray = field(init=False) # TODO add dims + # unique_block_identifier: str = field( + # init=False + # ) # Could be datetime TODO: Can't put a string in data + imap_spin_angle_bin_cntr: np.ndarray = field(init=False) # Same size as bins + histogram_flag_array: np.ndarray = field(init=False) spin_period_ground_average: np.double = field(init=False) # retrieved from SPICE? spin_period_ground_std_dev: np.double = field(init=False) # retrieved from SPICE? position_angle_offset_average: np.double = field(init=False) # retrieved from SPICE @@ -464,13 +468,12 @@ class HistogramL1B: spacecraft_location_std_dev: np.ndarray = field(init=False) # retrieved from SPIC spacecraft_velocity_average: np.ndarray = field(init=False) # retrieved from SPIC spacecraft_velocity_std_dev: np.ndarray = field(init=False) # retrieved from SPIC - # TODO make these human - readable - # flags: np.ndarray = field(init=False) # Generated per-histogram - + flags: np.ndarray = field(init=False) # TODO: # - Determine a good way to output flags as "human readable" # - Add spice pieces # - add in the filenames for the input files - should they be global attributes? + # - also unique identifiers # - Bad angle algorithm using SPICE locations # - Move ancillary file to AWS @@ -526,7 +529,22 @@ def __post_init__(self): "pulse_length", self.pulse_length_std_dev ) - self.histogram_flag_array = np.zeros((17, 3600)) - self.unique_block_identifier = np.datetime_as_string( - np.datetime64(int(self.imap_start_time), "ns"), "s" - ) + self.histogram_flag_array = np.zeros((4, 3600)) + # self.unique_block_identifier = np.datetime_as_string( + # np.datetime64(int(self.imap_start_time), "ns"), "s" + # ) + self.flags = np.zeros((17, 3600)) + + def output_data(self) -> tuple: + """ + Output the L1B DataArrays as a tuple. + + It is faster to return the values like this than to use to_dict() from + dataclasses. + + Returns + ------- + tuple: + A tuple containing each attribute value in the class. + """ + return tuple(getattr(self, out.name) for out in dataclasses.fields(self)) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index 86eeeed1f..dc38b9312 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -4,7 +4,8 @@ import pytest import xarray as xr -from imap_processing.glows.l1b.glows_l1b import process_de, process_histogram +from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.glows.l1b.glows_l1b import glows_l1b, process_de, process_histogram from imap_processing.glows.l1b.glows_l1b_data import ( AncillaryParameters, DirectEventL1B, @@ -36,11 +37,24 @@ def hist_dataset(): "glows_start_time": np.zeros((20,)), "glows_time_offset": np.zeros((20,)), } - epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"]) + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("glows") + cdf_attrs.add_instrument_variable_attrs("glows", "l1b") + + epoch = xr.DataArray( + np.arange(20), + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), + ) bins = xr.DataArray(np.arange(3600), name="bins", dims=["bins"]) - ds = xr.Dataset(coords={"epoch": epoch}) + ds = xr.Dataset( + coords={"epoch": epoch}, + attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_hist"), + ) + ds["histograms"] = xr.DataArray( np.zeros((20, 3600)), dims=["epoch", "bins"], @@ -79,7 +93,17 @@ def de_dataset(): "pulse_test_in_progress": np.zeros((20,)), "memory_error_detected": np.zeros((20,)), } - epoch = xr.DataArray(np.arange(20), name="epoch", dims=["epoch"]) + + cdf_attrs = ImapCdfAttributes() + cdf_attrs.add_instrument_global_attrs("glows") + cdf_attrs.add_instrument_variable_attrs("glows", "l1b") + + epoch = xr.DataArray( + np.arange(20), + name="epoch", + dims=["epoch"], + attrs=cdf_attrs.get_variable_attributes("epoch_attrs"), + ) per_second = xr.DataArray(np.arange(2295), name="per_second", dims=["per_second"]) direct_event = xr.DataArray( @@ -91,7 +115,10 @@ def de_dataset(): variables["filter_temperature"][0] = 100 - ds = xr.Dataset(coords={"epoch": epoch}) + ds = xr.Dataset( + coords={"epoch": epoch, "per_second": per_second, "direct_event": direct_event}, + attrs=cdf_attrs.get_global_attributes("imap_glows_l1b_de"), + ) ds["direct_events"] = xr.DataArray( de_data, @@ -243,4 +270,79 @@ def test_process_de(de_dataset, ancillary_dict): expected_temp = ancillary.decode("filter_temperature", 100.0) - assert np.isclose(output[9].data[0], expected_temp) + assert np.isclose(output[8].data[0], expected_temp) + + +def test_glows_l1b(de_dataset, hist_dataset): + hist_output = glows_l1b(hist_dataset, "V001") + + assert hist_output["histograms"].dims == ("epoch", "bins") + assert hist_output["histograms"].shape == (20, 3600) + + # This needs to be added eventually, but is skipped for now. + expected_de_data = [ + "flight_software_version", + "ground_software_version", + "pkts_file_name", + "seq_count_in_pkts_file", + "l1a_file_name", + "ancillary_data_files", + ] + + # From table 11 in the algorithm document + expected_hist_data = [ + # "unique_block_identifier", Left out until I determine where to put string data + "glows_start_time", + "glows_end_time_offset", + "imap_start_time", + "imap_end_time_offset", + "number_of_spins_per_block", + "number_of_bins_per_histogram", + "histograms", # named histogram in alg doc + "number_of_events", + "imap_spin_angle_bin_cntr", + "histogram_flag_array", + "filter_temperature_average", + "filter_temperature_std_dev", + "hv_voltage_average", + "hv_voltage_std_dev", + "spin_period_average", + "spin_period_std_dev", + "pulse_length_average", + "pulse_length_std_dev", + "spin_period_ground_average", + "spin_period_ground_std_dev", + "position_angle_offset_average", + "position_angle_offset_std_dev", + "spin_axis_orientation_std_dev", + "spin_axis_orientation_average", + "spacecraft_location_average", + "spacecraft_location_std_dev", + "spacecraft_velocity_average", + "spacecraft_velocity_std_dev", + "flags", + ] + + for key in expected_hist_data: + assert key in hist_output + + de_output = glows_l1b(de_dataset, "V001") + + # From table 15 in the algorithm document + expected_de_data = [ + # "unique_identifier", TODO: determine the best way to add this + "imap_time_last_pps", + "imap_time_next_pps", + "glows_time_last_pps", + "number_of_completed_spins", + "filter_temperature", + "hv_voltage", + "spin_period", + "spin_phase_at_next_pps", + "flags", + "direct_event_glows_times", + "direct_event_pulse_lengths", + ] + + for key in expected_de_data: + assert key in de_output