Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updated various CDF attributes/variable names to match expected nomenclature #813

Merged
merged 4 commits into from
Sep 11, 2024
Merged

Updated various CDF attributes/variable names to match expected nomenclature #813

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
422cfef
Updated various CDF attributes/variable names to match expected nomen…
bourque Sep 5, 2024
59d5b21
Fixed doc build error
bourque Sep 5, 2024
f694917
Added LABL_PTR_* attrs in anticipation of cdflib 1.3 upgrade
bourque Sep 9, 2024
466ab28
Switched esa_step VALIDMAX back to 127
bourque Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 36 additions & 47 deletions imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,94 +25,83 @@ codice_support_attrs: &support_default
VAR_TYPE: support_data

# <=== Coordinates ===>
energy_attrs:
esa_step:
<<: *default
CATDESC: Energy per charge (E/q) sweeping step
FIELDNAM: Energy step
FIELDNAM: Energy Index
FORMAT: I3
LABLAXIS: energy step
LABLAXIS: Energy Index
SI_CONVERSION: " > "
UNITS: ' '
VALIDMIN: 0
VALIDMAX: 127
VALIDMAX: 128
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure you want this to be 128? Just double checking

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I pulled this from the L1A validation CDFs that joey gave me but I do think it should be 127. I will double check with Joey.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Joey confirmed that this should be 127 (and that the exact values in his validation CDFs should not be considered the strict requirement).

VAR_TYPE: support_data

inst_az_attrs:
inst_az:
<<: *default
CATDESC: Azimuth
FIELDNAM: Azimuth
CATDESC: Azimuth Index
bourque marked this conversation as resolved.
Show resolved Hide resolved
FIELDNAM: Azimuth Index
FORMAT: I2
LABLAXIS: Azimuth
UNITS: ' '
LABLAXIS: " "
SI_CONVERSION: " > "
UNITS: " "
VALIDMIN: 0
VALIDMAX: 31
VAR_TYPE: support_data

spin_sector_attrs:
spin_sector:
<<: *default
CATDESC: Spin sector indicating range of spin angles
FIELDNAM: Spin sector
FIELDNAM: Spin Sector Index
FORMAT: I2
LABLAXIS: spin sector
UNITS: ' '
LABLAXIS: " "
SI_CONVERSION: " > "
UNITS: " "
VALIDMIN: 0
VALIDMAX: 11
VALIDMAX: 12
VAR_TYPE: support_data

# <=== Labels ===>
energy_label:
CATDESC: Energy per charge (E/q) sweeping step
DISPLAY_TYPE: ""
FIELDNAM: Energy step
FILLVAL: ""
FORMAT: A3
LABLAXIS: ""
REFERENCE_POSITION: ""
RESOLUTION: ""
TIME_BASE: ""
TIME_SCALE: ""
UNITS: ""
VALIDMAX: ""
VALIDMIN: ""
VAR_TYPE: metadata

# <=== Dataset Attributes ===>
acquisition_times_attrs:
# <=== Dataset Variable Attributes ===>
# The following are set in multiple data products
acquisition_time_per_step:
<<: *default
CATDESC: Time of acquisition for the energy step
DEPEND_1: energy
CATDESC: Acquisition time for each step of energy
FIELDNAM: Acquisition Time
FILLVAL: .NAN
FILLVAL: -1.0e+30
FORMAT: F10.3
LABLAXIS: Acq Time
LABLAXIS: " "
SI_CONVERSION: " > "
UNITS: ms
VALIDMIN: 0
VALIDMAX: 1000
VALIDMIN: 0.000000
VALIDMAX: 625.000000
VAR_TYPE: support_data

counters_attrs: &counters
<<: *default
CATDESC: Fill in at creation
DEPEND_0: epoch
DEPEND_1: energy
DEPEND_1: esa_step
DISPLAY_TYPE: time_series
FIELDNAM: Fill in at creation
LABL_PTR_1: energy_label
UNITS: counts
VALIDMIN: 0
VALIDMAX: 8388607 # max value for a signed 24-bit integer

esa_sweep_attrs:
energy_table:
<<: *default
CATDESC: ElectroStatic Analyzer Energy Values
DEPEND_1: energy
DEPEND_1: esa_step
FIELDNAM: ESA Voltage
FORMAT: I19
FORMAT: I5
LABLAXIS: ESA V
UNITS: V
VALIDMIN: 0
SCALETYP: log
SI_CONVERSION: " > "
UNITS: eV
VALIDMIN: 1
VALIDMAX: 14100
VAR_TYPE: support_data

# <=== Data Variable Attributes ===>
# The following are data product-specific
# hi-counters-aggregated
hi_counters_aggregated-aggregated:
<<: *counters
Expand Down
45 changes: 18 additions & 27 deletions imap_processing/codice/codice_l1a.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ class CoDICEL1aPipeline:
Create an ``xarray`` dataset for the unpacked science data.
get_acquisition_times()
Retrieve the acquisition times via the Lo stepping table.
get_esa_sweep_values()
get_energy_table()
Retrieve the ESA sweep values.
unpack_science_data()
Decompress, unpack, and restructure science data arrays.
Expand Down Expand Up @@ -136,27 +136,19 @@ def create_science_dataset(
np.arange(self.num_positions),
name="inst_az",
dims=["inst_az"],
attrs=cdf_attrs.get_variable_attributes("inst_az_attrs"),
attrs=cdf_attrs.get_variable_attributes("inst_az"),
)
spin_sector = xr.DataArray(
np.arange(self.num_spin_sectors),
name="spin_sector",
dims=["spin_sector"],
attrs=cdf_attrs.get_variable_attributes("spin_sector_attrs"),
attrs=cdf_attrs.get_variable_attributes("spin_sector"),
)
energy_steps = xr.DataArray(
esa_step = xr.DataArray(
np.arange(self.num_energy_steps),
name="energy",
dims=["energy"],
attrs=cdf_attrs.get_variable_attributes("energy_attrs"),
)

# Define labels
energy_label = xr.DataArray(
energy_steps.values.astype(str),
name="energy_label",
dims=["energy_label"],
attrs=cdf_attrs.get_variable_attributes("energy_label"),
name="esa_step",
dims=["esa_step"],
attrs=cdf_attrs.get_variable_attributes("esa_step"),
)

# Create the dataset to hold the data variables
Expand All @@ -165,8 +157,7 @@ def create_science_dataset(
"epoch": epoch,
"inst_az": inst_az,
"spin_sector": spin_sector,
"energy": energy_steps,
"energy_label": energy_label,
"esa_step": esa_step,
Comment on lines -168 to +160
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you remove those labels, you are gonna get error for data whose dimension have more than epoch as dimension when we need to update cdflib. To test it, some of us have been updating cdflib locally and then run the same test and it will raise those helpful errors. Can you do the same?

Eg.

dims = ["epoch", "inst_az", "spin_sector", "esa_step"]

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yes, I forgot that I added energy_label in anticipation of the cdflib upgrade in the first place. But I think I found a new (better) solution, which is to define the LABL_PTR_[1|2|3]s in the attribute YAML file. It appears to work in both cdflib 1.3 and 1.2.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good to know. I am surprised that you were able to reuse dimension cdf attrs for labels like this

    LABL_PTR_1: inst_az
    LABL_PTR_2: spin_sector
    LABL_PTR_3: esa_step

I thought it has to be metadata type.

},
attrs=cdf_attrs.get_global_attributes(self.dataset_name),
)
Expand All @@ -183,7 +174,7 @@ def create_science_dataset(
self.num_energy_steps,
)
)
dims = ["epoch", "inst_az", "spin_sector", "energy"]
dims = ["epoch", "inst_az", "spin_sector", "esa_step"]
elif self.instrument == "hi":
variable_data_arr = np.array(variable_data).reshape(
(
Expand All @@ -193,7 +184,7 @@ def create_science_dataset(
self.num_spin_sectors,
)
)
dims = ["epoch", "energy", "inst_az", "spin_sector"]
dims = ["epoch", "esa_step", "inst_az", "spin_sector"]

# Get the CDF attributes
cdf_attrs_key = (
Expand All @@ -211,17 +202,17 @@ def create_science_dataset(

# Add ESA Sweep Values and acquisition times (lo only)
if self.instrument == "lo":
self.get_esa_sweep_values()
self.get_energy_table()
self.get_acquisition_times()
dataset["esa_sweep_values"] = xr.DataArray(
self.esa_sweep_values,
dims=["energy"],
attrs=cdf_attrs.get_variable_attributes("esa_sweep_attrs"),
self.energy_table,
dims=["esa_step"],
attrs=cdf_attrs.get_variable_attributes("energy_table"),
)
dataset["acquisition_times"] = xr.DataArray(
self.acquisition_times,
dims=["energy"],
attrs=cdf_attrs.get_variable_attributes("acquisition_times_attrs"),
dims=["esa_step"],
attrs=cdf_attrs.get_variable_attributes("acquisition_time_per_step"),
)

return dataset
Expand Down Expand Up @@ -270,7 +261,7 @@ def get_acquisition_times(self) -> None:
row_number = np.argmax(energy_steps == str(step_number), axis=1).argmax()
self.acquisition_times.append(lo_stepping_values.acq_time[row_number])

def get_esa_sweep_values(self) -> None:
def get_energy_table(self) -> None:
"""
Retrieve the ESA sweep values.

Expand Down Expand Up @@ -299,7 +290,7 @@ def get_esa_sweep_values(self) -> None:

# Get the appropriate values
sweep_table = sweep_data[sweep_data["table_idx"] == sweep_table_id]
self.esa_sweep_values = sweep_table["esa_v"].values
self.energy_table = sweep_table["esa_v"].values

def unpack_science_data(self, science_values: str) -> None:
"""
Expand Down
Loading