From 24cc329cfeb28df8d0e188714b2bae9fbd999d63 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 13 Jun 2024 09:49:25 -0600 Subject: [PATCH 01/40] Writing initial TODO comments. --- .../cdf/tests/test_cdf_attribute_manager.py | 18 +++++++++++++++++- .../cdf/tests/test_imap_cdf_manager.py | 0 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 imap_processing/cdf/tests/test_imap_cdf_manager.py diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 2578f36ed..4d78ec338 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -2,6 +2,21 @@ from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager +# test __init__(self, data_dir: Path): (?) +# test load_global_attributes(self, file_path: str): +# test _load_yaml_data(file_path: str | Path) -> dict: (?) +# test load_variable_attributes(self, file_name: str) -> None: +# test get_variable_attributes(self, variable_name: str) -> dict: + +# test _load_default_global_attr_schema(sef) -> dict: (?) +# test _load_default_variable_attr_schema(self) -> dict: (?) +def test_default_attr_schema(): + # TODO: initialize CdfAttributeManager object + # TODO: assert statement to test default global schema + # TODO: assert statement to test default variable schema + +def test_load_yaml_data(): + # TODO: Check if file is properly loaded in ... ? # @pytest.mark.xfail(reason="Missing IMAP specific global schema") def test_global_attribute(): @@ -37,7 +52,6 @@ def test_global_attribute(): assert mag_l1a_global_attrs["Descriptor"] == "MAG>Magnetometer" assert mag_l1a_global_attrs["Logical_source"] == "imap_mag_l1a_norm-raw" - def test_variable_attribute(): cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") @@ -56,3 +70,5 @@ def test_variable_attribute(): for variable_attrs in cdf_manager.variable_attributes.values(): for attr in expected_attributes: assert attr in variable_attrs.keys() + + # TODO: Call, and test get_variable_attributes \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py new file mode 100644 index 000000000..e69de29bb From f7d0262623e6e44c73abc097cc8dea1e5c1ac1c9 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 13 Jun 2024 15:35:04 -0600 Subject: [PATCH 02/40] Troubleshooting --- imap_processing/cdf/tests/__init__.py | 0 .../cdf/tests/imap_test_global.yaml | 56 +++++++++++++++++++ .../cdf/tests/imap_test_variable.yaml | 0 .../cdf/tests/test_cdf_attribute_manager.py | 48 +++++++++++----- .../cdf/tests/test_imap_cdf_manager.py | 9 +++ 5 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 imap_processing/cdf/tests/__init__.py create mode 100644 imap_processing/cdf/tests/imap_test_global.yaml create mode 100644 imap_processing/cdf/tests/imap_test_variable.yaml diff --git a/imap_processing/cdf/tests/__init__.py b/imap_processing/cdf/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml new file mode 100644 index 000000000..a6ea45631 --- /dev/null +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -0,0 +1,56 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_section_1: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. + +imap_test_section_2: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst>Level-1A raw burst rate + Logical_source: imap_mag_l1a_burst-raw + Logical_source_description: IMAP Mission MAG Burst Rate Instrument Level-1A Data. + + +imap_test_section_3: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-norm-mago>Level 1A MAGo normal rate + Logical_source: imap_mag_l1a_norm-mago + Logical_source_description: IMAP Mission MAGo Normal Rate Instrument Level-1A Data. + + +imap_test_section_4: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-norm-magi>Level 1A MAGi normal rate + Logical_source: imap_mag_l1a_norm-magi + Logical_source_description: IMAP Mission MAGi Normal Rate Instrument Level-1A Data. + + +imap_test_section_5: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst-mago>Level 1A MAGo burst rate + Logical_source: imap_mag_l1a_burst-mago + Logical_source_description: IMAP Mission MAGo Burst Rate Instrument Level-1A Data. + + +imap_test_section_7: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst-magi>Level 1A MAGi burst rate + Logical_source: imap_mag_l1a_burst-magi + Logical_source_description: IMAP Mission MAGi Burst Rate Instrument Level-1A Data. + + + diff --git a/imap_processing/cdf/tests/imap_test_variable.yaml b/imap_processing/cdf/tests/imap_test_variable.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 4d78ec338..fcf1e9456 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -2,26 +2,45 @@ from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager -# test __init__(self, data_dir: Path): (?) -# test load_global_attributes(self, file_path: str): -# test _load_yaml_data(file_path: str | Path) -> dict: (?) -# test load_variable_attributes(self, file_name: str) -> None: -# test get_variable_attributes(self, variable_name: str) -> dict: - -# test _load_default_global_attr_schema(sef) -> dict: (?) -# test _load_default_variable_attr_schema(self) -> dict: (?) + def test_default_attr_schema(): - # TODO: initialize CdfAttributeManager object - # TODO: assert statement to test default global schema - # TODO: assert statement to test default variable schema + """ + Test function that covers: + _load_default_global_attr_schema + _load_default_variable_attr_schema + """ + + # Initialize CdfAttributeManager object which loads in default schema + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Default global tests + # Check false case + assert cdf_manager.global_attribute_schema["DOI"]["required"] == "false" + + # Check true case + assert cdf_manager.global_attribute_schema["Data_level"] == "true" + + # Default variable tests + # Check false case + assert cdf_manager.variable_attribute_schema["TIME_BASE"]["required"] == "false" + + # Check true case + assert cdf_manager.variable_attribute_schema["RESOLUTION"]["required"] == "true" -def test_load_yaml_data(): - # TODO: Check if file is properly loaded in ... ? # @pytest.mark.xfail(reason="Missing IMAP specific global schema") + + def test_global_attribute(): + """ + Test function that covers: + load_global_attributes + get_global_attributes + """ + # Initialize CdfAttributeManager object which loads in default information cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + # Load in test data cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") # Expected failure: file_naming_convention is not in schema. @@ -52,6 +71,7 @@ def test_global_attribute(): assert mag_l1a_global_attrs["Descriptor"] == "MAG>Magnetometer" assert mag_l1a_global_attrs["Logical_source"] == "imap_mag_l1a_norm-raw" + def test_variable_attribute(): cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") @@ -71,4 +91,4 @@ def test_variable_attribute(): for attr in expected_attributes: assert attr in variable_attrs.keys() - # TODO: Call, and test get_variable_attributes \ No newline at end of file + # TODO: Call, and test get_variable_attributes diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index e69de29bb..e5d989496 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -0,0 +1,9 @@ +from pathlib import Path +from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager +from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes + +# test add_instrument_global_attrs(self, instrument: str): +def test_add_instrument_global_attrs(): + +# test add_instrument_variable_attrs(self, instrument: str, level: str): +def test_add_instrument_variable_attrs(): From 484d838fc524ff7dc34d9bbcbad897f830b89063 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Fri, 14 Jun 2024 10:01:18 -0600 Subject: [PATCH 03/40] Schema tests passed --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index fcf1e9456..2de7de343 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -15,17 +15,17 @@ def test_default_attr_schema(): # Default global tests # Check false case - assert cdf_manager.global_attribute_schema["DOI"]["required"] == "false" + assert cdf_manager.global_attribute_schema["DOI"]["required"] is False # Check true case - assert cdf_manager.global_attribute_schema["Data_level"] == "true" + assert cdf_manager.global_attribute_schema["Data_level"]["required"] is True # Default variable tests # Check false case - assert cdf_manager.variable_attribute_schema["TIME_BASE"]["required"] == "false" + assert cdf_manager.variable_attribute_schema['attribute_key']["ABSOLUTE_ERROR"]["required"] is False # Check true case - assert cdf_manager.variable_attribute_schema["RESOLUTION"]["required"] == "true" + assert cdf_manager.variable_attribute_schema['attribute_key']["RESOLUTION"]["required"] is True # @pytest.mark.xfail(reason="Missing IMAP specific global schema") From 56389b28b1f693b10a8d4e7edf37b9027682a923 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Fri, 14 Jun 2024 16:27:17 -0600 Subject: [PATCH 04/40] I think I understand instrument_id --- .../imap_default_global_test_cdf_attrs.yaml | 7 ++ .../cdf/config/imap_test_global.yaml | 0 .../cdf/tests/imap_test_global.yaml | 56 ------------- .../cdf/tests/test_cdf_attribute_manager.py | 79 ++++++++++++++----- 4 files changed, 65 insertions(+), 77 deletions(-) create mode 100644 imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml create mode 100644 imap_processing/cdf/config/imap_test_global.yaml delete mode 100644 imap_processing/cdf/tests/imap_test_global.yaml diff --git a/imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml b/imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml new file mode 100644 index 000000000..a40dfe3d1 --- /dev/null +++ b/imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml @@ -0,0 +1,7 @@ +Project: STP>Solar-Terrestrial Physics +Source_name: IMAP>Interstellar Mapping and Acceleration Probe +Mission_group: Dysfunctional Cats +PI_name: Ana Manica +PI_affiliation: LASP, CU +Data_version: 002 +Test_attribute: test \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_test_global.yaml b/imap_processing/cdf/config/imap_test_global.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml deleted file mode 100644 index a6ea45631..000000000 --- a/imap_processing/cdf/tests/imap_test_global.yaml +++ /dev/null @@ -1,56 +0,0 @@ -Descriptor: &desc - TEST>Testinstrument -TEXT: &txt > - This file is nothing more than a test yamal. This description? This is a test description. - This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. - This test file is led by me, Ana Manica. Lasp undergraduate employee. - I don't have a website. Sorry. - -imap_test_section_1: - Descriptor: *desc - TEXT: *txt - Data_type: T1_test-one>Test-1 test one - Logical_source: imap_test_T1_test - Logical_source_description: IMAP Mission TEST one document Level-T1. - -imap_test_section_2: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst>Level-1A raw burst rate - Logical_source: imap_mag_l1a_burst-raw - Logical_source_description: IMAP Mission MAG Burst Rate Instrument Level-1A Data. - - -imap_test_section_3: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-norm-mago>Level 1A MAGo normal rate - Logical_source: imap_mag_l1a_norm-mago - Logical_source_description: IMAP Mission MAGo Normal Rate Instrument Level-1A Data. - - -imap_test_section_4: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-norm-magi>Level 1A MAGi normal rate - Logical_source: imap_mag_l1a_norm-magi - Logical_source_description: IMAP Mission MAGi Normal Rate Instrument Level-1A Data. - - -imap_test_section_5: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst-mago>Level 1A MAGo burst rate - Logical_source: imap_mag_l1a_burst-mago - Logical_source_description: IMAP Mission MAGo Burst Rate Instrument Level-1A Data. - - -imap_test_section_7: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst-magi>Level 1A MAGi burst rate - Logical_source: imap_mag_l1a_burst-magi - Logical_source_description: IMAP Mission MAGi Burst Rate Instrument Level-1A Data. - - - diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 2de7de343..01b78a526 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -1,5 +1,7 @@ from pathlib import Path +import pytest + from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager @@ -37,39 +39,74 @@ def test_global_attribute(): load_global_attributes get_global_attributes """ - # Initialize CdfAttributeManager object which loads in default information + + # Initialize CdfAttributeManager object which loads in default global variables and schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - # Load in test data - cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") + # Test that default information was loaded in from "imap_default_global_cdf_attrs.yaml" + assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + assert cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" + assert cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" + assert cdf_manager.global_attributes["Mission_group"] == "IMAP>Interstellar Mapping and Acceleration Probe" + assert cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" + assert cdf_manager.global_attributes[ + "PI_affiliation"] == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540" + assert cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" + assert cdf_manager.global_attributes["Data_version"] == 1 + + # Load in different data, test what was carried over + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + assert cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" + assert cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" + assert cdf_manager.global_attributes["Mission_group"] == "Dysfunctional Cats" + assert cdf_manager.global_attributes["PI_name"] == "Ana Manica" + assert cdf_manager.global_attributes["PI_affiliation"] == "LASP, CU" + assert cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" + assert cdf_manager.global_attributes["Data_version"] == 2 + assert cdf_manager.global_attributes["Test_attribute"] == "test" # Expected failure: file_naming_convention is not in schema. # for attr in cdf_manager.global_attributes.keys(): # assert attr in cdf_manager.global_attribute_schema.keys() - assert ( - cdf_manager.global_attributes["Mission_group"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) - # Load additional global attributes - cdf_manager.load_global_attributes("imap_mag_global_cdf_attrs.yaml") + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Creating dictionary for loaded information + mag_test_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + + assert mag_test_global_attrs["Mission_group"] == "Dysfunctional Cats" + + # Testing if statement + assert mag_test_global_attrs["Descriptor"] == "TEST>Testinstrument" + assert mag_test_global_attrs["Logical_source"] == "imap_test_T1_test" + # Testing non required data from schema + assert mag_test_global_attrs["Data_type"] == "T1_test-one>Test-1 test one" + # TODO: write call that tests "Bad_name" attribute + + # Testing first elif statement + assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["Project"] + assert mag_test_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] + # TODO: check with maxine that the below code SHOULD cause an error because "Project" is not in self.global_attributes[inst_id] + # assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] + assert mag_test_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] + + for attr_name in cdf_manager.global_attributes["imap_test_T1_test"].keys(): + assert attr_name in mag_test_global_attrs.keys() + + # Testing that required schema keys are in get_global_attributes + for attr_name in cdf_manager.global_attribute_schema.keys(): + required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] + if required_schema is True: + assert attr_name in mag_test_global_attrs.keys() - assert ( - cdf_manager.global_attributes["Mission_group"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) - assert cdf_manager.global_attributes["Descriptor"] == "MAG>Magnetometer" + # BUT not everything in mag_test_global_attrs will be in cdf_manager.global_attributes["imap_test_T1_test"] - mag_l1a_global_attrs = cdf_manager.get_global_attributes("imap_mag_l1a_norm-raw") + # Testing second elif statement + # Should throw error - assert ( - mag_l1a_global_attrs["Mission_group"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) - assert mag_l1a_global_attrs["Descriptor"] == "MAG>Magnetometer" - assert mag_l1a_global_attrs["Logical_source"] == "imap_mag_l1a_norm-raw" def test_variable_attribute(): From af3f48fbad3aea297196b41d057aef3c67b1a6c8 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 09:36:49 -0600 Subject: [PATCH 05/40] Fixing test file --- .../cdf/config/imap_test_global.yaml | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/imap_processing/cdf/config/imap_test_global.yaml b/imap_processing/cdf/config/imap_test_global.yaml index e69de29bb..ee03b04ed 100644 --- a/imap_processing/cdf/config/imap_test_global.yaml +++ b/imap_processing/cdf/config/imap_test_global.yaml @@ -0,0 +1,55 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_T1_test: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. + +imap_test_section_2: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst>Level-1A raw burst rate + Logical_source: imap_mag_l1a_burst-raw + Logical_source_description: IMAP Mission MAG Burst Rate Instrument Level-1A Data. + + +imap_test_section_3: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-norm-mago>Level 1A MAGo normal rate + Logical_source: imap_mag_l1a_norm-mago + Logical_source_description: IMAP Mission MAGo Normal Rate Instrument Level-1A Data. + + +imap_test_section_4: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-norm-magi>Level 1A MAGi normal rate + Logical_source: imap_mag_l1a_norm-magi + Logical_source_description: IMAP Mission MAGi Normal Rate Instrument Level-1A Data. + + +imap_test_section_5: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst-mago>Level 1A MAGo burst rate + Logical_source: imap_mag_l1a_burst-mago + Logical_source_description: IMAP Mission MAGo Burst Rate Instrument Level-1A Data. + + +imap_test_section_7: + Descriptor: *desc + TEXT: *txt + Data_type: L1A_raw-burst-magi>Level 1A MAGi burst rate + Logical_source: imap_mag_l1a_burst-magi + Logical_source_description: IMAP Mission MAGi Burst Rate Instrument Level-1A Data. + + From 64a3672c5b1501a7a85638738248b9b11c6f2743 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 10:47:36 -0600 Subject: [PATCH 06/40] Var attribute tests --- .../cdf/config/imap_test_variable.yaml | 24 ++++++++++++ .../cdf/tests/test_cdf_attribute_manager.py | 38 ++++++++++++++++--- .../cdf/tests/test_imap_cdf_manager.py | 2 + 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 imap_processing/cdf/config/imap_test_variable.yaml diff --git a/imap_processing/cdf/config/imap_test_variable.yaml b/imap_processing/cdf/config/imap_test_variable.yaml new file mode 100644 index 000000000..ce6a8b6c3 --- /dev/null +++ b/imap_processing/cdf/config/imap_test_variable.yaml @@ -0,0 +1,24 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: test_depend + DISPLAY_TYPE: test_display_type + FILLVAL: -10 + FORMAT: I1 + VALIDMIN: 0 + VALIDMAX: 10 + VAR_TYPE: test_var_type + +test_field_1: + <<: *default + CATDESC: test time + FIELDNAM: test_field_1 + LABLAXIS: test_labaxis + UNITS: test_units + VAR_TYPE: test_variable_type + SCALETYP: test_scaletyp + MONOTON: test_monoton + TIME_BASE: 10 + TIME_SCALE: test_time_scale + REFERENCE_POSITION: test_reference_position + + # TODO: come back to format \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 01b78a526..0604fdc57 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -88,8 +88,10 @@ def test_global_attribute(): # Testing first elif statement assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["Project"] assert mag_test_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] - # TODO: check with maxine that the below code SHOULD cause an error because "Project" is not in self.global_attributes[inst_id] - # assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] + # BUT not everything in mag_test_global_attrs will be in cdf_manager.global_attributes["imap_test_T1_test"] + # TODO: check with maxine that the below code SHOULD cause an error because "Project" is not in self.global_ + # attributes[inst_id] + # assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] assert mag_test_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] for attr_name in cdf_manager.global_attributes["imap_test_T1_test"].keys(): @@ -101,15 +103,16 @@ def test_global_attribute(): if required_schema is True: assert attr_name in mag_test_global_attrs.keys() - - # BUT not everything in mag_test_global_attrs will be in cdf_manager.global_attributes["imap_test_T1_test"] - # Testing second elif statement # Should throw error - def test_variable_attribute(): + """ + Test function that covers: + load_variable_attributes + get_variable_attributes + """ cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") cdf_manager.load_variable_attributes("imap_mag_l1a_variable_attrs.yaml") @@ -129,3 +132,26 @@ def test_variable_attribute(): assert attr in variable_attrs.keys() # TODO: Call, and test get_variable_attributes + imap_test_variable = cdf_manager.get_variable_attributes("test_field_1") + + # Make sure all expected attributes are here + for variable_attrs_2 in cdf_manager.variable_attribute_schema.keys(): + required_var_attributes = cdf_manager.variable_attribute_schema[variable_attrs_2] + if required_var_attributes is True: + assert variable_attrs_2 in imap_test_variable.keys() + + # Calling required attributes + # imap_test_variable["CATDESC"] == "test time" + # imap_test_variable["DEPEND_0"] == "test_depend" + # imap_test_variable["DISPLAY_TYPE"] == "test_display_type" + # imap_test_variable["FIELDNAM"] == "test_display_type" + # imap_test_variable["FILLVAL"] == -10 + # imap_test_variable["FORMAT"] == "I1" + # imap_test_variable["LABLAXIS"] == "test_labaxis" + # imap_test_variable["UNITS"] == "test_units" + # imap_test_variable["VALIDMIN"] == 0 + # imap_test_variable["VALIDMAX"] == 10 + # imap_test_variable["VAR_TYPE"] == "test_var_type" + + + diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index e5d989496..658b38937 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -4,6 +4,8 @@ # test add_instrument_global_attrs(self, instrument: str): def test_add_instrument_global_attrs(): + pass # test add_instrument_variable_attrs(self, instrument: str, level: str): def test_add_instrument_variable_attrs(): + pass From d336f9087bed2e4fe0f3ce62f10ef9d4a2cbdcef Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 12:52:22 -0600 Subject: [PATCH 07/40] First draft --- .../imap_instrument_global_cdf_attrs.yaml | 14 +++++++ .../imap_instrument_level_variable_attrs.yaml | 23 +++++++++++ .../cdf/config/imap_test_variable.yaml | 1 + .../cdf/tests/test_cdf_attribute_manager.py | 33 ++++++++-------- .../cdf/tests/test_imap_cdf_manager.py | 38 ++++++++++++++++++- 5 files changed, 92 insertions(+), 17 deletions(-) create mode 100644 imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml create mode 100644 imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml diff --git a/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml new file mode 100644 index 000000000..0e08c76dd --- /dev/null +++ b/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml @@ -0,0 +1,14 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_T1_test: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml b/imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml new file mode 100644 index 000000000..8a243fbfc --- /dev/null +++ b/imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml @@ -0,0 +1,23 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: test_depend + DISPLAY_TYPE: test_display_type + FILLVAL: -10 + FORMAT: I1 + VALIDMIN: 0 + VALIDMAX: 10 + VAR_TYPE: test_var_type + +test_field_1: + <<: *default + CATDESC: test time + FIELDNAM: test_field_1 + LABLAXIS: test_labaxis + UNITS: test_units + VAR_TYPE: test_variable_type + SCALETYP: test_scaletyp + MONOTON: test_monoton + TIME_BASE: 10 + TIME_SCALE: test_time_scale + REFERENCE_POSITION: test_reference_position + NOT_REQUIRED: test_not_required \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_test_variable.yaml b/imap_processing/cdf/config/imap_test_variable.yaml index ce6a8b6c3..b2d5fd3af 100644 --- a/imap_processing/cdf/config/imap_test_variable.yaml +++ b/imap_processing/cdf/config/imap_test_variable.yaml @@ -20,5 +20,6 @@ test_field_1: TIME_BASE: 10 TIME_SCALE: test_time_scale REFERENCE_POSITION: test_reference_position + NOT_REQUIRED: test_not_required # TODO: come back to format \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 0604fdc57..c19f648e0 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -42,7 +42,6 @@ def test_global_attribute(): # Initialize CdfAttributeManager object which loads in default global variables and schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - # Test that default information was loaded in from "imap_default_global_cdf_attrs.yaml" assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" @@ -141,17 +140,21 @@ def test_variable_attribute(): assert variable_attrs_2 in imap_test_variable.keys() # Calling required attributes - # imap_test_variable["CATDESC"] == "test time" - # imap_test_variable["DEPEND_0"] == "test_depend" - # imap_test_variable["DISPLAY_TYPE"] == "test_display_type" - # imap_test_variable["FIELDNAM"] == "test_display_type" - # imap_test_variable["FILLVAL"] == -10 - # imap_test_variable["FORMAT"] == "I1" - # imap_test_variable["LABLAXIS"] == "test_labaxis" - # imap_test_variable["UNITS"] == "test_units" - # imap_test_variable["VALIDMIN"] == 0 - # imap_test_variable["VALIDMAX"] == 10 - # imap_test_variable["VAR_TYPE"] == "test_var_type" - - - + # assert imap_test_variable["CATDESC"] == "test time" + # assert imap_test_variable["DEPEND_0"] == "test_depend" + # assert imap_test_variable["DISPLAY_TYPE"] == "test_display_type" + # assert imap_test_variable["FIELDNAM"] == "test_display_type" + # assert imap_test_variable["FILLVAL"] == -10 + # assert imap_test_variable["FORMAT"] == "I1" + # assert imap_test_variable["LABLAXIS"] == "test_labaxis" + # assert imap_test_variable["UNITS"] == "test_units" + # assert imap_test_variable["VALIDMIN"] == 0 + # assert imap_test_variable["VALIDMAX"] == 10 + # assert imap_test_variable["VAR_TYPE"] == "test_var_type" + + # Calling to non required attributes + # assert imap_test_variable["NOT_REQUIRED"] == "test_not_required" + + # Calling attribute name that does not exist + # TODO: should throw error + # assert imap_test_variable["DNE"] == "test time" diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 658b38937..d09a0dfb7 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -4,8 +4,42 @@ # test add_instrument_global_attrs(self, instrument: str): def test_add_instrument_global_attrs(): - pass + # Create an ImapCdfAttributes object + imap_cdf_manager = ImapCdfAttributes() + # Call function + imap_cdf_manager.add_instrument_global_attrs("instrument") + + # Testing the actual function + imap_instrument = imap_cdf_manager.get_global_attributes("imap_test_T1_test") + assert imap_instrument["Data_type"] == "T1_test-one>Test-1 test one" + assert imap_instrument["Project"] == "STP>Solar-Terrestrial Physics" + + # Testing reloading data + imap_cdf_manager.add_instrument_global_attrs("swe") + + #Testing again + swe_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci") + assert swe_instrument["Data_type"] == "L1A_SCI>Level-1A Science data" + assert swe_instrument["Project"] == "STP>Solar-Terrestrial Physics" # test add_instrument_variable_attrs(self, instrument: str, level: str): def test_add_instrument_variable_attrs(): - pass + # Create an ImapCdfAttributes object + imap_cdf_manager = ImapCdfAttributes() + # Call function + imap_cdf_manager.add_instrument_variable_attrs("instrument", "level") + + # Testing the actual function + imap_instrument = imap_cdf_manager.get_variable_attributes("imap_test_T1_test") + # assert imap_instrument["DEPEND_0"] == "test_depend" + # assert imap_instrument["CATDESC"] == "test time" + # assert imap_instrument["VALIDMAX"] == 10 + + # Testing reloading data + imap_cdf_manager.add_instrument_variable_attrs("ultra", "l1b") + + # Testing again + ultra_instrument = imap_cdf_manager.get_variable_attributes("imap_swe_l1a_sci") + # assert ultra_instrument["DEPEND_0"] == "epoch" + # assert ultra_instrument["CATDESC"] == "Time, number of nanoseconds since J2000 with leap seconds included" + # assert ultra_instrument["VALIDMAX"] == 9223372036854775807 From 6258bcac68a8e4e940e60f8b137003c512d1a30c Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 13:18:49 -0600 Subject: [PATCH 08/40] Deleting file --- imap_processing/cdf/tests/imap_test_variable.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 imap_processing/cdf/tests/imap_test_variable.yaml diff --git a/imap_processing/cdf/tests/imap_test_variable.yaml b/imap_processing/cdf/tests/imap_test_variable.yaml deleted file mode 100644 index e69de29bb..000000000 From 34c7a98827f4914a748447e15c6a6df0f2b2e89a Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 13:24:05 -0600 Subject: [PATCH 09/40] Test --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index d09a0dfb7..52b30ce8d 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -3,6 +3,7 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes # test add_instrument_global_attrs(self, instrument: str): +# Test def test_add_instrument_global_attrs(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes() From fbceece0fc8b3deccb7908bcc8746bb70079ac34 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 13:43:47 -0600 Subject: [PATCH 10/40] Fixing pulled errors --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index c19f648e0..23d645c3f 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -51,7 +51,6 @@ def test_global_attribute(): assert cdf_manager.global_attributes[ "PI_affiliation"] == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540" assert cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" - assert cdf_manager.global_attributes["Data_version"] == 1 # Load in different data, test what was carried over cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") @@ -114,7 +113,7 @@ def test_variable_attribute(): """ cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") - cdf_manager.load_variable_attributes("imap_mag_l1a_variable_attrs.yaml") + cdf_manager.load_variable_attributes("imap_mag_l1_variable_attrs.yaml") # All variables required to have: expected_attributes = [ From 5348fb7cc9fd2a8094c2e562155a5c913cb32d5b Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 14:38:29 -0600 Subject: [PATCH 11/40] Quick --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 23d645c3f..a29ccfa40 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -113,7 +113,7 @@ def test_variable_attribute(): """ cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") - cdf_manager.load_variable_attributes("imap_mag_l1_variable_attrs.yaml") + cdf_manager.load_variable_attributes("imap_test_variable.yaml") # All variables required to have: expected_attributes = [ From 8dc2b82b68524033e11416d47acfe6b70d23061f Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 20:17:26 -0600 Subject: [PATCH 12/40] Trying different things to pass pre-checks --- .../cdf/tests/test_cdf_attribute_manager.py | 2 +- imap_processing/cdf/tests/test_imap_cdf_manager.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index a29ccfa40..dfdd43fb4 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -1,6 +1,6 @@ from pathlib import Path -import pytest +# import pytest from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 52b30ce8d..dbd08622f 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -1,9 +1,10 @@ -from pathlib import Path -from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager +# from pathlib import Path +# from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes # test add_instrument_global_attrs(self, instrument: str): -# Test + + def test_add_instrument_global_attrs(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes() @@ -18,11 +19,12 @@ def test_add_instrument_global_attrs(): # Testing reloading data imap_cdf_manager.add_instrument_global_attrs("swe") - #Testing again + # Testing again swe_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci") assert swe_instrument["Data_type"] == "L1A_SCI>Level-1A Science data" assert swe_instrument["Project"] == "STP>Solar-Terrestrial Physics" + # test add_instrument_variable_attrs(self, instrument: str, level: str): def test_add_instrument_variable_attrs(): # Create an ImapCdfAttributes object From 1d56ef91a4fb00618cf4e04bf4e0562306cfae48 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 20:37:18 -0600 Subject: [PATCH 13/40] Testing --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index dbd08622f..1b29ccecb 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -1,5 +1,6 @@ # from pathlib import Path # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager +# This is a test from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes # test add_instrument_global_attrs(self, instrument: str): From 9e14421e502e911d8949a6f65024d2cb15e9b444 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 20:45:47 -0600 Subject: [PATCH 14/40] TEST --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index dfdd43fb4..f775d5bf0 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -12,6 +12,8 @@ def test_default_attr_schema(): _load_default_variable_attr_schema """ + # Test + # Initialize CdfAttributeManager object which loads in default schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") From b1c5209562b2e17cd16a3466abeed81583f31208 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 17 Jun 2024 20:47:53 -0600 Subject: [PATCH 15/40] Fixing failed pre-check --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 1b29ccecb..a8a89f4ca 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -34,7 +34,7 @@ def test_add_instrument_variable_attrs(): imap_cdf_manager.add_instrument_variable_attrs("instrument", "level") # Testing the actual function - imap_instrument = imap_cdf_manager.get_variable_attributes("imap_test_T1_test") + # imap_instrument = imap_cdf_manager.get_variable_attributes("imap_test_T1_test") # assert imap_instrument["DEPEND_0"] == "test_depend" # assert imap_instrument["CATDESC"] == "test time" # assert imap_instrument["VALIDMAX"] == 10 @@ -43,7 +43,7 @@ def test_add_instrument_variable_attrs(): imap_cdf_manager.add_instrument_variable_attrs("ultra", "l1b") # Testing again - ultra_instrument = imap_cdf_manager.get_variable_attributes("imap_swe_l1a_sci") + # ultra_instrument = imap_cdf_manager.get_variable_attributes("imap_swe_l1a_sci") # assert ultra_instrument["DEPEND_0"] == "epoch" # assert ultra_instrument["CATDESC"] == "Time, number of nanoseconds since J2000 with leap seconds included" # assert ultra_instrument["VALIDMAX"] == 9223372036854775807 From 24239217ef16658f9ca925dc2dee12e03e021d36 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 08:24:10 -0600 Subject: [PATCH 16/40] Test2 --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index a8a89f4ca..a3d484fc7 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -1,6 +1,6 @@ # from pathlib import Path # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager -# This is a test +# This is a test2 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes # test add_instrument_global_attrs(self, instrument: str): From 4e1990f40ee481f88ef3845beaa42dfe859c43fd Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 08:35:35 -0600 Subject: [PATCH 17/40] Fixing pre-commit issues --- .../imap_instrument_global_cdf_attrs.yaml | 2 +- .../cdf/config/imap_test_global.yaml | 2 +- .../cdf/tests/test_cdf_attribute_manager.py | 76 ++++++++++++++----- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml index 0e08c76dd..a5e6c5ef1 100644 --- a/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml @@ -3,7 +3,7 @@ Descriptor: &desc TEXT: &txt > This file is nothing more than a test yamal. This description? This is a test description. This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. - This test file is led by me, Ana Manica. Lasp undergraduate employee. + This test file is led by me, Ana Manica. Lasp undergraduate employee. I don't have a website. Sorry. imap_test_T1_test: diff --git a/imap_processing/cdf/config/imap_test_global.yaml b/imap_processing/cdf/config/imap_test_global.yaml index ee03b04ed..78d2d5874 100644 --- a/imap_processing/cdf/config/imap_test_global.yaml +++ b/imap_processing/cdf/config/imap_test_global.yaml @@ -3,7 +3,7 @@ Descriptor: &desc TEXT: &txt > This file is nothing more than a test yamal. This description? This is a test description. This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. - This test file is led by me, Ana Manica. Lasp undergraduate employee. + This test file is led by me, Ana Manica. Lasp undergraduate employee. I don't have a website. Sorry. imap_test_T1_test: diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index f775d5bf0..8ca432d3f 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -1,7 +1,6 @@ from pathlib import Path # import pytest - from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager @@ -26,10 +25,18 @@ def test_default_attr_schema(): # Default variable tests # Check false case - assert cdf_manager.variable_attribute_schema['attribute_key']["ABSOLUTE_ERROR"]["required"] is False + assert ( + cdf_manager.variable_attribute_schema["attribute_key"]["ABSOLUTE_ERROR"][ + "required" + ] + is False + ) # Check true case - assert cdf_manager.variable_attribute_schema['attribute_key']["RESOLUTION"]["required"] is True + assert ( + cdf_manager.variable_attribute_schema["attribute_key"]["RESOLUTION"]["required"] + is True + ) # @pytest.mark.xfail(reason="Missing IMAP specific global schema") @@ -46,23 +53,46 @@ def test_global_attribute(): cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") # Test that default information was loaded in from "imap_default_global_cdf_attrs.yaml" assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" - assert cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" - assert cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" - assert cdf_manager.global_attributes["Mission_group"] == "IMAP>Interstellar Mapping and Acceleration Probe" + assert ( + cdf_manager.global_attributes["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert ( + cdf_manager.global_attributes["Discipline"] + == "Solar Physics>Heliospheric Physics" + ) + assert ( + cdf_manager.global_attributes["Mission_group"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) assert cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" - assert cdf_manager.global_attributes[ - "PI_affiliation"] == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540" - assert cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" + assert ( + cdf_manager.global_attributes["PI_affiliation"] + == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540" + ) + assert ( + cdf_manager.global_attributes["File_naming_convention"] + == "source_descriptor_datatype_yyyyMMdd_vNNN" + ) # Load in different data, test what was carried over cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" - assert cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" - assert cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" + assert ( + cdf_manager.global_attributes["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert ( + cdf_manager.global_attributes["Discipline"] + == "Solar Physics>Heliospheric Physics" + ) assert cdf_manager.global_attributes["Mission_group"] == "Dysfunctional Cats" assert cdf_manager.global_attributes["PI_name"] == "Ana Manica" assert cdf_manager.global_attributes["PI_affiliation"] == "LASP, CU" - assert cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" + assert ( + cdf_manager.global_attributes["File_naming_convention"] + == "source_descriptor_datatype_yyyyMMdd_vNNN" + ) assert cdf_manager.global_attributes["Data_version"] == 2 assert cdf_manager.global_attributes["Test_attribute"] == "test" @@ -87,12 +117,18 @@ def test_global_attribute(): # Testing first elif statement assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["Project"] - assert mag_test_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] + assert ( + mag_test_global_attrs["Source_name"] + == cdf_manager.global_attributes["Source_name"] + ) # BUT not everything in mag_test_global_attrs will be in cdf_manager.global_attributes["imap_test_T1_test"] # TODO: check with maxine that the below code SHOULD cause an error because "Project" is not in self.global_ # attributes[inst_id] # assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] - assert mag_test_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] + assert ( + mag_test_global_attrs["Data_type"] + == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] + ) for attr_name in cdf_manager.global_attributes["imap_test_T1_test"].keys(): assert attr_name in mag_test_global_attrs.keys() @@ -109,10 +145,10 @@ def test_global_attribute(): def test_variable_attribute(): """ - Test function that covers: - load_variable_attributes - get_variable_attributes - """ + Test function that covers: + load_variable_attributes + get_variable_attributes + """ cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") cdf_manager.load_variable_attributes("imap_test_variable.yaml") @@ -136,7 +172,9 @@ def test_variable_attribute(): # Make sure all expected attributes are here for variable_attrs_2 in cdf_manager.variable_attribute_schema.keys(): - required_var_attributes = cdf_manager.variable_attribute_schema[variable_attrs_2] + required_var_attributes = cdf_manager.variable_attribute_schema[ + variable_attrs_2 + ] if required_var_attributes is True: assert variable_attrs_2 in imap_test_variable.keys() From 79f3ba98fa3295d5c39d0b85f1c0894ec61785ee Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 08:44:00 -0600 Subject: [PATCH 18/40] Fixing prechecks --- .../cdf/tests/test_cdf_attribute_manager.py | 21 ++++++++++++------- .../cdf/tests/test_imap_cdf_manager.py | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 8ca432d3f..87a7403e0 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -49,9 +49,11 @@ def test_global_attribute(): get_global_attributes """ - # Initialize CdfAttributeManager object which loads in default global variables and schema + # Initialize CdfAttributeManager object which loads in default global + # variables and schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - # Test that default information was loaded in from "imap_default_global_cdf_attrs.yaml" + # Test that default information was loaded in from + # "imap_default_global_cdf_attrs.yaml" assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( cdf_manager.global_attributes["Source_name"] @@ -68,7 +70,8 @@ def test_global_attribute(): assert cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" assert ( cdf_manager.global_attributes["PI_affiliation"] - == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540" + == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton," + "NJ 08540" ) assert ( cdf_manager.global_attributes["File_naming_convention"] @@ -121,10 +124,12 @@ def test_global_attribute(): mag_test_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] ) - # BUT not everything in mag_test_global_attrs will be in cdf_manager.global_attributes["imap_test_T1_test"] - # TODO: check with maxine that the below code SHOULD cause an error because "Project" is not in self.global_ - # attributes[inst_id] - # assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] + # BUT not everything in mag_test_global_attrs will be in + # cdf_manager.global_attributes["imap_test_T1_test"] + # TODO: check with maxine that the below code SHOULD cause an error because + # "Project" is not in self.global_attributes[inst_id] + # assert mag_test_global_attrs["Project"] + # == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] assert ( mag_test_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] @@ -196,4 +201,4 @@ def test_variable_attribute(): # Calling attribute name that does not exist # TODO: should throw error - # assert imap_test_variable["DNE"] == "test time" + # assert imap_test_variable["DOES_NOT_EXIST"] == "test time" diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index a3d484fc7..fd01f1db1 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -45,5 +45,6 @@ def test_add_instrument_variable_attrs(): # Testing again # ultra_instrument = imap_cdf_manager.get_variable_attributes("imap_swe_l1a_sci") # assert ultra_instrument["DEPEND_0"] == "epoch" - # assert ultra_instrument["CATDESC"] == "Time, number of nanoseconds since J2000 with leap seconds included" + # assert ultra_instrument["CATDESC"] == "Time, number of nanoseconds + # since J2000 with leap seconds included" # assert ultra_instrument["VALIDMAX"] == 9223372036854775807 From a6426c06503e0032b97f38478ab4986195afaeb6 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 08:53:02 -0600 Subject: [PATCH 19/40] Removing test statements --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index fd01f1db1..10e5dfd94 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -1,6 +1,5 @@ # from pathlib import Path # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager -# This is a test2 from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes # test add_instrument_global_attrs(self, instrument: str): From 6936749b1b0f99379a5b11cab0e1f810777fb8e5 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 09:37:07 -0600 Subject: [PATCH 20/40] Moving files --- .../cdf/config/imap_test_global.yaml | 55 --- .../imap_default_global_test_cdf_attrs.yaml | 0 .../imap_instrument_global_cdf_attrs.yaml | 0 .../imap_instrument_level_variable_attrs.yaml | 0 .../cdf/tests/imap_test_global.yaml | 14 + .../{config => tests}/imap_test_variable.yaml | 0 .../default_global_cdf_attrs_schema_test.yaml | 246 +++++++++ ...efault_variable_cdf_attrs_schema_test.yaml | 466 ++++++++++++++++++ .../cdf/tests/test_cdf_attribute_manager.py | 10 +- 9 files changed, 732 insertions(+), 59 deletions(-) delete mode 100644 imap_processing/cdf/config/imap_test_global.yaml rename imap_processing/cdf/{config => tests}/imap_default_global_test_cdf_attrs.yaml (100%) rename imap_processing/cdf/{config => tests}/imap_instrument_global_cdf_attrs.yaml (100%) rename imap_processing/cdf/{config => tests}/imap_instrument_level_variable_attrs.yaml (100%) create mode 100644 imap_processing/cdf/tests/imap_test_global.yaml rename imap_processing/cdf/{config => tests}/imap_test_variable.yaml (100%) create mode 100644 imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml create mode 100644 imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml diff --git a/imap_processing/cdf/config/imap_test_global.yaml b/imap_processing/cdf/config/imap_test_global.yaml deleted file mode 100644 index 78d2d5874..000000000 --- a/imap_processing/cdf/config/imap_test_global.yaml +++ /dev/null @@ -1,55 +0,0 @@ -Descriptor: &desc - TEST>Testinstrument -TEXT: &txt > - This file is nothing more than a test yamal. This description? This is a test description. - This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. - This test file is led by me, Ana Manica. Lasp undergraduate employee. - I don't have a website. Sorry. - -imap_test_T1_test: - Descriptor: *desc - TEXT: *txt - Data_type: T1_test-one>Test-1 test one - Logical_source: imap_test_T1_test - Logical_source_description: IMAP Mission TEST one document Level-T1. - -imap_test_section_2: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst>Level-1A raw burst rate - Logical_source: imap_mag_l1a_burst-raw - Logical_source_description: IMAP Mission MAG Burst Rate Instrument Level-1A Data. - - -imap_test_section_3: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-norm-mago>Level 1A MAGo normal rate - Logical_source: imap_mag_l1a_norm-mago - Logical_source_description: IMAP Mission MAGo Normal Rate Instrument Level-1A Data. - - -imap_test_section_4: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-norm-magi>Level 1A MAGi normal rate - Logical_source: imap_mag_l1a_norm-magi - Logical_source_description: IMAP Mission MAGi Normal Rate Instrument Level-1A Data. - - -imap_test_section_5: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst-mago>Level 1A MAGo burst rate - Logical_source: imap_mag_l1a_burst-mago - Logical_source_description: IMAP Mission MAGo Burst Rate Instrument Level-1A Data. - - -imap_test_section_7: - Descriptor: *desc - TEXT: *txt - Data_type: L1A_raw-burst-magi>Level 1A MAGi burst rate - Logical_source: imap_mag_l1a_burst-magi - Logical_source_description: IMAP Mission MAGi Burst Rate Instrument Level-1A Data. - - diff --git a/imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml similarity index 100% rename from imap_processing/cdf/config/imap_default_global_test_cdf_attrs.yaml rename to imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml diff --git a/imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument_global_cdf_attrs.yaml similarity index 100% rename from imap_processing/cdf/config/imap_instrument_global_cdf_attrs.yaml rename to imap_processing/cdf/tests/imap_instrument_global_cdf_attrs.yaml diff --git a/imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument_level_variable_attrs.yaml similarity index 100% rename from imap_processing/cdf/config/imap_instrument_level_variable_attrs.yaml rename to imap_processing/cdf/tests/imap_instrument_level_variable_attrs.yaml diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml new file mode 100644 index 000000000..a5e6c5ef1 --- /dev/null +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -0,0 +1,14 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_T1_test: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_test_variable.yaml b/imap_processing/cdf/tests/imap_test_variable.yaml similarity index 100% rename from imap_processing/cdf/config/imap_test_variable.yaml rename to imap_processing/cdf/tests/imap_test_variable.yaml diff --git a/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml new file mode 100644 index 000000000..802c305fc --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml @@ -0,0 +1,246 @@ +DOI: + description: > + DOI is a persistent Unique Digital Identifier with the form + https://doi.org// with the identifying the DOI + registration authority and the identifying the dataset. The DOI should point to + a landing page for additional information about the dataset. DOIs are typically created by + the SPASE naming authority or archive. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true # include in validation + overwrite: false +Data_level: + description: > + This attribute is used in file name creation and records the level of processsing done + on the dataset. For HERMES the following are valid values: + - l0>Level 0 + - l1>Level 1 + - l2>Level 2 + - l3>Level 3 + - l4>Level 4 + - ql>Quicklook + default: null + required: true # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_product_descriptor: + description: > + This is an optional field that may not be needed for all products. Where it is used, identifier + should be short (e.q. 3-8 characters) descriptors that are helpful to end- users. If a + descriptor contains multiple components, underscores are used to separate those components. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_type: + description: > + This attribute is used by CDF file writing software to create a filename. It is a + combination of the following filename components: mode, data level, and optional data + product descriptor. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_version: + description: > + This attribute identifies the version of a particular CDF data file. + default: null + required: true + validate: true + overwrite: false +Descriptor: + description: > + This attribute identifies the name of the instrument or sensor that collected the data. Both + a long name and a short name are given. For any data file, only a single value is allowed. + For HERMES, the following are valid values: + - EEA>Electron Electrostatic Analyzer + - MERIT>Miniaturized Electron pRoton Telescope + - NEMISIS> Noise Eliminating Magnetometer In a Small Integrated System + - SPAN-I>Solar Probe Analyzer for Ions + default: null + required: true + validate: true + overwrite: false +Discipline: + description: > + This attribute describes both the science discipline and sub discipline. For HERMES, + this value should always be "Space Physics>Magnetospheric Science." + default: Space Physics>Magnetospheric Science + required: true + validate: true + overwrite: false +File_naming_convention: + description: > + If File_naming_convention was not set, it uses default setting: + source_datatype_descriptor_yyyyMMdd + default: source_datatype_descriptor_yyyyMMdd + required: false + validate: false + overwrite: true +Generation_date: + description: > + Date stamps the creation of the file using the syntax yyyymmdd, e.g., " + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: true +HTTP_LINK: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Instrument_mode: + description: > + TBS + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: false +Instrument_type: + description: > + This attribute is used to facilitate making choices of instrument type. More than one entry + is allowed. Acceptable values for HERMES include: + - Magnetic Fields (space) + - Particles (space) + - Plasma and Solar Wind + - Ephemeris -> Ephemeris/Attitude/Ancillary + default: null + required: true + validate: true + overwrite: false +LINK_TEXT: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +LINK_TITLE: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Logical_file_id: + description: > + This attribute stores the name of the CDF file but without the + file extension (e.g. ".cdf"). This attribute is required to avoid loss of the original source + in the case of accidental (or intentional) renaming. + default: null + required: true + validate: true + overwrite: true +Logical_source: + description: > + This attribute determines the file naming convention in the SKT Editor and is used by + CDA Web. It is composed of the following values: + - source_name - (e.g. spacecraft identifier) + - descriptor - (e.g. instrument identifier - see Section Error! Reference source not + found.) + - data_type - (e.g. mode, data level, and optional data product descriptor - value + come from 'Data_type' attribute) + default: null + required: true + validate: true + overwrite: true +Logical_source_description: + description: > + This attribute writes out the full words associated with the encrypted Logical_source + above, e.g., "Level 1 Dual Electron Spectrometer Survey Data". Users on CDAWeb see + this value on their website. + default: null + required: true + validate: true + overwrite: true +MODS: + description: > + This attribute is an SPDF standard global attribute, which is used to denote the history of + modifications made to the CDF data set. The MODS attribute should contain a + description of all significant changes to the data set, essentially capturing a log of high- + level release notes. This attribute can have as many entries as necessary and should be + updated if the Interface Number ("X") of the version number changes. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Mission_group: + description: > + This attribute has a single value and is used to facilitate making choices of source through + CDAWeb. This value should be "HERMES." + default: HERMES + required: true + validate: true + overwrite: false +PI_affiliation: + description: > + This attribute value should include the HERMES mission PI affiliation followed by a + comma-separated list of any Co-I affiliations that are responsible for this particular + dataset. The following are valid HERMES values, of which the abbreviations should be + used exclusively within this attribute value, and the full text of the affiliation included in + the general 'text' attribute as it is used solely in plot labels. + - GSFC - Goddard Space Flight Center + - UCB - University of California, Berkeley + - SSL - Space Sciences Laboratory, UCB + - UM - University of Michigan + default: null + required: true + validate: true + overwrite: false +PI_name: + description: > + This attribute value should include first initial and last name of the HERMES mission PI + followed by a comma-separated list of any Co-Is that are responsible for this particular + dataset. + default: null + required: true + validate: true + overwrite: false +Project: + description: > + This attribute identifies the name of the project and indicates ownership. For HERMES, + this value should be "STP>Solar-Terrestrial Physics". + default: STP>Solar-Terrestrial Physics + required: true + validate: true + overwrite: false +Source_name: + description: > + This attribute identifies the observatory where the data originated. The following are + valid values for HERMES: + - HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + default: HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + required: true + validate: true + overwrite: false +Start_time: + description: > + The start time of the contained data given in YYYYMMDD_hhmmss + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +TEXT: + description: > + This attribute is an SPDF standard global attribute, which is a text description of the + experiment whose data is included in the CDF. A reference to a journal article(s) or to a + World Wide Web page describing the experiment is essential and constitutes the + minimum requirement. A written description of the data set is also desirable. This + attribute can have as many entries as necessary to contain the desired information. + Typically, this attribute is about a paragraph in length and is not shown on CDAWeb. + CDAWeb is the web portal for access to SPDF data, available at https://cdaweb.gsfc.nasa.gov. + default: null + required: true + validate: true + overwrite: false diff --git a/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml new file mode 100644 index 000000000..4adea4723 --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml @@ -0,0 +1,466 @@ +# Based on : https://spdf.gsfc.nasa.gov/istp_guide/vattributes.html +# HERMES CDF Format Guide +attribute_key: + # ===== EPOCH-ONLY VARIABLE ATTRIBUTES ===== + TIME_BASE: + description: > + fixed (0AD, 1900, 1970 (POSIX), J2000 (used by CDF_TIME_TT2000), + 4714 BC (Julian)) or flexible (provider-defined) + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RESOLUTION: + description: > + Using ISO8601 relative time format, for example: "1s" = 1 second. + Resolution provides the smallest change in time that is measured. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + TIME_SCALE: + description: > + TT (same as TDT, used by CDF_TIME_TT2000), TAI (same as IAT, TT-32.184s), + UTC (includes leap seconds), TDB (same as SPICE ET), EME1950 [default: UTC] + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + REFERENCE_POSITION: + description: > + Topocenter (local), Geocenter , rotating Earth geoid (used by CDF_TIME_TT2000). + Reference_Position is optional metadata to account for time variance with position in + the gravity wells and with relative velocity. While we could use a combined + TimeSystem attribute that defines mission-specific time scales where needed, such as + UTC-at-STEREO-B, it's cleaner to keep them separate as Time_Scale=UTC and + Reference_Position=STEREO-B. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + LEAP_SECONDS_INCLUDED: + description: > + comma-delimited list (within brackets) of leap seconds included in the form of a lists + of ISO8601 times when each leap second was added, appended with the size of the leap + second in ISO8601 relative time (+/- time, most commonly: "+1s") [default: standard + list of leap seconds up to time of data]. Leap_Seconds_Included is needed to account + for time scales that don't have all 34 (in 2009) leap seconds and for the clocks in + various countries that started using leap seconds at different times. The full list is + required to handle the equally or more common case where a time scale starts at a + pecific UTC but continues on without leap seconds in TAI mode; this is basically what + missions that don't add leap seconds are doing. + $ cat tai-utc.dat | awk 'ORS="," { val = $7 - prev } {prev = $7} { print $1$2"01+" val "s" }' + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + ABSOLUTE_ERROR: + description: > + Absolute or systematic error, in same units as Units attribute. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RELATIVE_ERROR: + description: > + Relative or random error, in same units as Units attribute - to specify the accuracy + of the time stamps relative to each other. This is usually much smaller than Absolute_Error. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + BIN_LOCATION: + description: > + relative position of time stamp to the data measurement bin, with 0.0 at the beginning + of time bin and 1.0 at the end. Default is 0.5 for the time at the center of the data + measurement. Since clock readings are usually truncated, the real value may be closer to 0.0. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + # ===== DATA VARIABLE ATTRIBUTES ===== + CATDESC: + description: > + This is a human readable description of the data variable. Generally, this is an 80- + character string which describes the variable and what it depends on. + required: true + overwrite: false + valid_values: null + alternate: null + DELTA_MINUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DELTA_PLUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DEPEND_0: + description: > + Explicitly ties a data variable to the time variable on which it depends. All variables + which change with time must have a DEPEND_0 attribute defined. See section 5.2.1 + which specifies the HERMES usage of DEPEND_0. + required: true + overwrite: false + valid_values: null + alternate: null + DEPEND_i: # For i'th dimensional data variables + description: > + Ties a dimensional data variable to a SUPPORT_DATA variable on which the i-th + dimension of the data variable depends. The number of DEPEND attributes must match + the dimensionality of the variable, i.e., a one-dimensional variable must have a + DEPEND_1, a two-dimensional variable must have a DEPEND_1 and a DEPEND_2 + attribute, etc. The value of the attribute must be a variable in the same CDF data set. It is + strongly recommended that DEPEND_i variables hold values in physical units. + DEPEND_i variables also require their own attributes, as described in section 5.1.4. + required: false + overwrite: false + valid_values: null + alternate: null + DISPLAY_TYPE: + description: > + This tells automated software, such as CDAWeb, how the data should be displayed. + required: true + overwrite: false + valid_values: + time_series + time_series>noerrorbars + spectrogram + stack_plot + image + alternate: null + FIELDNAM: + description: > + A shortened version of CATDESC which can be used to label a plot axis or as a data + listing heading. This is a string, up to ~30 characters in length. + required: true + overwrite: false + valid_values: null + alternate: null + FILLVAL: + description: > + Identifies the fill value used where data values are known to be bad or missing. + FILLVAL is required for time-varying variables. Fill data are always non-valid data. The + ISTP standard fill values are listed in Table 5-4. + required: true + overwrite: false + valid_values: null + alternate: null + FORMAT: # NOTE Only one of FORMAT or FORM_PTR should be present + description: > + This field allows software to properly format the associated data when displayed on a + screen or output to a file. Format can be specified using either Fortran or C format codes. + For instance, "F10.3" indicates that the data should be displayed across 10 characters + where 3 of those characters are to the right of the decimal. For a description of FORTRAN + formatting codes see the docs here: + https://docs.oracle.com/cd/E19957-01/805-4939/z40007437a2e/index.html + required: true + overwrite: false + valid_values: null + alternate: FORM_PTR + FORM_PTR: + description: > + The value of this field is a variable which stores the character string that represents the + desired output format for the associated data. + required: false + overwrite: false + valid_values: null + alternate: FORMAT + LABLAXIS: # NOTE Only one of LABLAXIS or LABL_PTR_i should be present + description: > + Used to label a plot axis or to provide a heading for a data listing. This field is generally + 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: true + overwrite: false + valid_values: null + alternate: LABL_PTR_1 + LABL_PTR_i: + description: > + Used to label a dimensional variable when one value of LABLAXIS is not sufficient to + describe the variable or to label all the axes. LABL_PTR_i is used instead of + LABLAXIS, where i can take on any value from 1 to n where n is the total number of + dimensions of the original variable. The value of LABL_PTR_1 is a variable which will + contain the short character strings which describe the first dimension of the original + variable. The value of the attribute must be a variable in the same CDF data set and is + generally 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: false + overwrite: false + valid_values: null + alternate: LABLAXIS + SI_CONVERSION: + description: > + The conversion factor to SI units. This is the factor that the variable must be multiplied + by in order to convert it to generic SI units. This parameter contains two text fields + separated by the ">" delimiter. The first component is the conversion factor and the + second is the standard SI unit. Units are defined according to their standard SI symbols + (ie. Tesla = T, Newtons = N, Meters = m, etc.) For data variables that are inherently + unitless, and thus lack a conversion factor, this data attribute will be " > " where ' ' is + a blank space and the quotation marks are not included. Units which are not conveniently + transformed into SI should follow the blank syntax " > " described above. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + UNITS: # NOTE Only one of UNITS or UNIT_PTR should be present + description: > + A 6-20 character string that identifies the units of the variable (e.g. nT for magnetic + field). Use a blank character, rather than "None" or "unitless", for variables that have no + units (e.g., a ratio or a direction cosine). An active list of HERMES standard UNITS and + their SI_CONVERSIONs is maintained on the mission web-pages at + https://lasp.colorado.edu/galaxy/display/HERMES/Units+of+Measure, accessible via the + HERMES Science Working Team pages. Those pages also lay out the rules for + formatting the UNITS string. + required: true + overwrite: false + valid_values: null + alternate: UNIT_PTR + UNIT_PTR: + description: > + The value of this field is a variable which stores short character strings which identify the + units of the variable. Use a blank character, rather than "None" or "unitless", for + variables that have no units (e.g., a ratio or a direction cosine). The value of this attribute + must be a variable in the same CDF data set. + required: false + overwrite: false + valid_values: null + alternate: UNITS + VALIDMIN: + description: > + The minimum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VALIDMAX: + description: > + The maximum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VAR_TYPE: + description: > + Used in CDAWeb to indicate if the data should be used directly by users. + required: true + overwrite: false + valid_values: + data + support_data + metadata + ignore_data + alternate: null + COORDINATE_SYSTEM: + description: > + All variables for which the values are dependent on the system of coordinates are + strongly recommended to have this attribute. This includes both full vectors, tensors, etc. + or individual values, e.g. of an angle with respect to some axis. The attribute is a text + string which takes the form: "XXX[>optional long name]" + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TENSOR_ORDER: + description: > + All variables which hold physical vectors, tensors, etc., or sub-parts thereof, are strongly + recommended to have their tensorial properties held by this numerical value. Vectors + have TENSOR_ORDER=1, pressure tensors have TENSOR_ORDER=2, etc. Variables + which hold single components or sub-parts of a vector or tensor, e.g., the x-component of + velocity or the three diagonal elements of a tensor, use this attribute to establish the + underlying object from which they are extracted. TENSOR_ORDER is a number, usually + held as a CDF_INT4, rather than a character string. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + REPRESENTATION_i: + description: > + This strongly recommended attribute holds the way vector or tensor variables are held, + e.g., as Cartesian or polar forms, and their sequence order in the dimension i in which + they are held. Cartesians are indicated by x,y,z; polar coordinates by r (magnitude), t + (theta - from z-axis), p (phi - longitude or azimuth around z-axis from x axis), l (lambda + = latitude). Examples follow. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + OPERATOR_TYPE: + description: > + This has been introduced to describe HERMES quaternions (see Section 5.2 below). It + has allowed values "UNIT_QUATERNION" or "ROTATION_MATRIX" although other + values could be added. Unit quaternions correspond to pure spatial rotations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + WCSAXES: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. + The value field shall contain a non-negative integer no + greater than 999, representing the number of axes in the associated + data array. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + MJDREF: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the reference time position of the time stamps along the + 0'th axis of the measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEUNIT: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a character + string giving the units of the time stamps along the 0'th axis of the measurement. + The TIMEUNIT should match the CUNITi along the time axis of the measurement + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEDEL: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the resolution of the time stamps along the 0'th axis of + the measurement. The TIMEDEL should match the CRDELi along the time axis of the + measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CNAMEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall contain a charachter string represnting the name of the i'th axis. + The name is used for comment/documentation purposes only and is not used as a part + of the i'th axis coordinate transformations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CTYPEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a character string, giving + the name of the coordinate represented by axis i. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CUNITi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall be the units along axis i, compatible with CTYPEi to be used for + scaling and coordinate transformations along the i'th axis. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRPIXi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + identifying the location of a reference point along axis i, in units of + the axis index. This value is based upon a counter that runs from 1 to + NAXISn with an increment of 1 per pixel. The reference point value + need not be that for the center of a pixel nor lie within the actual + data array. Use comments to indicate the location of the index point + relative to the pixel. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRVALi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + giving the value of the coordinate specified by the CTYPEn keyword at + the reference point CRPIXi. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CDELTi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number + giving the partial derivative of the coordinate specified by the CTYPEi + keywords with respect to the pixel index, evaluated at the reference + point CRPIXi, in units of the coordinate specified by the CTYPEi + keyword. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null +data: + - CATDESC + - DEPEND_0 + - DISPLAY_TYPE + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +support_data: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +metadata: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - VAR_TYPE diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 87a7403e0..a8b06f64a 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -49,9 +49,10 @@ def test_global_attribute(): get_global_attributes """ - # Initialize CdfAttributeManager object which loads in default global - # variables and schema + # Initialize CdfAttributeManager object which loads in default + # global/variable schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + # Test that default information was loaded in from # "imap_default_global_cdf_attrs.yaml" assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" @@ -70,8 +71,8 @@ def test_global_attribute(): assert cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" assert ( cdf_manager.global_attributes["PI_affiliation"] - == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton," - "NJ 08540" + == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, " + "Princeton, NJ 08540" ) assert ( cdf_manager.global_attributes["File_naming_convention"] @@ -79,6 +80,7 @@ def test_global_attribute(): ) # Load in different data, test what was carried over + cdf_manager.source_dir = Path(__file__).parent.parent / "test" cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( From 33bd9562d9e92d4c522ee1ea94ab163b97fc5d36 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 11:22:57 -0600 Subject: [PATCH 21/40] Fixing PR draft comments --- .../imap_default_global_test_cdf_attrs.yaml | 2 +- .../cdf/tests/imap_test_global.yaml | 10 +- .../cdf/tests/test_cdf_attribute_manager.py | 102 +++++++++++------- 3 files changed, 71 insertions(+), 43 deletions(-) diff --git a/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml index a40dfe3d1..4f3f1ce9e 100644 --- a/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml +++ b/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml @@ -4,4 +4,4 @@ Mission_group: Dysfunctional Cats PI_name: Ana Manica PI_affiliation: LASP, CU Data_version: 002 -Test_attribute: test \ No newline at end of file +DOI: test \ No newline at end of file diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml index a5e6c5ef1..a6e1242f5 100644 --- a/imap_processing/cdf/tests/imap_test_global.yaml +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -11,4 +11,12 @@ imap_test_T1_test: TEXT: *txt Data_type: T1_test-one>Test-1 test one Logical_source: imap_test_T1_test - Logical_source_description: IMAP Mission TEST one document Level-T1. \ No newline at end of file + Logical_source_description: IMAP Mission TEST one document Level-T1. + +imap_test_T2_test: + Descriptor: *desc + TEXT: *txt + Data_type: T2_test-two>Test-2 test two + Logical_source: imap_test_T2_test + Logical_source_description: IMAP Mission TEST two document Level-T2. + Additional_info: Test two additional info \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index a8b06f64a..e2fd761f5 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -11,8 +11,6 @@ def test_default_attr_schema(): _load_default_variable_attr_schema """ - # Test - # Initialize CdfAttributeManager object which loads in default schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") @@ -78,73 +76,93 @@ def test_global_attribute(): cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) + # This fails (and it should), how do I write that as a test? + # assert cdf_manager.global_attributes["DOI"] == "test" # Load in different data, test what was carried over - cdf_manager.source_dir = Path(__file__).parent.parent / "test" + # Moving-over to tests folder + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + # Loading in test data cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") - assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" - assert ( - cdf_manager.global_attributes["Source_name"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) + + # Testing attributes in default_global_cdf_attrs_schema.yaml assert ( cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" ) - assert cdf_manager.global_attributes["Mission_group"] == "Dysfunctional Cats" - assert cdf_manager.global_attributes["PI_name"] == "Ana Manica" - assert cdf_manager.global_attributes["PI_affiliation"] == "LASP, CU" assert ( cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) + # Testing attributes in imap_default_global_test_cdf_attrs.yaml + assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + assert ( + cdf_manager.global_attributes["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert cdf_manager.global_attributes["Mission_group"] == "Dysfunctional Cats" + assert cdf_manager.global_attributes["PI_name"] == "Ana Manica" + assert cdf_manager.global_attributes["PI_affiliation"] == "LASP, CU" assert cdf_manager.global_attributes["Data_version"] == 2 - assert cdf_manager.global_attributes["Test_attribute"] == "test" + assert cdf_manager.global_attributes["DOI"] == "test" - # Expected failure: file_naming_convention is not in schema. - # for attr in cdf_manager.global_attributes.keys(): - # assert attr in cdf_manager.global_attribute_schema.keys() + # Testing that everything loaded into the global attrs is present in + # the global attrs schema + for attr in cdf_manager.global_attributes.keys(): + assert attr in cdf_manager.global_attribute_schema.keys() # Load additional global attributes cdf_manager.load_global_attributes("imap_test_global.yaml") - # Creating dictionary for loaded information - mag_test_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") - assert mag_test_global_attrs["Mission_group"] == "Dysfunctional Cats" + # Testing information previously loaded into global attributes + assert test_get_global_attrs["Project"] == "STP>Solar-Terrestrial Physics" + assert ( + test_get_global_attrs["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert test_get_global_attrs["Mission_group"] == "Dysfunctional Cats" # Testing if statement - assert mag_test_global_attrs["Descriptor"] == "TEST>Testinstrument" - assert mag_test_global_attrs["Logical_source"] == "imap_test_T1_test" - # Testing non required data from schema - assert mag_test_global_attrs["Data_type"] == "T1_test-one>Test-1 test one" + assert test_get_global_attrs["Descriptor"] == "TEST>Testinstrument" + # "Data_type" not required according to default schema + assert test_get_global_attrs["Data_type"] == "T1_test-one>Test-1 test one" + assert test_get_global_attrs["Logical_source"] == "imap_test_T1_test" + assert ( + test_get_global_attrs["Logical_source_description"] + == "IMAP Mission TEST one document Level-T1." + ) # TODO: write call that tests "Bad_name" attribute # Testing first elif statement - assert mag_test_global_attrs["Project"] == cdf_manager.global_attributes["Project"] + assert test_get_global_attrs["Project"] == cdf_manager.global_attributes["Project"] assert ( - mag_test_global_attrs["Source_name"] + test_get_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] ) - # BUT not everything in mag_test_global_attrs will be in + # BUT not everything in test_get_global_attrs will be in # cdf_manager.global_attributes["imap_test_T1_test"] # TODO: check with maxine that the below code SHOULD cause an error because # "Project" is not in self.global_attributes[inst_id] # assert mag_test_global_attrs["Project"] # == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] assert ( - mag_test_global_attrs["Data_type"] + test_get_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] ) + # Testing that everything in the global attributes was carried over + # during get_global_attributes + # This test is kind of stupid. for attr_name in cdf_manager.global_attributes["imap_test_T1_test"].keys(): - assert attr_name in mag_test_global_attrs.keys() + assert attr_name in test_get_global_attrs.keys() # Testing that required schema keys are in get_global_attributes for attr_name in cdf_manager.global_attribute_schema.keys(): required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] if required_schema is True: - assert attr_name in mag_test_global_attrs.keys() + assert attr_name in test_get_global_attrs.keys() # Testing second elif statement # Should throw error @@ -157,7 +175,9 @@ def test_variable_attribute(): get_variable_attributes """ cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + # Loading in test data cdf_manager.load_variable_attributes("imap_test_variable.yaml") # All variables required to have: @@ -186,20 +206,20 @@ def test_variable_attribute(): assert variable_attrs_2 in imap_test_variable.keys() # Calling required attributes - # assert imap_test_variable["CATDESC"] == "test time" - # assert imap_test_variable["DEPEND_0"] == "test_depend" - # assert imap_test_variable["DISPLAY_TYPE"] == "test_display_type" - # assert imap_test_variable["FIELDNAM"] == "test_display_type" - # assert imap_test_variable["FILLVAL"] == -10 - # assert imap_test_variable["FORMAT"] == "I1" - # assert imap_test_variable["LABLAXIS"] == "test_labaxis" - # assert imap_test_variable["UNITS"] == "test_units" - # assert imap_test_variable["VALIDMIN"] == 0 - # assert imap_test_variable["VALIDMAX"] == 10 - # assert imap_test_variable["VAR_TYPE"] == "test_var_type" + assert imap_test_variable["CATDESC"] == "test time" + assert imap_test_variable["DEPEND_0"] == "test_depend" + assert imap_test_variable["DISPLAY_TYPE"] == "test_display_type" + assert imap_test_variable["FIELDNAM"] == "test_field_1" + assert imap_test_variable["FILLVAL"] == -10 + assert imap_test_variable["FORMAT"] == "I1" + assert imap_test_variable["LABLAXIS"] == "test_labaxis" + assert imap_test_variable["UNITS"] == "test_units" + assert imap_test_variable["VALIDMIN"] == 0 + assert imap_test_variable["VALIDMAX"] == 10 + assert imap_test_variable["VAR_TYPE"] == "test_variable_type" # Calling to non required attributes - # assert imap_test_variable["NOT_REQUIRED"] == "test_not_required" + assert imap_test_variable["NOT_REQUIRED"] == "test_not_required" # Calling attribute name that does not exist # TODO: should throw error From 4c3c4f2b0cbee5be1c8cf9631afb126cea5896ed Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 11:49:13 -0600 Subject: [PATCH 22/40] Adding additional tests --- .../cdf/tests/imap_test_global.yaml | 2 +- .../cdf/tests/test_cdf_attribute_manager.py | 44 ++++++++++++++++--- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml index a6e1242f5..859d30d40 100644 --- a/imap_processing/cdf/tests/imap_test_global.yaml +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -19,4 +19,4 @@ imap_test_T2_test: Data_type: T2_test-two>Test-2 test two Logical_source: imap_test_T2_test Logical_source_description: IMAP Mission TEST two document Level-T2. - Additional_info: Test two additional info \ No newline at end of file + LINK_TEXT: Test two additional info \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index e2fd761f5..0f753358d 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -53,6 +53,7 @@ def test_global_attribute(): # Test that default information was loaded in from # "imap_default_global_cdf_attrs.yaml" + # THIS MAY BE AN ISSUE assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( cdf_manager.global_attributes["Source_name"] @@ -86,14 +87,17 @@ def test_global_attribute(): cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") # Testing attributes in default_global_cdf_attrs_schema.yaml - assert ( - cdf_manager.global_attributes["Discipline"] - == "Solar Physics>Heliospheric Physics" - ) assert ( cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) + + # Testing attributes in 1st loaded file, and NOT in second loaded file: + assert ( + cdf_manager.global_attributes["Discipline"] + == "Solar Physics>Heliospheric Physics" + ) + # Testing attributes in imap_default_global_test_cdf_attrs.yaml assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( @@ -152,6 +156,35 @@ def test_global_attribute(): == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] ) + # Testing second elif statement + # Should throw error + + # Load in more data using get_global_attributes + test_get_global_attrs_2 = cdf_manager.get_global_attributes("imap_test_T2_test") + # Testing information previously loaded into global attributes + assert test_get_global_attrs_2["Project"] == "STP>Solar-Terrestrial Physics" + assert ( + test_get_global_attrs_2["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert test_get_global_attrs_2["Mission_group"] == "Dysfunctional Cats" + + # Testing if statement + assert test_get_global_attrs_2["Descriptor"] == "TEST>Testinstrument" + # "Data_type" not required according to default schema + assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" + assert test_get_global_attrs_2["Logical_source"] == "imap_test_T2_test" + assert ( + test_get_global_attrs_2["Logical_source_description"] + == "IMAP Mission TEST two document Level-T2." + ) + assert test_get_global_attrs_2["LINK_TEXT"] == "Test two additional info" + + # Trying to update a default global using get_global_attributes does not work. + # For example, thing about DOI event. + + # END END END + # Testing that everything in the global attributes was carried over # during get_global_attributes # This test is kind of stupid. @@ -164,9 +197,6 @@ def test_global_attribute(): if required_schema is True: assert attr_name in test_get_global_attrs.keys() - # Testing second elif statement - # Should throw error - def test_variable_attribute(): """ From 4f6e3fbd86c2a12c7174ddf8a8bb486a802c4603 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 13:07:05 -0600 Subject: [PATCH 23/40] Breaking up test functions --- .../cdf/tests/test_cdf_attribute_manager.py | 127 ++++++++---------- 1 file changed, 57 insertions(+), 70 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 0f753358d..4fff82f20 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -15,22 +15,16 @@ def test_default_attr_schema(): cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") # Default global tests - # Check false case assert cdf_manager.global_attribute_schema["DOI"]["required"] is False - - # Check true case assert cdf_manager.global_attribute_schema["Data_level"]["required"] is True # Default variable tests - # Check false case assert ( cdf_manager.variable_attribute_schema["attribute_key"]["ABSOLUTE_ERROR"][ "required" ] is False ) - - # Check true case assert ( cdf_manager.variable_attribute_schema["attribute_key"]["RESOLUTION"]["required"] is True @@ -44,16 +38,13 @@ def test_global_attribute(): """ Test function that covers: load_global_attributes - get_global_attributes """ - # Initialize CdfAttributeManager object which loads in default - # global/variable schema + # Initialize CdfAttributeManager object which loads in default info cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") - # Test that default information was loaded in from - # "imap_default_global_cdf_attrs.yaml" - # THIS MAY BE AN ISSUE + # Testing information has been loaded in assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( cdf_manager.global_attributes["Source_name"] @@ -77,28 +68,24 @@ def test_global_attribute(): cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) - # This fails (and it should), how do I write that as a test? + # TODO: write the below test so it fails (it should fail) # assert cdf_manager.global_attributes["DOI"] == "test" - # Load in different data, test what was carried over - # Moving-over to tests folder + # Load in different data cdf_manager.source_dir = Path(__file__).parent.parent / "tests" - # Loading in test data cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") - # Testing attributes in default_global_cdf_attrs_schema.yaml + # Testing attributes carried over assert ( cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) - - # Testing attributes in 1st loaded file, and NOT in second loaded file: assert ( cdf_manager.global_attributes["Discipline"] == "Solar Physics>Heliospheric Physics" ) - # Testing attributes in imap_default_global_test_cdf_attrs.yaml + # Testing attributes newly loaded assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" assert ( cdf_manager.global_attributes["Source_name"] @@ -115,22 +102,32 @@ def test_global_attribute(): for attr in cdf_manager.global_attributes.keys(): assert attr in cdf_manager.global_attribute_schema.keys() - # Load additional global attributes + +def test_get_global_attributes(): + """ + Test function that covers: + get_global_attributes + """ + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") cdf_manager.load_global_attributes("imap_test_global.yaml") - # Creating dictionary for loaded information + + # Loading in instrument specific attributes test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") - # Testing information previously loaded into global attributes + # Testing information previously loaded into global attributes (first if statement) assert test_get_global_attrs["Project"] == "STP>Solar-Terrestrial Physics" assert ( test_get_global_attrs["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" ) assert test_get_global_attrs["Mission_group"] == "Dysfunctional Cats" - - # Testing if statement + # Testing instrument specific global attributes (first elif statement) assert test_get_global_attrs["Descriptor"] == "TEST>Testinstrument" - # "Data_type" not required according to default schema assert test_get_global_attrs["Data_type"] == "T1_test-one>Test-1 test one" assert test_get_global_attrs["Logical_source"] == "imap_test_T1_test" assert ( @@ -138,59 +135,39 @@ def test_global_attribute(): == "IMAP Mission TEST one document Level-T1." ) # TODO: write call that tests "Bad_name" attribute + # TODO: Testing second elif statement + # How do? + # Load in more data using get_global_attributes + test_get_global_attrs_2 = cdf_manager.get_global_attributes("imap_test_T2_test") + # Testing information previously loaded into global attributes (first if statement) + assert test_get_global_attrs_2["Project"] == "STP>Solar-Terrestrial Physics" # Testing first elif statement + assert test_get_global_attrs_2["Descriptor"] == "TEST>Testinstrument" + # "Data_type" not required according to default schema + assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" + # TODO: Testing second elif statement + + # Testing how instrument_id operates assert test_get_global_attrs["Project"] == cdf_manager.global_attributes["Project"] assert ( test_get_global_attrs["Source_name"] == cdf_manager.global_attributes["Source_name"] ) - # BUT not everything in test_get_global_attrs will be in - # cdf_manager.global_attributes["imap_test_T1_test"] - # TODO: check with maxine that the below code SHOULD cause an error because - # "Project" is not in self.global_attributes[inst_id] - # assert mag_test_global_attrs["Project"] - # == cdf_manager.global_attributes["imap_test_T1_test"]["Project"] assert ( test_get_global_attrs["Data_type"] == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] ) - - # Testing second elif statement - # Should throw error - - # Load in more data using get_global_attributes - test_get_global_attrs_2 = cdf_manager.get_global_attributes("imap_test_T2_test") - # Testing information previously loaded into global attributes - assert test_get_global_attrs_2["Project"] == "STP>Solar-Terrestrial Physics" - assert ( - test_get_global_attrs_2["Source_name"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) - assert test_get_global_attrs_2["Mission_group"] == "Dysfunctional Cats" - - # Testing if statement - assert test_get_global_attrs_2["Descriptor"] == "TEST>Testinstrument" - # "Data_type" not required according to default schema - assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" - assert test_get_global_attrs_2["Logical_source"] == "imap_test_T2_test" assert ( - test_get_global_attrs_2["Logical_source_description"] - == "IMAP Mission TEST two document Level-T2." + cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] + == "imap_test_T1_test" ) - assert test_get_global_attrs_2["LINK_TEXT"] == "Test two additional info" + # TODO: The following test should throw an error + # assert cdf_manager.global_attributes["imap_test_T1_test"]["Project"] # Trying to update a default global using get_global_attributes does not work. # For example, thing about DOI event. - # END END END - - # Testing that everything in the global attributes was carried over - # during get_global_attributes - # This test is kind of stupid. - for attr_name in cdf_manager.global_attributes["imap_test_T1_test"].keys(): - assert attr_name in test_get_global_attrs.keys() - # Testing that required schema keys are in get_global_attributes for attr_name in cdf_manager.global_attribute_schema.keys(): required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] @@ -204,6 +181,8 @@ def test_variable_attribute(): load_variable_attributes get_variable_attributes """ + + # Creating CdfAttributeManager object, loading in default data cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") cdf_manager.source_dir = Path(__file__).parent.parent / "tests" cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") @@ -224,16 +203,24 @@ def test_variable_attribute(): for attr in expected_attributes: assert attr in variable_attrs.keys() - # TODO: Call, and test get_variable_attributes + +def test_get_variable_attributes(): + # Creating CdfAttributeManager object, loading in default data + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_variable_attributes("imap_test_variable.yaml") + + # Loading in instrument specific attributes imap_test_variable = cdf_manager.get_variable_attributes("test_field_1") - # Make sure all expected attributes are here - for variable_attrs_2 in cdf_manager.variable_attribute_schema.keys(): - required_var_attributes = cdf_manager.variable_attribute_schema[ - variable_attrs_2 - ] + # Make sure all expected attributes are present + for variable_attrs in cdf_manager.variable_attribute_schema.keys(): + required_var_attributes = cdf_manager.variable_attribute_schema[variable_attrs] if required_var_attributes is True: - assert variable_attrs_2 in imap_test_variable.keys() + assert variable_attrs in imap_test_variable.keys() # Calling required attributes assert imap_test_variable["CATDESC"] == "test time" @@ -253,4 +240,4 @@ def test_variable_attribute(): # Calling attribute name that does not exist # TODO: should throw error - # assert imap_test_variable["DOES_NOT_EXIST"] == "test time" + # assert imap_test_variable["DOES_NOT_EXIST"] == "test time" From 9e43ad817ecc1bc5a2385eb42005e11875a925ac Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 15:41:01 -0600 Subject: [PATCH 24/40] Adding more tests --- .../cdf/tests/imap_test_global.yaml | 6 +- .../cdf/tests/imap_test_variable.yaml | 13 ++- .../cdf/tests/test_cdf_attribute_manager.py | 104 ++++++++++++++---- 3 files changed, 102 insertions(+), 21 deletions(-) diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml index 859d30d40..ae00269d6 100644 --- a/imap_processing/cdf/tests/imap_test_global.yaml +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -19,4 +19,8 @@ imap_test_T2_test: Data_type: T2_test-two>Test-2 test two Logical_source: imap_test_T2_test Logical_source_description: IMAP Mission TEST two document Level-T2. - LINK_TEXT: Test two additional info \ No newline at end of file + LINK_TEXT: Test two additional info + +imap_test_T3_test: + Descriptor: *desc + TEXT: *txt diff --git a/imap_processing/cdf/tests/imap_test_variable.yaml b/imap_processing/cdf/tests/imap_test_variable.yaml index b2d5fd3af..cbdd98cfa 100644 --- a/imap_processing/cdf/tests/imap_test_variable.yaml +++ b/imap_processing/cdf/tests/imap_test_variable.yaml @@ -22,4 +22,15 @@ test_field_1: REFERENCE_POSITION: test_reference_position NOT_REQUIRED: test_not_required - # TODO: come back to format \ No newline at end of file +test_field_2: + <<: *default + CATDESC: test time 2 + FIELDNAM: test_field_2 + LABLAXIS: test_labaxis_2 + UNITS: test_units_2 + VAR_TYPE: test_variable_type_2 + SCALETYP: test_scaletyp_2 + MONOTON: test_monoton_2 + TIME_BASE: 11 + TIME_SCALE: test_time_scale_2 + REFERENCE_POSITION: test_reference_position_2 \ No newline at end of file diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 4fff82f20..ff0fc1f64 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -1,6 +1,7 @@ from pathlib import Path -# import pytest +import pytest + from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager @@ -68,8 +69,10 @@ def test_global_attribute(): cdf_manager.global_attributes["File_naming_convention"] == "source_descriptor_datatype_yyyyMMdd_vNNN" ) - # TODO: write the below test so it fails (it should fail) - # assert cdf_manager.global_attributes["DOI"] == "test" + # The following test should fail because "DOI" is not an attribute in + # imap_default_global_cdf_attrs.yaml + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["DOI"] == "test" # Load in different data cdf_manager.source_dir = Path(__file__).parent.parent / "tests" @@ -134,9 +137,15 @@ def test_get_global_attributes(): test_get_global_attrs["Logical_source_description"] == "IMAP Mission TEST one document Level-T1." ) - # TODO: write call that tests "Bad_name" attribute - # TODO: Testing second elif statement - # How do? + # Not given, and not required information + assert test_get_global_attrs["Data_level"] is None + with pytest.raises(KeyError): + assert test_get_global_attrs["bad_name"] == "false info" + + # Testing second elif statement + test_error_elif = cdf_manager.get_global_attributes("imap_test_T3_test") + with pytest.raises(KeyError): + assert test_error_elif["Data_type"] == "Does Not Exist" # Load in more data using get_global_attributes test_get_global_attrs_2 = cdf_manager.get_global_attributes("imap_test_T2_test") @@ -146,7 +155,6 @@ def test_get_global_attributes(): assert test_get_global_attrs_2["Descriptor"] == "TEST>Testinstrument" # "Data_type" not required according to default schema assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" - # TODO: Testing second elif statement # Testing how instrument_id operates assert test_get_global_attrs["Project"] == cdf_manager.global_attributes["Project"] @@ -162,8 +170,8 @@ def test_get_global_attributes(): cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] == "imap_test_T1_test" ) - # TODO: The following test should throw an error - # assert cdf_manager.global_attributes["imap_test_T1_test"]["Project"] + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["imap_test_T1_test"]["Project"] # Trying to update a default global using get_global_attributes does not work. # For example, thing about DOI event. @@ -175,6 +183,9 @@ def test_get_global_attributes(): assert attr_name in test_get_global_attrs.keys() +# TODO: Test add_global_attribute + + def test_variable_attribute(): """ Test function that covers: @@ -191,17 +202,41 @@ def test_variable_attribute(): # All variables required to have: expected_attributes = [ + "CATDESC", + "DEPEND_0", "DISPLAY_TYPE", + "FIELDNAM", "FILLVAL", "FORMAT", + "LABLAXIS", + "UNITS", "VALIDMIN", "VALIDMAX", "VAR_TYPE", ] - for variable_attrs in cdf_manager.variable_attributes.values(): - for attr in expected_attributes: - assert attr in variable_attrs.keys() + # Assuring all required attributes are loaded in + for attr_name in cdf_manager.variable_attribute_schema["attribute_key"]: + attribute = cdf_manager.variable_attribute_schema["attribute_key"][attr_name] + if attribute["required"] is True: + for exp_attr in expected_attributes: + assert ( + exp_attr + in cdf_manager.variable_attribute_schema["attribute_key"].keys() + ) + + # Testing specific things + assert ( + cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] + == cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] + ) + assert cdf_manager.variable_attributes["default_attrs"]["FILLVAL"] == -10 + assert cdf_manager.variable_attributes["test_field_1"]["DEPEND_0"] == "test_depend" + assert ( + cdf_manager.variable_attributes["default_attrs"]["VAR_TYPE"] == "test_var_type" + ) + with pytest.raises(KeyError): + assert cdf_manager.variable_attributes["default_attrs"]["CATDESC"] == "test" def test_get_variable_attributes(): @@ -222,22 +257,53 @@ def test_get_variable_attributes(): if required_var_attributes is True: assert variable_attrs in imap_test_variable.keys() - # Calling required attributes - assert imap_test_variable["CATDESC"] == "test time" + # Calling default attributes assert imap_test_variable["DEPEND_0"] == "test_depend" assert imap_test_variable["DISPLAY_TYPE"] == "test_display_type" - assert imap_test_variable["FIELDNAM"] == "test_field_1" assert imap_test_variable["FILLVAL"] == -10 assert imap_test_variable["FORMAT"] == "I1" - assert imap_test_variable["LABLAXIS"] == "test_labaxis" - assert imap_test_variable["UNITS"] == "test_units" assert imap_test_variable["VALIDMIN"] == 0 assert imap_test_variable["VALIDMAX"] == 10 assert imap_test_variable["VAR_TYPE"] == "test_variable_type" + # Calling required attributes + assert imap_test_variable["CATDESC"] == "test time" + assert imap_test_variable["FIELDNAM"] == "test_field_1" + assert imap_test_variable["LABLAXIS"] == "test_labaxis" + assert imap_test_variable["UNITS"] == "test_units" + assert imap_test_variable["VAR_TYPE"] == "test_variable_type" + assert imap_test_variable["SCALETYP"] == "test_scaletyp" + assert imap_test_variable["MONOTON"] == "test_monoton" + assert imap_test_variable["TIME_BASE"] == 10 + assert imap_test_variable["TIME_SCALE"] == "test_time_scale" + assert imap_test_variable["REFERENCE_POSITION"] == "test_reference_position" + # Calling to non required attributes assert imap_test_variable["NOT_REQUIRED"] == "test_not_required" # Calling attribute name that does not exist - # TODO: should throw error - # assert imap_test_variable["DOES_NOT_EXIST"] == "test time" + with pytest.raises(KeyError): + assert imap_test_variable["DOES_NOT_EXIST"] == "test time" + + # TODO: Load in different data, test again + imap_test_variable_2 = cdf_manager.get_variable_attributes("test_field_2") + # Calling default attributes + assert imap_test_variable_2["DEPEND_0"] == "test_depend" + assert imap_test_variable_2["DISPLAY_TYPE"] == "test_display_type" + assert imap_test_variable_2["FILLVAL"] == -10 + assert imap_test_variable_2["FORMAT"] == "I1" + assert imap_test_variable_2["VALIDMIN"] == 0 + assert imap_test_variable_2["VALIDMAX"] == 10 + assert imap_test_variable_2["VAR_TYPE"] == "test_variable_type_2" + + # Calling required attributes + assert imap_test_variable_2["CATDESC"] == "test time 2" + assert imap_test_variable_2["FIELDNAM"] == "test_field_2" + assert imap_test_variable_2["LABLAXIS"] == "test_labaxis_2" + assert imap_test_variable_2["UNITS"] == "test_units_2" + assert imap_test_variable_2["VAR_TYPE"] == "test_variable_type_2" + assert imap_test_variable_2["SCALETYP"] == "test_scaletyp_2" + assert imap_test_variable_2["MONOTON"] == "test_monoton_2" + assert imap_test_variable_2["TIME_BASE"] == 11 + assert imap_test_variable_2["TIME_SCALE"] == "test_time_scale_2" + assert imap_test_variable_2["REFERENCE_POSITION"] == "test_reference_position_2" From 3d797d1233bfe59d6bff1a440054bba2f81c2941 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 15:59:12 -0600 Subject: [PATCH 25/40] Getting closer to final draft --- .../cdf/tests/test_cdf_attribute_manager.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index ff0fc1f64..01a19f001 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -183,7 +183,20 @@ def test_get_global_attributes(): assert attr_name in test_get_global_attrs.keys() -# TODO: Test add_global_attribute +def test_add_global_attribute(): + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Changing a dynamic global variable + cdf_manager.add_global_attribute("Project", "Test Project") + assert cdf_manager.global_attributes["Project"] == "Test Project" + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + assert test_get_global_attrs["Project"] == "Test Project" def test_variable_attribute(): @@ -285,7 +298,7 @@ def test_get_variable_attributes(): with pytest.raises(KeyError): assert imap_test_variable["DOES_NOT_EXIST"] == "test time" - # TODO: Load in different data, test again + # Load in different data, test again imap_test_variable_2 = cdf_manager.get_variable_attributes("test_field_2") # Calling default attributes assert imap_test_variable_2["DEPEND_0"] == "test_depend" From 0bb6b88074d426427846801c4bb97a0fa01f885c Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Tue, 18 Jun 2024 17:06:49 -0600 Subject: [PATCH 26/40] Finishing touches --- imap_processing/cdf/imap_cdf_manager.py | 8 +- .../imap_instrument2_global_cdf_attrs.yaml | 24 ++ ...map_instrument2_level2_variable_attrs.yaml | 310 ++++++++++++++++++ .../cdf/tests/test_imap_cdf_manager.py | 46 +-- 4 files changed, 363 insertions(+), 25 deletions(-) create mode 100644 imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 1081869a8..78ae0332a 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -13,9 +13,13 @@ class ImapCdfAttributes(CdfAttributeManager): """Contains IMAP specific tools and settings for CDF management.""" - def __init__(self): + def __init__(self, source_dir_input=None): """Set the path to the config directory.""" - super().__init__(Path(__file__).parent / "config") + if source_dir_input is None: + super().__init__(Path(__file__).parent / "config") + else: + super().__init__(Path(__file__).parent / "config") + self.source_dir = Path(__file__).parent / source_dir_input def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml new file mode 100644 index 000000000..5376fe029 --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml @@ -0,0 +1,24 @@ +instrument_base: &instrument_base + Descriptor: SWE>Solar Wind Electron + TEXT: > + The IMAP Solar Wind Electron (SWE) instrument measures the solar wind electrons + in the heliosphere. SWE will contribute to our understanding of the acceleration + and transport of charged particles in the heliosphere. + SWE design and assembly is led by the Princeton Plasma Physics Laboratory. See + https://imap.princeton.edu/instruments/swe for more details. + Instrument_type: "Particles (space)" + +imap_swe_l1a_sci: + <<: *instrument_base + # NOTE: Right now, this Data_level is required to produce valid CDF + Data_level: 1A + Data_type: L1A_SCI>Level-1A Science data + Logical_source: imap_swe_l1a_sci + Logical_source_description: SWE Instrument Level-1A Science Data + +imap_swe_l1b_sci: + <<: *instrument_base + Data_level: 1A + Data_type: L1B_SCI>Level-1B Science data + Logical_source: imap_swe_l1b_sci + Logical_source_description: SWE Instrument Level-1B Science Data diff --git a/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml new file mode 100644 index 000000000..fdb826007 --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml @@ -0,0 +1,310 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: epoch + DISPLAY_TYPE: time_series + FILLVAL: -9223372036854775808 + FORMAT: I19 + VALIDMIN: 0 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data + +epoch: + <<: *default + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + +x_front: + <<: *default + CATDESC: x front position + FIELDNAM: Event x-position on front foil + LABLAXIS: x front position + # TODO: come back to format + UNITS: mm + +y_front: + <<: *default + CATDESC: y front position + FIELDNAM: Event y-position on front foil + LABLAXIS: y front position + # TODO: come back to format + UNITS: mm + +x_back: + <<: *default + CATDESC: x back position + FIELDNAM: x_back + LABLAXIS: x back position + # TODO: come back to format + UNITS: mm + +y_back: + <<: *default + CATDESC: y back position + FIELDNAM: y_back + LABLAXIS: y back position + # TODO: come back to format + UNITS: mm + +x_coin: + <<: *default + CATDESC: x coincidence position + FIELDNAM: x_coin + LABLAXIS: x coincidence position + # TODO: come back to format + UNITS: mm + +tof_start_stop: + <<: *default + CATDESC: Particle time of flight from start to stop + FIELDNAM: tof_start_stop + LABLAXIS: tof start stop + # TODO: come back to format + UNITS: ns + +tof_stop_coin: + <<: *default + CATDESC: Particle time of flight from stop to coincidence + FIELDNAM: tof_stop_coin + LABLAXIS: tof stop coin + # TODO: come back to format + UNITS: ns + +tof_corrected: + <<: *default + CATDESC: Corrected time of flight + FIELDNAM: tof_corrected + LABLAXIS: tof corrected + # TODO: come back to format + UNITS: ns + +eventtype: + <<: *default + CATDESC: Eventtype (1-2 top and bottom stop types; 8-15 ssd stop types) + FIELDNAM: eventtype + LABLAXIS: eventtype + # TODO: come back to format + UNITS: " " + +vx_ultra: + <<: *default + CATDESC: Normalized component of the velocity vector in x direction in the instrument frame. + FIELDNAM: vx_ultra + LABLAXIS: vx ultra + # TODO: come back to format + UNITS: " " + +vy_ultra: + <<: *default + CATDESC: Normalized component of the velocity vector in y direction in the instrument frame. + FIELDNAM: vy_ultra + LABLAXIS: vy ultra + # TODO: come back to format + UNITS: " " + +vz_ultra: + <<: *default + CATDESC: Normalized component of the velocity vector in z direction in the instrument frame. + FIELDNAM: vz_ultra + LABLAXIS: vz ultra + # TODO: come back to format + UNITS: " " + +energy: + <<: *default + CATDESC: Energy measured using the pulse height from the stop anode (DN) or Energy measured using the SSD (keV) depending on eventtype. + FIELDNAM: energy + LABLAXIS: energy + # TODO: come back to format + UNITS: keV + +species: + <<: *default + CATDESC: Species bin. + FIELDNAM: species + LABLAXIS: species + # TODO: come back to format + UNITS: " " + +event_efficiency: + <<: *default + CATDESC: Estimated event efficiency for this path through the instrument. + FIELDNAM: event_efficiency + LABLAXIS: event efficiency + # TODO: come back to format + UNITS: " " + +vx_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in x direction in the spacecraft frame. + FIELDNAM: vx_sc + LABLAXIS: vx sc + # TODO: come back to format + UNITS: " " + +vy_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in y direction in the spacecraft frame. + FIELDNAM: vy_sc + LABLAXIS: vy sc + # TODO: come back to format + UNITS: " " + +vz_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in z direction in the spacecraft frame. + FIELDNAM: vz_sc + LABLAXIS: vz sc + # TODO: come back to format + UNITS: " " + +vx_dps_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) + FIELDNAM: vx_dps_sc + LABLAXIS: vx dps sc + # TODO: come back to format + UNITS: " " + +vy_dps_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) + FIELDNAM: vy_dps_sc + LABLAXIS: vy dps sc + # TODO: come back to format + UNITS: " " + +vz_dps_sc: + <<: *default + CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) + FIELDNAM: vz_dps_sc + LABLAXIS: vz dps sc + # TODO: come back to format + UNITS: " " + +vx_dps_helio: + <<: *default + CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT heliosphere. + FIELDNAM: vx_dps_helio + LABLAXIS: vx dps helio + # TODO: come back to format + UNITS: " " + +vy_dps_helio: + <<: *default + CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT heliosphere. + FIELDNAM: vy_dps_helio + LABLAXIS: vy dps helio + # TODO: come back to format + UNITS: " " + +vz_dps_helio: + <<: *default + CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT heliosphere. + FIELDNAM: vz_dps_helio + LABLAXIS: vz dps helio + # TODO: come back to format + UNITS: " " + +eventtimes: + <<: *default + CATDESC: Event times calculated from event and universal spin table. + FIELDNAM: eventtimes + LABLAXIS: event times + # TODO: come back to format + UNITS: " " + +spin_number: + <<: *default + CATDESC: Spin number from Universal Spin Table. + FIELDNAM: spin_number + LABLAXIS: spin_number + # TODO: come back to format + UNITS: " " + +spin_start_time: + <<: *default + CATDESC: Spin start time from Universal Spin Table. + FIELDNAM: spin_start_time + LABLAXIS: spin start time + # TODO: come back to format + UNITS: s + +avg_spin_period: + <<: *default + CATDESC: Average spin period from Universal Spin Table. + FIELDNAM: avg_spin_period + LABLAXIS: avg_spin_period + # TODO: come back to format + UNITS: s + +rate_start_pulses: + <<: *default + CATDESC: Rate of start pulses (/s). + FIELDNAM: rate_start_pulses + LABLAXIS: rate start pulses + # TODO: come back to format + UNITS: 1/s + +rate_stop_pulses: + <<: *default + CATDESC: Rate of stop pulses (/s). + FIELDNAM: rate_stop_pulses + LABLAXIS: rate stop pulses + # TODO: come back to format + UNITS: 1/s + +rate_coin_pulses: + <<: *default + CATDESC: Rate of coincidence pulses (/s). + FIELDNAM: rate_coin_pulses + LABLAXIS: rate coincidence pulses + # TODO: come back to format + UNITS: 1/s + +rate_processed_events: + <<: *default + CATDESC: Rate of processed events (/s). + FIELDNAM: rate_processed_events + LABLAXIS: rate processed events + # TODO: come back to format + UNITS: 1/s + +rate_rejected_events: + <<: *default + CATDESC: Rate of rejected events (/s). + FIELDNAM: rate_rejected_events + LABLAXIS: rate rejected events + # TODO: come back to format + UNITS: 1/s + +quality_hk: + <<: *default + CATDESC: Spin filter derived from Ultra housekeeping. Bitwise flagging used to filter the spin. + FIELDNAM: quality_hk + LABLAXIS: quality hk + # TODO: come back to format + UNITS: " " + +quality_attitude: + <<: *default + CATDESC: Spin filter derived from IMAP attitude. Bitwise flagging used to filter the spin. + FIELDNAM: quality_attitude + LABLAXIS: quality attitude + # TODO: come back to format + UNITS: " " + +quality_instruments: + <<: *default + CATDESC: Spin filter derived from instruments other than Ultra. Bitwise flagging used to filter the spin. + FIELDNAM: quality_instruments + LABLAXIS: quality instruments + # TODO: come back to format + UNITS: " " diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 10e5dfd94..b3fa12534 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -2,48 +2,48 @@ # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -# test add_instrument_global_attrs(self, instrument: str): - def test_add_instrument_global_attrs(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes() - # Call function + imap_cdf_manager = ImapCdfAttributes("tests") imap_cdf_manager.add_instrument_global_attrs("instrument") - # Testing the actual function + # Testing data loaded in imap_instrument = imap_cdf_manager.get_global_attributes("imap_test_T1_test") assert imap_instrument["Data_type"] == "T1_test-one>Test-1 test one" assert imap_instrument["Project"] == "STP>Solar-Terrestrial Physics" # Testing reloading data - imap_cdf_manager.add_instrument_global_attrs("swe") + imap_cdf_manager.add_instrument_global_attrs("instrument2") - # Testing again - swe_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci") - assert swe_instrument["Data_type"] == "L1A_SCI>Level-1A Science data" - assert swe_instrument["Project"] == "STP>Solar-Terrestrial Physics" + # Testing data carried over, and overwritten + instrument2_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci") + assert instrument2_instrument["Data_type"] == "L1A_SCI>Level-1A Science data" + assert instrument2_instrument["Project"] == "STP>Solar-Terrestrial Physics" -# test add_instrument_variable_attrs(self, instrument: str, level: str): def test_add_instrument_variable_attrs(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes() - # Call function + imap_cdf_manager = ImapCdfAttributes("tests") imap_cdf_manager.add_instrument_variable_attrs("instrument", "level") # Testing the actual function - # imap_instrument = imap_cdf_manager.get_variable_attributes("imap_test_T1_test") - # assert imap_instrument["DEPEND_0"] == "test_depend" - # assert imap_instrument["CATDESC"] == "test time" - # assert imap_instrument["VALIDMAX"] == 10 + imap_instrument = imap_cdf_manager.get_variable_attributes("test_field_1") + assert imap_instrument["DEPEND_0"] == "test_depend" + assert imap_instrument["CATDESC"] == "test time" + assert imap_instrument["FIELDNAM"] == "test_field_1" + assert imap_instrument["UNITS"] == "test_units" # Testing reloading data - imap_cdf_manager.add_instrument_variable_attrs("ultra", "l1b") + imap_cdf_manager.add_instrument_variable_attrs("instrument2", "level2") # Testing again - # ultra_instrument = imap_cdf_manager.get_variable_attributes("imap_swe_l1a_sci") - # assert ultra_instrument["DEPEND_0"] == "epoch" - # assert ultra_instrument["CATDESC"] == "Time, number of nanoseconds - # since J2000 with leap seconds included" - # assert ultra_instrument["VALIDMAX"] == 9223372036854775807 + instrument2_instrument = imap_cdf_manager.get_variable_attributes("epoch") + assert instrument2_instrument["DEPEND_0"] == "epoch" + assert instrument2_instrument["DISPLAY_TYPE"] == "time_series" + assert instrument2_instrument["VALIDMAX"] == 9223372036854775807 + assert ( + instrument2_instrument["CATDESC"] + == "Time, number of nanoseconds since J2000 with leap seconds included" + ) + assert instrument2_instrument["UNITS"] == "ns" From d2fe9041ae45966aaa84b076f589480fd047ffec Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 11:31:57 -0600 Subject: [PATCH 27/40] Codcov check --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index b3fa12534..0f69708a8 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -22,6 +22,15 @@ def test_add_instrument_global_attrs(): assert instrument2_instrument["Project"] == "STP>Solar-Terrestrial Physics" +def test_base_global_info(): + # Create an ImapCdfAttributes object + imap_cdf_manager = ImapCdfAttributes() + assert ( + imap_cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + ) + assert imap_cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" + + def test_add_instrument_variable_attrs(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes("tests") From 4045f94b624ad8153c1d00a1e74ba70821b4291c Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 11:54:55 -0600 Subject: [PATCH 28/40] Codcov check --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 0f69708a8..19282ad08 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -22,13 +22,14 @@ def test_add_instrument_global_attrs(): assert instrument2_instrument["Project"] == "STP>Solar-Terrestrial Physics" -def test_base_global_info(): +def testing_source_dir(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes() assert ( - imap_cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + str(imap_cdf_manager.source_dir) + == "/Users/anma6676/Desktop/Repositories/imap_processing" + "/imap_processing/cdf/config" ) - assert imap_cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" def test_add_instrument_variable_attrs(): From 84b78a906b89a1b174aeedffefc3186304729c7f Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 14:11:39 -0600 Subject: [PATCH 29/40] Else Testing --- imap_processing/cdf/imap_cdf_manager.py | 1 + imap_processing/cdf/tests/test_imap_cdf_manager.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 78ae0332a..3426d4e01 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -19,6 +19,7 @@ def __init__(self, source_dir_input=None): super().__init__(Path(__file__).parent / "config") else: super().__init__(Path(__file__).parent / "config") + # self.source_dir = Path(__file__).parent / "config" self.source_dir = Path(__file__).parent / source_dir_input def add_instrument_global_attrs(self, instrument: str): diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 19282ad08..620ad7c88 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -24,11 +24,11 @@ def test_add_instrument_global_attrs(): def testing_source_dir(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager = ImapCdfAttributes("tests") assert ( str(imap_cdf_manager.source_dir) == "/Users/anma6676/Desktop/Repositories/imap_processing" - "/imap_processing/cdf/config" + "/imap_processing/cdf/tests" ) From c454d3c5e9c60a7182b9a764074684610c3a5719 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 14:30:33 -0600 Subject: [PATCH 30/40] Hopefully done --- imap_processing/cdf/imap_cdf_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 3426d4e01..78ae0332a 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -19,7 +19,6 @@ def __init__(self, source_dir_input=None): super().__init__(Path(__file__).parent / "config") else: super().__init__(Path(__file__).parent / "config") - # self.source_dir = Path(__file__).parent / "config" self.source_dir = Path(__file__).parent / source_dir_input def add_instrument_global_attrs(self, instrument: str): From be0d064feee1965c6b5d6ee87da1733b96dab4ac Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 16:06:33 -0600 Subject: [PATCH 31/40] Fixing PR comments --- imap_processing/cdf/imap_cdf_manager.py | 11 +- .../tests/imap_default_global_cdf_attrs.yaml | 8 + .../imap_instrument2_global_cdf_attrs.yaml | 1 - ...map_instrument2_level2_variable_attrs.yaml | 282 +---------- .../default_global_cdf_attrs_schema_test.yaml | 246 --------- ...efault_variable_cdf_attrs_schema_test.yaml | 466 ------------------ .../cdf/tests/test_cdf_attribute_manager.py | 52 +- 7 files changed, 57 insertions(+), 1009 deletions(-) create mode 100644 imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml delete mode 100644 imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml delete mode 100644 imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 78ae0332a..669ac3e40 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -13,13 +13,18 @@ class ImapCdfAttributes(CdfAttributeManager): """Contains IMAP specific tools and settings for CDF management.""" - def __init__(self, source_dir_input=None): + def __init__(self, source_dir=None): """Set the path to the config directory.""" - if source_dir_input is None: + if source_dir is None: super().__init__(Path(__file__).parent / "config") else: super().__init__(Path(__file__).parent / "config") - self.source_dir = Path(__file__).parent / source_dir_input + self.source_dir = Path(__file__).parent / source_dir + # if source_dir_input is None: + # super().__init__(Path(__file__).parent / source_dir_input + # else: + # super().__init__(Path(__file__).parent / "config") + # self.source_dir = Path(__file__).parent / source_dir_input def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml new file mode 100644 index 000000000..259f7b70e --- /dev/null +++ b/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml @@ -0,0 +1,8 @@ +Project: STP>Solar-Terrestrial Physics +Source_name: IMAP>Interstellar Mapping and Acceleration Probe +Discipline: Solar Physics>Heliospheric Physics +# TODO: CDF docs say this value should be IMAP +Mission_group: IMAP>Interstellar Mapping and Acceleration Probe +PI_name: Dr. David J. McComas +PI_affiliation: Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540 +File_naming_convention: source_descriptor_datatype_yyyyMMdd_vNNN diff --git a/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml index 5376fe029..f28c9bc79 100644 --- a/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml +++ b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml @@ -10,7 +10,6 @@ instrument_base: &instrument_base imap_swe_l1a_sci: <<: *instrument_base - # NOTE: Right now, this Data_level is required to produce valid CDF Data_level: 1A Data_type: L1A_SCI>Level-1A Science data Logical_source: imap_swe_l1a_sci diff --git a/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml index fdb826007..c01deccaf 100644 --- a/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml +++ b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml @@ -27,284 +27,4 @@ x_front: FIELDNAM: Event x-position on front foil LABLAXIS: x front position # TODO: come back to format - UNITS: mm - -y_front: - <<: *default - CATDESC: y front position - FIELDNAM: Event y-position on front foil - LABLAXIS: y front position - # TODO: come back to format - UNITS: mm - -x_back: - <<: *default - CATDESC: x back position - FIELDNAM: x_back - LABLAXIS: x back position - # TODO: come back to format - UNITS: mm - -y_back: - <<: *default - CATDESC: y back position - FIELDNAM: y_back - LABLAXIS: y back position - # TODO: come back to format - UNITS: mm - -x_coin: - <<: *default - CATDESC: x coincidence position - FIELDNAM: x_coin - LABLAXIS: x coincidence position - # TODO: come back to format - UNITS: mm - -tof_start_stop: - <<: *default - CATDESC: Particle time of flight from start to stop - FIELDNAM: tof_start_stop - LABLAXIS: tof start stop - # TODO: come back to format - UNITS: ns - -tof_stop_coin: - <<: *default - CATDESC: Particle time of flight from stop to coincidence - FIELDNAM: tof_stop_coin - LABLAXIS: tof stop coin - # TODO: come back to format - UNITS: ns - -tof_corrected: - <<: *default - CATDESC: Corrected time of flight - FIELDNAM: tof_corrected - LABLAXIS: tof corrected - # TODO: come back to format - UNITS: ns - -eventtype: - <<: *default - CATDESC: Eventtype (1-2 top and bottom stop types; 8-15 ssd stop types) - FIELDNAM: eventtype - LABLAXIS: eventtype - # TODO: come back to format - UNITS: " " - -vx_ultra: - <<: *default - CATDESC: Normalized component of the velocity vector in x direction in the instrument frame. - FIELDNAM: vx_ultra - LABLAXIS: vx ultra - # TODO: come back to format - UNITS: " " - -vy_ultra: - <<: *default - CATDESC: Normalized component of the velocity vector in y direction in the instrument frame. - FIELDNAM: vy_ultra - LABLAXIS: vy ultra - # TODO: come back to format - UNITS: " " - -vz_ultra: - <<: *default - CATDESC: Normalized component of the velocity vector in z direction in the instrument frame. - FIELDNAM: vz_ultra - LABLAXIS: vz ultra - # TODO: come back to format - UNITS: " " - -energy: - <<: *default - CATDESC: Energy measured using the pulse height from the stop anode (DN) or Energy measured using the SSD (keV) depending on eventtype. - FIELDNAM: energy - LABLAXIS: energy - # TODO: come back to format - UNITS: keV - -species: - <<: *default - CATDESC: Species bin. - FIELDNAM: species - LABLAXIS: species - # TODO: come back to format - UNITS: " " - -event_efficiency: - <<: *default - CATDESC: Estimated event efficiency for this path through the instrument. - FIELDNAM: event_efficiency - LABLAXIS: event efficiency - # TODO: come back to format - UNITS: " " - -vx_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in x direction in the spacecraft frame. - FIELDNAM: vx_sc - LABLAXIS: vx sc - # TODO: come back to format - UNITS: " " - -vy_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in y direction in the spacecraft frame. - FIELDNAM: vy_sc - LABLAXIS: vy sc - # TODO: come back to format - UNITS: " " - -vz_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in z direction in the spacecraft frame. - FIELDNAM: vz_sc - LABLAXIS: vz sc - # TODO: come back to format - UNITS: " " - -vx_dps_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) - FIELDNAM: vx_dps_sc - LABLAXIS: vx dps sc - # TODO: come back to format - UNITS: " " - -vy_dps_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) - FIELDNAM: vy_dps_sc - LABLAXIS: vy dps sc - # TODO: come back to format - UNITS: " " - -vz_dps_sc: - <<: *default - CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT spacecraft (velocity w/ spacecraft frame subtracted) - FIELDNAM: vz_dps_sc - LABLAXIS: vz dps sc - # TODO: come back to format - UNITS: " " - -vx_dps_helio: - <<: *default - CATDESC: Normalized component of the velocity vector in x direction in the DPS frame at rest WRT heliosphere. - FIELDNAM: vx_dps_helio - LABLAXIS: vx dps helio - # TODO: come back to format - UNITS: " " - -vy_dps_helio: - <<: *default - CATDESC: Normalized component of the velocity vector in y direction in the DPS frame at rest WRT heliosphere. - FIELDNAM: vy_dps_helio - LABLAXIS: vy dps helio - # TODO: come back to format - UNITS: " " - -vz_dps_helio: - <<: *default - CATDESC: Normalized component of the velocity vector in z direction in the DPS frame at rest WRT heliosphere. - FIELDNAM: vz_dps_helio - LABLAXIS: vz dps helio - # TODO: come back to format - UNITS: " " - -eventtimes: - <<: *default - CATDESC: Event times calculated from event and universal spin table. - FIELDNAM: eventtimes - LABLAXIS: event times - # TODO: come back to format - UNITS: " " - -spin_number: - <<: *default - CATDESC: Spin number from Universal Spin Table. - FIELDNAM: spin_number - LABLAXIS: spin_number - # TODO: come back to format - UNITS: " " - -spin_start_time: - <<: *default - CATDESC: Spin start time from Universal Spin Table. - FIELDNAM: spin_start_time - LABLAXIS: spin start time - # TODO: come back to format - UNITS: s - -avg_spin_period: - <<: *default - CATDESC: Average spin period from Universal Spin Table. - FIELDNAM: avg_spin_period - LABLAXIS: avg_spin_period - # TODO: come back to format - UNITS: s - -rate_start_pulses: - <<: *default - CATDESC: Rate of start pulses (/s). - FIELDNAM: rate_start_pulses - LABLAXIS: rate start pulses - # TODO: come back to format - UNITS: 1/s - -rate_stop_pulses: - <<: *default - CATDESC: Rate of stop pulses (/s). - FIELDNAM: rate_stop_pulses - LABLAXIS: rate stop pulses - # TODO: come back to format - UNITS: 1/s - -rate_coin_pulses: - <<: *default - CATDESC: Rate of coincidence pulses (/s). - FIELDNAM: rate_coin_pulses - LABLAXIS: rate coincidence pulses - # TODO: come back to format - UNITS: 1/s - -rate_processed_events: - <<: *default - CATDESC: Rate of processed events (/s). - FIELDNAM: rate_processed_events - LABLAXIS: rate processed events - # TODO: come back to format - UNITS: 1/s - -rate_rejected_events: - <<: *default - CATDESC: Rate of rejected events (/s). - FIELDNAM: rate_rejected_events - LABLAXIS: rate rejected events - # TODO: come back to format - UNITS: 1/s - -quality_hk: - <<: *default - CATDESC: Spin filter derived from Ultra housekeeping. Bitwise flagging used to filter the spin. - FIELDNAM: quality_hk - LABLAXIS: quality hk - # TODO: come back to format - UNITS: " " - -quality_attitude: - <<: *default - CATDESC: Spin filter derived from IMAP attitude. Bitwise flagging used to filter the spin. - FIELDNAM: quality_attitude - LABLAXIS: quality attitude - # TODO: come back to format - UNITS: " " - -quality_instruments: - <<: *default - CATDESC: Spin filter derived from instruments other than Ultra. Bitwise flagging used to filter the spin. - FIELDNAM: quality_instruments - LABLAXIS: quality instruments - # TODO: come back to format - UNITS: " " + UNITS: mm \ No newline at end of file diff --git a/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml deleted file mode 100644 index 802c305fc..000000000 --- a/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema_test.yaml +++ /dev/null @@ -1,246 +0,0 @@ -DOI: - description: > - DOI is a persistent Unique Digital Identifier with the form - https://doi.org// with the identifying the DOI - registration authority and the identifying the dataset. The DOI should point to - a landing page for additional information about the dataset. DOIs are typically created by - the SPASE naming authority or archive. - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true # include in validation - overwrite: false -Data_level: - description: > - This attribute is used in file name creation and records the level of processsing done - on the dataset. For HERMES the following are valid values: - - l0>Level 0 - - l1>Level 1 - - l2>Level 2 - - l3>Level 3 - - l4>Level 4 - - ql>Quicklook - default: null - required: true # NOT Required in ISTP Guide (Derived) - validate: false - overwrite: true -Data_product_descriptor: - description: > - This is an optional field that may not be needed for all products. Where it is used, identifier - should be short (e.q. 3-8 characters) descriptors that are helpful to end- users. If a - descriptor contains multiple components, underscores are used to separate those components. - default: null - required: false # NOT Required in ISTP Guide (Derived) - validate: false - overwrite: true -Data_type: - description: > - This attribute is used by CDF file writing software to create a filename. It is a - combination of the following filename components: mode, data level, and optional data - product descriptor. - default: null - required: false # NOT Required in ISTP Guide (Derived) - validate: false - overwrite: true -Data_version: - description: > - This attribute identifies the version of a particular CDF data file. - default: null - required: true - validate: true - overwrite: false -Descriptor: - description: > - This attribute identifies the name of the instrument or sensor that collected the data. Both - a long name and a short name are given. For any data file, only a single value is allowed. - For HERMES, the following are valid values: - - EEA>Electron Electrostatic Analyzer - - MERIT>Miniaturized Electron pRoton Telescope - - NEMISIS> Noise Eliminating Magnetometer In a Small Integrated System - - SPAN-I>Solar Probe Analyzer for Ions - default: null - required: true - validate: true - overwrite: false -Discipline: - description: > - This attribute describes both the science discipline and sub discipline. For HERMES, - this value should always be "Space Physics>Magnetospheric Science." - default: Space Physics>Magnetospheric Science - required: true - validate: true - overwrite: false -File_naming_convention: - description: > - If File_naming_convention was not set, it uses default setting: - source_datatype_descriptor_yyyyMMdd - default: source_datatype_descriptor_yyyyMMdd - required: false - validate: false - overwrite: true -Generation_date: - description: > - Date stamps the creation of the file using the syntax yyyymmdd, e.g., " - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true - overwrite: true -HTTP_LINK: - description: > - The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a - description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires - the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. - If text is not needed for these attributes, use an empty string "". - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true - overwrite: false -Instrument_mode: - description: > - TBS - default: null - required: false # NOT Required in ISTP Guide (Derived) - validate: false - overwrite: false -Instrument_type: - description: > - This attribute is used to facilitate making choices of instrument type. More than one entry - is allowed. Acceptable values for HERMES include: - - Magnetic Fields (space) - - Particles (space) - - Plasma and Solar Wind - - Ephemeris -> Ephemeris/Attitude/Ancillary - default: null - required: true - validate: true - overwrite: false -LINK_TEXT: - description: > - The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a - description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires - the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. - If text is not needed for these attributes, use an empty string "". - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true - overwrite: false -LINK_TITLE: - description: > - The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a - description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires - the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. - If text is not needed for these attributes, use an empty string "". - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true - overwrite: false -Logical_file_id: - description: > - This attribute stores the name of the CDF file but without the - file extension (e.g. ".cdf"). This attribute is required to avoid loss of the original source - in the case of accidental (or intentional) renaming. - default: null - required: true - validate: true - overwrite: true -Logical_source: - description: > - This attribute determines the file naming convention in the SKT Editor and is used by - CDA Web. It is composed of the following values: - - source_name - (e.g. spacecraft identifier) - - descriptor - (e.g. instrument identifier - see Section Error! Reference source not - found.) - - data_type - (e.g. mode, data level, and optional data product descriptor - value - come from 'Data_type' attribute) - default: null - required: true - validate: true - overwrite: true -Logical_source_description: - description: > - This attribute writes out the full words associated with the encrypted Logical_source - above, e.g., "Level 1 Dual Electron Spectrometer Survey Data". Users on CDAWeb see - this value on their website. - default: null - required: true - validate: true - overwrite: true -MODS: - description: > - This attribute is an SPDF standard global attribute, which is used to denote the history of - modifications made to the CDF data set. The MODS attribute should contain a - description of all significant changes to the data set, essentially capturing a log of high- - level release notes. This attribute can have as many entries as necessary and should be - updated if the Interface Number ("X") of the version number changes. - default: null - required: false # NOT Required in ISTP Guide (Recommended) - validate: true - overwrite: false -Mission_group: - description: > - This attribute has a single value and is used to facilitate making choices of source through - CDAWeb. This value should be "HERMES." - default: HERMES - required: true - validate: true - overwrite: false -PI_affiliation: - description: > - This attribute value should include the HERMES mission PI affiliation followed by a - comma-separated list of any Co-I affiliations that are responsible for this particular - dataset. The following are valid HERMES values, of which the abbreviations should be - used exclusively within this attribute value, and the full text of the affiliation included in - the general 'text' attribute as it is used solely in plot labels. - - GSFC - Goddard Space Flight Center - - UCB - University of California, Berkeley - - SSL - Space Sciences Laboratory, UCB - - UM - University of Michigan - default: null - required: true - validate: true - overwrite: false -PI_name: - description: > - This attribute value should include first initial and last name of the HERMES mission PI - followed by a comma-separated list of any Co-Is that are responsible for this particular - dataset. - default: null - required: true - validate: true - overwrite: false -Project: - description: > - This attribute identifies the name of the project and indicates ownership. For HERMES, - this value should be "STP>Solar-Terrestrial Physics". - default: STP>Solar-Terrestrial Physics - required: true - validate: true - overwrite: false -Source_name: - description: > - This attribute identifies the observatory where the data originated. The following are - valid values for HERMES: - - HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite - default: HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite - required: true - validate: true - overwrite: false -Start_time: - description: > - The start time of the contained data given in YYYYMMDD_hhmmss - default: null - required: false # NOT Required in ISTP Guide (Derived) - validate: false - overwrite: true -TEXT: - description: > - This attribute is an SPDF standard global attribute, which is a text description of the - experiment whose data is included in the CDF. A reference to a journal article(s) or to a - World Wide Web page describing the experiment is essential and constitutes the - minimum requirement. A written description of the data set is also desirable. This - attribute can have as many entries as necessary to contain the desired information. - Typically, this attribute is about a paragraph in length and is not shown on CDAWeb. - CDAWeb is the web portal for access to SPDF data, available at https://cdaweb.gsfc.nasa.gov. - default: null - required: true - validate: true - overwrite: false diff --git a/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml deleted file mode 100644 index 4adea4723..000000000 --- a/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema_test.yaml +++ /dev/null @@ -1,466 +0,0 @@ -# Based on : https://spdf.gsfc.nasa.gov/istp_guide/vattributes.html -# HERMES CDF Format Guide -attribute_key: - # ===== EPOCH-ONLY VARIABLE ATTRIBUTES ===== - TIME_BASE: - description: > - fixed (0AD, 1900, 1970 (POSIX), J2000 (used by CDF_TIME_TT2000), - 4714 BC (Julian)) or flexible (provider-defined) - required: true # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - RESOLUTION: - description: > - Using ISO8601 relative time format, for example: "1s" = 1 second. - Resolution provides the smallest change in time that is measured. - required: true # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - TIME_SCALE: - description: > - TT (same as TDT, used by CDF_TIME_TT2000), TAI (same as IAT, TT-32.184s), - UTC (includes leap seconds), TDB (same as SPICE ET), EME1950 [default: UTC] - required: true # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - REFERENCE_POSITION: - description: > - Topocenter (local), Geocenter , rotating Earth geoid (used by CDF_TIME_TT2000). - Reference_Position is optional metadata to account for time variance with position in - the gravity wells and with relative velocity. While we could use a combined - TimeSystem attribute that defines mission-specific time scales where needed, such as - UTC-at-STEREO-B, it's cleaner to keep them separate as Time_Scale=UTC and - Reference_Position=STEREO-B. - required: true # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - LEAP_SECONDS_INCLUDED: - description: > - comma-delimited list (within brackets) of leap seconds included in the form of a lists - of ISO8601 times when each leap second was added, appended with the size of the leap - second in ISO8601 relative time (+/- time, most commonly: "+1s") [default: standard - list of leap seconds up to time of data]. Leap_Seconds_Included is needed to account - for time scales that don't have all 34 (in 2009) leap seconds and for the clocks in - various countries that started using leap seconds at different times. The full list is - required to handle the equally or more common case where a time scale starts at a - pecific UTC but continues on without leap seconds in TAI mode; this is basically what - missions that don't add leap seconds are doing. - $ cat tai-utc.dat | awk 'ORS="," { val = $7 - prev } {prev = $7} { print $1$2"01+" val "s" }' - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - ABSOLUTE_ERROR: - description: > - Absolute or systematic error, in same units as Units attribute. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - RELATIVE_ERROR: - description: > - Relative or random error, in same units as Units attribute - to specify the accuracy - of the time stamps relative to each other. This is usually much smaller than Absolute_Error. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - BIN_LOCATION: - description: > - relative position of time stamp to the data measurement bin, with 0.0 at the beginning - of time bin and 1.0 at the end. Default is 0.5 for the time at the center of the data - measurement. Since clock readings are usually truncated, the real value may be closer to 0.0. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - # ===== DATA VARIABLE ATTRIBUTES ===== - CATDESC: - description: > - This is a human readable description of the data variable. Generally, this is an 80- - character string which describes the variable and what it depends on. - required: true - overwrite: false - valid_values: null - alternate: null - DELTA_MINUS_VAR: - description: > - DEPEND_i variables are typically physical values along the corresponding i-th - dimension of the parent data variable, such as energy levels or spectral frequencies. The - discreet set of values are located with respect to the sampling bin by - DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name - containing the distance from the value to the bin edge. It is strongly recommended that - HERMES DEPEND_i variables include DELTA_PLUS_VAR and - DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located - elsewhere in the CDF file. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - DELTA_PLUS_VAR: - description: > - DEPEND_i variables are typically physical values along the corresponding i-th - dimension of the parent data variable, such as energy levels or spectral frequencies. The - discreet set of values are located with respect to the sampling bin by - DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name - containing the distance from the value to the bin edge. It is strongly recommended that - HERMES DEPEND_i variables include DELTA_PLUS_VAR and - DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located - elsewhere in the CDF file. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - DEPEND_0: - description: > - Explicitly ties a data variable to the time variable on which it depends. All variables - which change with time must have a DEPEND_0 attribute defined. See section 5.2.1 - which specifies the HERMES usage of DEPEND_0. - required: true - overwrite: false - valid_values: null - alternate: null - DEPEND_i: # For i'th dimensional data variables - description: > - Ties a dimensional data variable to a SUPPORT_DATA variable on which the i-th - dimension of the data variable depends. The number of DEPEND attributes must match - the dimensionality of the variable, i.e., a one-dimensional variable must have a - DEPEND_1, a two-dimensional variable must have a DEPEND_1 and a DEPEND_2 - attribute, etc. The value of the attribute must be a variable in the same CDF data set. It is - strongly recommended that DEPEND_i variables hold values in physical units. - DEPEND_i variables also require their own attributes, as described in section 5.1.4. - required: false - overwrite: false - valid_values: null - alternate: null - DISPLAY_TYPE: - description: > - This tells automated software, such as CDAWeb, how the data should be displayed. - required: true - overwrite: false - valid_values: - time_series - time_series>noerrorbars - spectrogram - stack_plot - image - alternate: null - FIELDNAM: - description: > - A shortened version of CATDESC which can be used to label a plot axis or as a data - listing heading. This is a string, up to ~30 characters in length. - required: true - overwrite: false - valid_values: null - alternate: null - FILLVAL: - description: > - Identifies the fill value used where data values are known to be bad or missing. - FILLVAL is required for time-varying variables. Fill data are always non-valid data. The - ISTP standard fill values are listed in Table 5-4. - required: true - overwrite: false - valid_values: null - alternate: null - FORMAT: # NOTE Only one of FORMAT or FORM_PTR should be present - description: > - This field allows software to properly format the associated data when displayed on a - screen or output to a file. Format can be specified using either Fortran or C format codes. - For instance, "F10.3" indicates that the data should be displayed across 10 characters - where 3 of those characters are to the right of the decimal. For a description of FORTRAN - formatting codes see the docs here: - https://docs.oracle.com/cd/E19957-01/805-4939/z40007437a2e/index.html - required: true - overwrite: false - valid_values: null - alternate: FORM_PTR - FORM_PTR: - description: > - The value of this field is a variable which stores the character string that represents the - desired output format for the associated data. - required: false - overwrite: false - valid_values: null - alternate: FORMAT - LABLAXIS: # NOTE Only one of LABLAXIS or LABL_PTR_i should be present - description: > - Used to label a plot axis or to provide a heading for a data listing. This field is generally - 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. - required: true - overwrite: false - valid_values: null - alternate: LABL_PTR_1 - LABL_PTR_i: - description: > - Used to label a dimensional variable when one value of LABLAXIS is not sufficient to - describe the variable or to label all the axes. LABL_PTR_i is used instead of - LABLAXIS, where i can take on any value from 1 to n where n is the total number of - dimensions of the original variable. The value of LABL_PTR_1 is a variable which will - contain the short character strings which describe the first dimension of the original - variable. The value of the attribute must be a variable in the same CDF data set and is - generally 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. - required: false - overwrite: false - valid_values: null - alternate: LABLAXIS - SI_CONVERSION: - description: > - The conversion factor to SI units. This is the factor that the variable must be multiplied - by in order to convert it to generic SI units. This parameter contains two text fields - separated by the ">" delimiter. The first component is the conversion factor and the - second is the standard SI unit. Units are defined according to their standard SI symbols - (ie. Tesla = T, Newtons = N, Meters = m, etc.) For data variables that are inherently - unitless, and thus lack a conversion factor, this data attribute will be " > " where ' ' is - a blank space and the quotation marks are not included. Units which are not conveniently - transformed into SI should follow the blank syntax " > " described above. - required: false # NOT Required in ISTP Guide - overwrite: false - valid_values: null - alternate: null - UNITS: # NOTE Only one of UNITS or UNIT_PTR should be present - description: > - A 6-20 character string that identifies the units of the variable (e.g. nT for magnetic - field). Use a blank character, rather than "None" or "unitless", for variables that have no - units (e.g., a ratio or a direction cosine). An active list of HERMES standard UNITS and - their SI_CONVERSIONs is maintained on the mission web-pages at - https://lasp.colorado.edu/galaxy/display/HERMES/Units+of+Measure, accessible via the - HERMES Science Working Team pages. Those pages also lay out the rules for - formatting the UNITS string. - required: true - overwrite: false - valid_values: null - alternate: UNIT_PTR - UNIT_PTR: - description: > - The value of this field is a variable which stores short character strings which identify the - units of the variable. Use a blank character, rather than "None" or "unitless", for - variables that have no units (e.g., a ratio or a direction cosine). The value of this attribute - must be a variable in the same CDF data set. - required: false - overwrite: false - valid_values: null - alternate: UNITS - VALIDMIN: - description: > - The minimum value for a particular variable that is expected over the lifetime of the - mission. Used by application software to filter out values that are out of range. The - value must match the data type of the variable. - required: true - overwrite: false - valid_values: null - alternate: null - VALIDMAX: - description: > - The maximum value for a particular variable that is expected over the lifetime of the - mission. Used by application software to filter out values that are out of range. The - value must match the data type of the variable. - required: true - overwrite: false - valid_values: null - alternate: null - VAR_TYPE: - description: > - Used in CDAWeb to indicate if the data should be used directly by users. - required: true - overwrite: false - valid_values: - data - support_data - metadata - ignore_data - alternate: null - COORDINATE_SYSTEM: - description: > - All variables for which the values are dependent on the system of coordinates are - strongly recommended to have this attribute. This includes both full vectors, tensors, etc. - or individual values, e.g. of an angle with respect to some axis. The attribute is a text - string which takes the form: "XXX[>optional long name]" - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - TENSOR_ORDER: - description: > - All variables which hold physical vectors, tensors, etc., or sub-parts thereof, are strongly - recommended to have their tensorial properties held by this numerical value. Vectors - have TENSOR_ORDER=1, pressure tensors have TENSOR_ORDER=2, etc. Variables - which hold single components or sub-parts of a vector or tensor, e.g., the x-component of - velocity or the three diagonal elements of a tensor, use this attribute to establish the - underlying object from which they are extracted. TENSOR_ORDER is a number, usually - held as a CDF_INT4, rather than a character string. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - REPRESENTATION_i: - description: > - This strongly recommended attribute holds the way vector or tensor variables are held, - e.g., as Cartesian or polar forms, and their sequence order in the dimension i in which - they are held. Cartesians are indicated by x,y,z; polar coordinates by r (magnitude), t - (theta - from z-axis), p (phi - longitude or azimuth around z-axis from x axis), l (lambda - = latitude). Examples follow. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - OPERATOR_TYPE: - description: > - This has been introduced to describe HERMES quaternions (see Section 5.2 below). It - has allowed values "UNIT_QUATERNION" or "ROTATION_MATRIX" although other - values could be added. Unit quaternions correspond to pure spatial rotations. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - WCSAXES: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. - The value field shall contain a non-negative integer no - greater than 999, representing the number of axes in the associated - data array. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - MJDREF: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. The value shall contain a floating - point number representing the reference time position of the time stamps along the - 0'th axis of the measurement. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - TIMEUNIT: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. The value shall contain a character - string giving the units of the time stamps along the 0'th axis of the measurement. - The TIMEUNIT should match the CUNITi along the time axis of the measurement - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - TIMEDEL: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. The value shall contain a floating - point number representing the resolution of the time stamps along the 0'th axis of - the measurement. The TIMEDEL should match the CRDELi along the time axis of the - measurement. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CNAMEi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value shall contain a charachter string represnting the name of the i'th axis. - The name is used for comment/documentation purposes only and is not used as a part - of the i'th axis coordinate transformations. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CTYPEi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value field shall contain a character string, giving - the name of the coordinate represented by axis i. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CUNITi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value shall be the units along axis i, compatible with CTYPEi to be used for - scaling and coordinate transformations along the i'th axis. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CRPIXi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value field shall contain a floating point number, - identifying the location of a reference point along axis i, in units of - the axis index. This value is based upon a counter that runs from 1 to - NAXISn with an increment of 1 per pixel. The reference point value - need not be that for the center of a pixel nor lie within the actual - data array. Use comments to indicate the location of the index point - relative to the pixel. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CRVALi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value field shall contain a floating point number, - giving the value of the coordinate specified by the CTYPEn keyword at - the reference point CRPIXi. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null - CDELTi: - description: > - This is a FITS WCS Keyword being repurposed for handling WCS transformations with - high-dimensional or spectral CDF data variables. This metadata attribte should be - used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. - The value field shall contain a floating point number - giving the partial derivative of the coordinate specified by the CTYPEi - keywords with respect to the pixel index, evaluated at the reference - point CRPIXi, in units of the coordinate specified by the CTYPEi - keyword. - required: false # NOT Required in HERMES Format Guide - overwrite: false - valid_values: null - alternate: null -data: - - CATDESC - - DEPEND_0 - - DISPLAY_TYPE - - FIELDNAM - - FILLVAL - - FORMAT - - LABLAXIS - - SI_CONVERSION - - UNITS - - VALIDMIN - - VALIDMAX - - VAR_TYPE -support_data: - - CATDESC - - FIELDNAM - - FILLVAL - - FORMAT - - LABLAXIS - - SI_CONVERSION - - UNITS - - VALIDMIN - - VALIDMAX - - VAR_TYPE -metadata: - - CATDESC - - FIELDNAM - - FILLVAL - - FORMAT - - VAR_TYPE diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 01a19f001..4686dbce6 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -156,6 +156,25 @@ def test_get_global_attributes(): # "Data_type" not required according to default schema assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" + # Testing that required schema keys are in get_global_attributes + for attr_name in cdf_manager.global_attribute_schema.keys(): + required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] + if required_schema is True: + assert attr_name in test_get_global_attrs.keys() + + +def test_instrument_id_format(): + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Loading in instrument specific attributes + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + # Testing how instrument_id operates assert test_get_global_attrs["Project"] == cdf_manager.global_attributes["Project"] assert ( @@ -173,15 +192,6 @@ def test_get_global_attributes(): with pytest.raises(KeyError): assert cdf_manager.global_attributes["imap_test_T1_test"]["Project"] - # Trying to update a default global using get_global_attributes does not work. - # For example, thing about DOI event. - - # Testing that required schema keys are in get_global_attributes - for attr_name in cdf_manager.global_attribute_schema.keys(): - required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] - if required_schema is True: - assert attr_name in test_get_global_attrs.keys() - def test_add_global_attribute(): # Initialize CdfAttributeManager object which loads in default info @@ -189,7 +199,6 @@ def test_add_global_attribute(): # Change filepath to load test global attributes cdf_manager.source_dir = Path(__file__).parent.parent / "tests" - cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") cdf_manager.load_global_attributes("imap_test_global.yaml") # Changing a dynamic global variable @@ -198,6 +207,26 @@ def test_add_global_attribute(): test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") assert test_get_global_attrs["Project"] == "Test Project" + # Testing adding required global attribute + # TODO: Assert that get_global_attiribute fails + cdf_manager.global_attributes.__delitem__("Source_name") + with pytest.raises(KeyError): + assert ( + cdf_manager.global_attributes["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + + # The following code does not throw an error. Instead, Source_name is just "None" + # assert cdf_manager.get_global_attributes("imap_test_T1_test") is None + + cdf_manager.add_global_attribute("Source_name", "anas_source") + assert cdf_manager.global_attributes["Source_name"] == "anas_source" + + # TODO: Should I test this for instrument attributes like logical_source? + # The code below does NOT work. The logical_source is not deleted + # test_get_global_attrs.__delitem__("Logical_source") + # assert cdf_manager.get_global_attributes("imap_test_T1_test") is None + def test_variable_attribute(): """ @@ -234,8 +263,7 @@ def test_variable_attribute(): if attribute["required"] is True: for exp_attr in expected_attributes: assert ( - exp_attr - in cdf_manager.variable_attribute_schema["attribute_key"].keys() + exp_attr in cdf_manager.variable_attribute_schema["attribute_key"] ) # Testing specific things From f446db3496fdffe21b4793b4fec0b7db9b555020 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Thu, 20 Jun 2024 20:35:11 -0600 Subject: [PATCH 32/40] More work done --- imap_processing/cdf/imap_cdf_manager.py | 5 ----- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 9 --------- 2 files changed, 14 deletions(-) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 669ac3e40..19ba84ba5 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -20,11 +20,6 @@ def __init__(self, source_dir=None): else: super().__init__(Path(__file__).parent / "config") self.source_dir = Path(__file__).parent / source_dir - # if source_dir_input is None: - # super().__init__(Path(__file__).parent / source_dir_input - # else: - # super().__init__(Path(__file__).parent / "config") - # self.source_dir = Path(__file__).parent / source_dir_input def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 4686dbce6..8019100bc 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -208,7 +208,6 @@ def test_add_global_attribute(): assert test_get_global_attrs["Project"] == "Test Project" # Testing adding required global attribute - # TODO: Assert that get_global_attiribute fails cdf_manager.global_attributes.__delitem__("Source_name") with pytest.raises(KeyError): assert ( @@ -216,17 +215,9 @@ def test_add_global_attribute(): == "IMAP>Interstellar Mapping and Acceleration Probe" ) - # The following code does not throw an error. Instead, Source_name is just "None" - # assert cdf_manager.get_global_attributes("imap_test_T1_test") is None - cdf_manager.add_global_attribute("Source_name", "anas_source") assert cdf_manager.global_attributes["Source_name"] == "anas_source" - # TODO: Should I test this for instrument attributes like logical_source? - # The code below does NOT work. The logical_source is not deleted - # test_get_global_attrs.__delitem__("Logical_source") - # assert cdf_manager.get_global_attributes("imap_test_T1_test") is None - def test_variable_attribute(): """ From 67271cc82a16edd8624d63c3b0c75c394183e731 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Fri, 21 Jun 2024 08:39:04 -0600 Subject: [PATCH 33/40] Cleaning up --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 8019100bc..454e19f4a 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -210,10 +210,8 @@ def test_add_global_attribute(): # Testing adding required global attribute cdf_manager.global_attributes.__delitem__("Source_name") with pytest.raises(KeyError): - assert ( - cdf_manager.global_attributes["Source_name"] - == "IMAP>Interstellar Mapping and Acceleration Probe" - ) + assert cdf_manager.global_attributes["Source_name"] == "Wrong" + assert cdf_manager.global_attributes["Source_name"] is None cdf_manager.add_global_attribute("Source_name", "anas_source") assert cdf_manager.global_attributes["Source_name"] == "anas_source" @@ -257,7 +255,7 @@ def test_variable_attribute(): exp_attr in cdf_manager.variable_attribute_schema["attribute_key"] ) - # Testing specific things + # Testing specific attributes assert ( cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] == cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] From 0bff645113de40e149bfbce6ebe1f2a96b75c5c5 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Fri, 21 Jun 2024 14:03:00 -0600 Subject: [PATCH 34/40] Figured out Path issue --- imap_processing/cdf/imap_cdf_manager.py | 3 +-- .../cdf/tests/test_cdf_attribute_manager.py | 22 ++++++++++++++----- .../cdf/tests/test_imap_cdf_manager.py | 14 +++++++----- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 19ba84ba5..57e3bef2a 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -18,8 +18,7 @@ def __init__(self, source_dir=None): if source_dir is None: super().__init__(Path(__file__).parent / "config") else: - super().__init__(Path(__file__).parent / "config") - self.source_dir = Path(__file__).parent / source_dir + super().__init__(Path(__file__).parent / source_dir) def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 454e19f4a..da94cd3fb 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -203,19 +203,31 @@ def test_add_global_attribute(): # Changing a dynamic global variable cdf_manager.add_global_attribute("Project", "Test Project") - assert cdf_manager.global_attributes["Project"] == "Test Project" test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + assert cdf_manager.global_attributes["Project"] == "Test Project" assert test_get_global_attrs["Project"] == "Test Project" - # Testing adding required global attribute + # Testing adding required deleted global attribute cdf_manager.global_attributes.__delitem__("Source_name") + # Reloading get_global_attributes to pick up deleted Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") with pytest.raises(KeyError): - assert cdf_manager.global_attributes["Source_name"] == "Wrong" - assert cdf_manager.global_attributes["Source_name"] is None - + assert cdf_manager.global_attributes["Source_name"] == "Does_not_exist" + assert test_get_global_attrs["Source_name"] is None + # Adding deleted global attribute cdf_manager.add_global_attribute("Source_name", "anas_source") assert cdf_manager.global_attributes["Source_name"] == "anas_source" + # Testing instrument specific data + cdf_manager.global_attributes["imap_test_T1_test"].__delitem__("Logical_source") + # Reloading get_global_attributes to pick up deleted Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + with pytest.raises(KeyError): + assert ( + cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] is None + ) + assert test_get_global_attrs["Logical_source"] is None + def test_variable_attribute(): """ diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 620ad7c88..dd2473d7f 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -1,11 +1,13 @@ -# from pathlib import Path +from pathlib import Path + # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes def test_add_instrument_global_attrs(): - # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes("tests") + # Create an ImapCdfAttributes object, set to correct file path + imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" imap_cdf_manager.add_instrument_global_attrs("instrument") # Testing data loaded in @@ -24,7 +26,8 @@ def test_add_instrument_global_attrs(): def testing_source_dir(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes("tests") + imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" assert ( str(imap_cdf_manager.source_dir) == "/Users/anma6676/Desktop/Repositories/imap_processing" @@ -34,7 +37,8 @@ def testing_source_dir(): def test_add_instrument_variable_attrs(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes("tests") + imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" imap_cdf_manager.add_instrument_variable_attrs("instrument", "level") # Testing the actual function From e1644bf185769ee294f1fa7cdac0f816437089c6 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Fri, 21 Jun 2024 14:19:31 -0600 Subject: [PATCH 35/40] Final touches (again) --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index da94cd3fb..97b959b08 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -32,9 +32,6 @@ def test_default_attr_schema(): ) -# @pytest.mark.xfail(reason="Missing IMAP specific global schema") - - def test_global_attribute(): """ Test function that covers: @@ -207,7 +204,7 @@ def test_add_global_attribute(): assert cdf_manager.global_attributes["Project"] == "Test Project" assert test_get_global_attrs["Project"] == "Test Project" - # Testing adding required deleted global attribute + # Testing adding required global attribute cdf_manager.global_attributes.__delitem__("Source_name") # Reloading get_global_attributes to pick up deleted Source_name test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") @@ -218,7 +215,7 @@ def test_add_global_attribute(): cdf_manager.add_global_attribute("Source_name", "anas_source") assert cdf_manager.global_attributes["Source_name"] == "anas_source" - # Testing instrument specific data + # Testing instrument specific attribute cdf_manager.global_attributes["imap_test_T1_test"].__delitem__("Logical_source") # Reloading get_global_attributes to pick up deleted Source_name test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") From 1029e2dabd39497c08961f9049674b7703684551 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 24 Jun 2024 08:51:07 -0600 Subject: [PATCH 36/40] Comment mods --- imap_processing/cdf/imap_cdf_manager.py | 12 +- .../default_global_cdf_attrs_schema.yaml | 246 +++++++++ .../default_variable_cdf_attrs_schema.yaml | 466 ++++++++++++++++++ .../cdf/tests/test_cdf_attribute_manager.py | 8 +- .../cdf/tests/test_imap_cdf_manager.py | 6 +- 5 files changed, 730 insertions(+), 8 deletions(-) create mode 100644 imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml create mode 100644 imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 57e3bef2a..440e10f0c 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -14,11 +14,19 @@ class ImapCdfAttributes(CdfAttributeManager): """Contains IMAP specific tools and settings for CDF management.""" def __init__(self, source_dir=None): - """Set the path to the config directory.""" + """ + + Set the path to the config directory. + + Parameters + ---------- + source_dir + Source directory. + """ if source_dir is None: super().__init__(Path(__file__).parent / "config") else: - super().__init__(Path(__file__).parent / source_dir) + super().__init__(source_dir) def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml new file mode 100644 index 000000000..802c305fc --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml @@ -0,0 +1,246 @@ +DOI: + description: > + DOI is a persistent Unique Digital Identifier with the form + https://doi.org// with the identifying the DOI + registration authority and the identifying the dataset. The DOI should point to + a landing page for additional information about the dataset. DOIs are typically created by + the SPASE naming authority or archive. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true # include in validation + overwrite: false +Data_level: + description: > + This attribute is used in file name creation and records the level of processsing done + on the dataset. For HERMES the following are valid values: + - l0>Level 0 + - l1>Level 1 + - l2>Level 2 + - l3>Level 3 + - l4>Level 4 + - ql>Quicklook + default: null + required: true # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_product_descriptor: + description: > + This is an optional field that may not be needed for all products. Where it is used, identifier + should be short (e.q. 3-8 characters) descriptors that are helpful to end- users. If a + descriptor contains multiple components, underscores are used to separate those components. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_type: + description: > + This attribute is used by CDF file writing software to create a filename. It is a + combination of the following filename components: mode, data level, and optional data + product descriptor. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_version: + description: > + This attribute identifies the version of a particular CDF data file. + default: null + required: true + validate: true + overwrite: false +Descriptor: + description: > + This attribute identifies the name of the instrument or sensor that collected the data. Both + a long name and a short name are given. For any data file, only a single value is allowed. + For HERMES, the following are valid values: + - EEA>Electron Electrostatic Analyzer + - MERIT>Miniaturized Electron pRoton Telescope + - NEMISIS> Noise Eliminating Magnetometer In a Small Integrated System + - SPAN-I>Solar Probe Analyzer for Ions + default: null + required: true + validate: true + overwrite: false +Discipline: + description: > + This attribute describes both the science discipline and sub discipline. For HERMES, + this value should always be "Space Physics>Magnetospheric Science." + default: Space Physics>Magnetospheric Science + required: true + validate: true + overwrite: false +File_naming_convention: + description: > + If File_naming_convention was not set, it uses default setting: + source_datatype_descriptor_yyyyMMdd + default: source_datatype_descriptor_yyyyMMdd + required: false + validate: false + overwrite: true +Generation_date: + description: > + Date stamps the creation of the file using the syntax yyyymmdd, e.g., " + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: true +HTTP_LINK: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Instrument_mode: + description: > + TBS + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: false +Instrument_type: + description: > + This attribute is used to facilitate making choices of instrument type. More than one entry + is allowed. Acceptable values for HERMES include: + - Magnetic Fields (space) + - Particles (space) + - Plasma and Solar Wind + - Ephemeris -> Ephemeris/Attitude/Ancillary + default: null + required: true + validate: true + overwrite: false +LINK_TEXT: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +LINK_TITLE: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Logical_file_id: + description: > + This attribute stores the name of the CDF file but without the + file extension (e.g. ".cdf"). This attribute is required to avoid loss of the original source + in the case of accidental (or intentional) renaming. + default: null + required: true + validate: true + overwrite: true +Logical_source: + description: > + This attribute determines the file naming convention in the SKT Editor and is used by + CDA Web. It is composed of the following values: + - source_name - (e.g. spacecraft identifier) + - descriptor - (e.g. instrument identifier - see Section Error! Reference source not + found.) + - data_type - (e.g. mode, data level, and optional data product descriptor - value + come from 'Data_type' attribute) + default: null + required: true + validate: true + overwrite: true +Logical_source_description: + description: > + This attribute writes out the full words associated with the encrypted Logical_source + above, e.g., "Level 1 Dual Electron Spectrometer Survey Data". Users on CDAWeb see + this value on their website. + default: null + required: true + validate: true + overwrite: true +MODS: + description: > + This attribute is an SPDF standard global attribute, which is used to denote the history of + modifications made to the CDF data set. The MODS attribute should contain a + description of all significant changes to the data set, essentially capturing a log of high- + level release notes. This attribute can have as many entries as necessary and should be + updated if the Interface Number ("X") of the version number changes. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Mission_group: + description: > + This attribute has a single value and is used to facilitate making choices of source through + CDAWeb. This value should be "HERMES." + default: HERMES + required: true + validate: true + overwrite: false +PI_affiliation: + description: > + This attribute value should include the HERMES mission PI affiliation followed by a + comma-separated list of any Co-I affiliations that are responsible for this particular + dataset. The following are valid HERMES values, of which the abbreviations should be + used exclusively within this attribute value, and the full text of the affiliation included in + the general 'text' attribute as it is used solely in plot labels. + - GSFC - Goddard Space Flight Center + - UCB - University of California, Berkeley + - SSL - Space Sciences Laboratory, UCB + - UM - University of Michigan + default: null + required: true + validate: true + overwrite: false +PI_name: + description: > + This attribute value should include first initial and last name of the HERMES mission PI + followed by a comma-separated list of any Co-Is that are responsible for this particular + dataset. + default: null + required: true + validate: true + overwrite: false +Project: + description: > + This attribute identifies the name of the project and indicates ownership. For HERMES, + this value should be "STP>Solar-Terrestrial Physics". + default: STP>Solar-Terrestrial Physics + required: true + validate: true + overwrite: false +Source_name: + description: > + This attribute identifies the observatory where the data originated. The following are + valid values for HERMES: + - HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + default: HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + required: true + validate: true + overwrite: false +Start_time: + description: > + The start time of the contained data given in YYYYMMDD_hhmmss + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +TEXT: + description: > + This attribute is an SPDF standard global attribute, which is a text description of the + experiment whose data is included in the CDF. A reference to a journal article(s) or to a + World Wide Web page describing the experiment is essential and constitutes the + minimum requirement. A written description of the data set is also desirable. This + attribute can have as many entries as necessary to contain the desired information. + Typically, this attribute is about a paragraph in length and is not shown on CDAWeb. + CDAWeb is the web portal for access to SPDF data, available at https://cdaweb.gsfc.nasa.gov. + default: null + required: true + validate: true + overwrite: false diff --git a/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml new file mode 100644 index 000000000..4adea4723 --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml @@ -0,0 +1,466 @@ +# Based on : https://spdf.gsfc.nasa.gov/istp_guide/vattributes.html +# HERMES CDF Format Guide +attribute_key: + # ===== EPOCH-ONLY VARIABLE ATTRIBUTES ===== + TIME_BASE: + description: > + fixed (0AD, 1900, 1970 (POSIX), J2000 (used by CDF_TIME_TT2000), + 4714 BC (Julian)) or flexible (provider-defined) + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RESOLUTION: + description: > + Using ISO8601 relative time format, for example: "1s" = 1 second. + Resolution provides the smallest change in time that is measured. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + TIME_SCALE: + description: > + TT (same as TDT, used by CDF_TIME_TT2000), TAI (same as IAT, TT-32.184s), + UTC (includes leap seconds), TDB (same as SPICE ET), EME1950 [default: UTC] + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + REFERENCE_POSITION: + description: > + Topocenter (local), Geocenter , rotating Earth geoid (used by CDF_TIME_TT2000). + Reference_Position is optional metadata to account for time variance with position in + the gravity wells and with relative velocity. While we could use a combined + TimeSystem attribute that defines mission-specific time scales where needed, such as + UTC-at-STEREO-B, it's cleaner to keep them separate as Time_Scale=UTC and + Reference_Position=STEREO-B. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + LEAP_SECONDS_INCLUDED: + description: > + comma-delimited list (within brackets) of leap seconds included in the form of a lists + of ISO8601 times when each leap second was added, appended with the size of the leap + second in ISO8601 relative time (+/- time, most commonly: "+1s") [default: standard + list of leap seconds up to time of data]. Leap_Seconds_Included is needed to account + for time scales that don't have all 34 (in 2009) leap seconds and for the clocks in + various countries that started using leap seconds at different times. The full list is + required to handle the equally or more common case where a time scale starts at a + pecific UTC but continues on without leap seconds in TAI mode; this is basically what + missions that don't add leap seconds are doing. + $ cat tai-utc.dat | awk 'ORS="," { val = $7 - prev } {prev = $7} { print $1$2"01+" val "s" }' + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + ABSOLUTE_ERROR: + description: > + Absolute or systematic error, in same units as Units attribute. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RELATIVE_ERROR: + description: > + Relative or random error, in same units as Units attribute - to specify the accuracy + of the time stamps relative to each other. This is usually much smaller than Absolute_Error. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + BIN_LOCATION: + description: > + relative position of time stamp to the data measurement bin, with 0.0 at the beginning + of time bin and 1.0 at the end. Default is 0.5 for the time at the center of the data + measurement. Since clock readings are usually truncated, the real value may be closer to 0.0. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + # ===== DATA VARIABLE ATTRIBUTES ===== + CATDESC: + description: > + This is a human readable description of the data variable. Generally, this is an 80- + character string which describes the variable and what it depends on. + required: true + overwrite: false + valid_values: null + alternate: null + DELTA_MINUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DELTA_PLUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DEPEND_0: + description: > + Explicitly ties a data variable to the time variable on which it depends. All variables + which change with time must have a DEPEND_0 attribute defined. See section 5.2.1 + which specifies the HERMES usage of DEPEND_0. + required: true + overwrite: false + valid_values: null + alternate: null + DEPEND_i: # For i'th dimensional data variables + description: > + Ties a dimensional data variable to a SUPPORT_DATA variable on which the i-th + dimension of the data variable depends. The number of DEPEND attributes must match + the dimensionality of the variable, i.e., a one-dimensional variable must have a + DEPEND_1, a two-dimensional variable must have a DEPEND_1 and a DEPEND_2 + attribute, etc. The value of the attribute must be a variable in the same CDF data set. It is + strongly recommended that DEPEND_i variables hold values in physical units. + DEPEND_i variables also require their own attributes, as described in section 5.1.4. + required: false + overwrite: false + valid_values: null + alternate: null + DISPLAY_TYPE: + description: > + This tells automated software, such as CDAWeb, how the data should be displayed. + required: true + overwrite: false + valid_values: + time_series + time_series>noerrorbars + spectrogram + stack_plot + image + alternate: null + FIELDNAM: + description: > + A shortened version of CATDESC which can be used to label a plot axis or as a data + listing heading. This is a string, up to ~30 characters in length. + required: true + overwrite: false + valid_values: null + alternate: null + FILLVAL: + description: > + Identifies the fill value used where data values are known to be bad or missing. + FILLVAL is required for time-varying variables. Fill data are always non-valid data. The + ISTP standard fill values are listed in Table 5-4. + required: true + overwrite: false + valid_values: null + alternate: null + FORMAT: # NOTE Only one of FORMAT or FORM_PTR should be present + description: > + This field allows software to properly format the associated data when displayed on a + screen or output to a file. Format can be specified using either Fortran or C format codes. + For instance, "F10.3" indicates that the data should be displayed across 10 characters + where 3 of those characters are to the right of the decimal. For a description of FORTRAN + formatting codes see the docs here: + https://docs.oracle.com/cd/E19957-01/805-4939/z40007437a2e/index.html + required: true + overwrite: false + valid_values: null + alternate: FORM_PTR + FORM_PTR: + description: > + The value of this field is a variable which stores the character string that represents the + desired output format for the associated data. + required: false + overwrite: false + valid_values: null + alternate: FORMAT + LABLAXIS: # NOTE Only one of LABLAXIS or LABL_PTR_i should be present + description: > + Used to label a plot axis or to provide a heading for a data listing. This field is generally + 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: true + overwrite: false + valid_values: null + alternate: LABL_PTR_1 + LABL_PTR_i: + description: > + Used to label a dimensional variable when one value of LABLAXIS is not sufficient to + describe the variable or to label all the axes. LABL_PTR_i is used instead of + LABLAXIS, where i can take on any value from 1 to n where n is the total number of + dimensions of the original variable. The value of LABL_PTR_1 is a variable which will + contain the short character strings which describe the first dimension of the original + variable. The value of the attribute must be a variable in the same CDF data set and is + generally 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: false + overwrite: false + valid_values: null + alternate: LABLAXIS + SI_CONVERSION: + description: > + The conversion factor to SI units. This is the factor that the variable must be multiplied + by in order to convert it to generic SI units. This parameter contains two text fields + separated by the ">" delimiter. The first component is the conversion factor and the + second is the standard SI unit. Units are defined according to their standard SI symbols + (ie. Tesla = T, Newtons = N, Meters = m, etc.) For data variables that are inherently + unitless, and thus lack a conversion factor, this data attribute will be " > " where ' ' is + a blank space and the quotation marks are not included. Units which are not conveniently + transformed into SI should follow the blank syntax " > " described above. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + UNITS: # NOTE Only one of UNITS or UNIT_PTR should be present + description: > + A 6-20 character string that identifies the units of the variable (e.g. nT for magnetic + field). Use a blank character, rather than "None" or "unitless", for variables that have no + units (e.g., a ratio or a direction cosine). An active list of HERMES standard UNITS and + their SI_CONVERSIONs is maintained on the mission web-pages at + https://lasp.colorado.edu/galaxy/display/HERMES/Units+of+Measure, accessible via the + HERMES Science Working Team pages. Those pages also lay out the rules for + formatting the UNITS string. + required: true + overwrite: false + valid_values: null + alternate: UNIT_PTR + UNIT_PTR: + description: > + The value of this field is a variable which stores short character strings which identify the + units of the variable. Use a blank character, rather than "None" or "unitless", for + variables that have no units (e.g., a ratio or a direction cosine). The value of this attribute + must be a variable in the same CDF data set. + required: false + overwrite: false + valid_values: null + alternate: UNITS + VALIDMIN: + description: > + The minimum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VALIDMAX: + description: > + The maximum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VAR_TYPE: + description: > + Used in CDAWeb to indicate if the data should be used directly by users. + required: true + overwrite: false + valid_values: + data + support_data + metadata + ignore_data + alternate: null + COORDINATE_SYSTEM: + description: > + All variables for which the values are dependent on the system of coordinates are + strongly recommended to have this attribute. This includes both full vectors, tensors, etc. + or individual values, e.g. of an angle with respect to some axis. The attribute is a text + string which takes the form: "XXX[>optional long name]" + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TENSOR_ORDER: + description: > + All variables which hold physical vectors, tensors, etc., or sub-parts thereof, are strongly + recommended to have their tensorial properties held by this numerical value. Vectors + have TENSOR_ORDER=1, pressure tensors have TENSOR_ORDER=2, etc. Variables + which hold single components or sub-parts of a vector or tensor, e.g., the x-component of + velocity or the three diagonal elements of a tensor, use this attribute to establish the + underlying object from which they are extracted. TENSOR_ORDER is a number, usually + held as a CDF_INT4, rather than a character string. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + REPRESENTATION_i: + description: > + This strongly recommended attribute holds the way vector or tensor variables are held, + e.g., as Cartesian or polar forms, and their sequence order in the dimension i in which + they are held. Cartesians are indicated by x,y,z; polar coordinates by r (magnitude), t + (theta - from z-axis), p (phi - longitude or azimuth around z-axis from x axis), l (lambda + = latitude). Examples follow. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + OPERATOR_TYPE: + description: > + This has been introduced to describe HERMES quaternions (see Section 5.2 below). It + has allowed values "UNIT_QUATERNION" or "ROTATION_MATRIX" although other + values could be added. Unit quaternions correspond to pure spatial rotations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + WCSAXES: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. + The value field shall contain a non-negative integer no + greater than 999, representing the number of axes in the associated + data array. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + MJDREF: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the reference time position of the time stamps along the + 0'th axis of the measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEUNIT: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a character + string giving the units of the time stamps along the 0'th axis of the measurement. + The TIMEUNIT should match the CUNITi along the time axis of the measurement + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEDEL: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the resolution of the time stamps along the 0'th axis of + the measurement. The TIMEDEL should match the CRDELi along the time axis of the + measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CNAMEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall contain a charachter string represnting the name of the i'th axis. + The name is used for comment/documentation purposes only and is not used as a part + of the i'th axis coordinate transformations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CTYPEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a character string, giving + the name of the coordinate represented by axis i. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CUNITi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall be the units along axis i, compatible with CTYPEi to be used for + scaling and coordinate transformations along the i'th axis. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRPIXi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + identifying the location of a reference point along axis i, in units of + the axis index. This value is based upon a counter that runs from 1 to + NAXISn with an increment of 1 per pixel. The reference point value + need not be that for the center of a pixel nor lie within the actual + data array. Use comments to indicate the location of the index point + relative to the pixel. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRVALi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + giving the value of the coordinate specified by the CTYPEn keyword at + the reference point CRPIXi. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CDELTi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number + giving the partial derivative of the coordinate specified by the CTYPEi + keywords with respect to the pixel index, evaluated at the reference + point CRPIXi, in units of the coordinate specified by the CTYPEi + keyword. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null +data: + - CATDESC + - DEPEND_0 + - DISPLAY_TYPE + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +support_data: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +metadata: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - VAR_TYPE diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 97b959b08..e49913719 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -209,11 +209,15 @@ def test_add_global_attribute(): # Reloading get_global_attributes to pick up deleted Source_name test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") with pytest.raises(KeyError): - assert cdf_manager.global_attributes["Source_name"] == "Does_not_exist" + assert cdf_manager.global_attributes["Source_name"] assert test_get_global_attrs["Source_name"] is None + # Adding deleted global attribute cdf_manager.add_global_attribute("Source_name", "anas_source") assert cdf_manager.global_attributes["Source_name"] == "anas_source" + # Reloading get_global_attributes to pick up added Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + assert test_get_global_attrs["Source_name"] == "anas_source" # Testing instrument specific attribute cdf_manager.global_attributes["imap_test_T1_test"].__delitem__("Logical_source") @@ -221,7 +225,7 @@ def test_add_global_attribute(): test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") with pytest.raises(KeyError): assert ( - cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] is None + cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] ) assert test_get_global_attrs["Logical_source"] is None diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index dd2473d7f..94a12a3d9 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -26,12 +26,10 @@ def test_add_instrument_global_attrs(): def testing_source_dir(): # Create an ImapCdfAttributes object - imap_cdf_manager = ImapCdfAttributes() - imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + imap_cdf_manager = ImapCdfAttributes(Path(__file__).parent.parent / "tests") assert ( str(imap_cdf_manager.source_dir) - == "/Users/anma6676/Desktop/Repositories/imap_processing" - "/imap_processing/cdf/tests" + == str(Path(__file__).parent.parent / "tests") ) From db68128218d16e490f6a43ba44c02e0447db2220 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 24 Jun 2024 08:59:20 -0600 Subject: [PATCH 37/40] Pre-commit check --- imap_processing/cdf/tests/test_cdf_attribute_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index e49913719..28c2dd478 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -108,6 +108,7 @@ def test_get_global_attributes(): Test function that covers: get_global_attributes """ + # Initialize CdfAttributeManager object which loads in default info cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") @@ -224,9 +225,7 @@ def test_add_global_attribute(): # Reloading get_global_attributes to pick up deleted Source_name test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") with pytest.raises(KeyError): - assert ( - cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] - ) + assert cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] assert test_get_global_attrs["Logical_source"] is None From 53cbb040dd9aa36ed78d16b0ee2227a69390b450 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 24 Jun 2024 09:07:50 -0600 Subject: [PATCH 38/40] Pre-commit numpydoc --- imap_processing/cdf/imap_cdf_manager.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 440e10f0c..ea1744a21 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -11,11 +11,17 @@ class ImapCdfAttributes(CdfAttributeManager): - """Contains IMAP specific tools and settings for CDF management.""" + """ + Contains IMAP specific tools and settings for CDF management. + + Parameters + ---------- + source_dir : Path + Source directory. + """ def __init__(self, source_dir=None): """ - Set the path to the config directory. Parameters From da126441197dca3f0de2baccfcccd11bd2aa70a6 Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 24 Jun 2024 09:11:21 -0600 Subject: [PATCH 39/40] Pre --- imap_processing/cdf/tests/test_imap_cdf_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index 94a12a3d9..c36d778c2 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -3,6 +3,8 @@ # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +# Test + def test_add_instrument_global_attrs(): # Create an ImapCdfAttributes object, set to correct file path @@ -27,9 +29,8 @@ def test_add_instrument_global_attrs(): def testing_source_dir(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes(Path(__file__).parent.parent / "tests") - assert ( - str(imap_cdf_manager.source_dir) - == str(Path(__file__).parent.parent / "tests") + assert str(imap_cdf_manager.source_dir) == str( + Path(__file__).parent.parent / "tests" ) From 4a7712a7b9493666fcba8f1a86b48cf3c53f0c4e Mon Sep 17 00:00:00 2001 From: Ana Manica Date: Mon, 24 Jun 2024 16:17:59 -0600 Subject: [PATCH 40/40] Final Final Commit --- imap_processing/cdf/imap_cdf_manager.py | 6 +++--- ...df_attrs.yaml => imap_instrument1_global_cdf_attrs.yaml} | 0 ...trs.yaml => imap_instrument1_level1_variable_attrs.yaml} | 0 imap_processing/cdf/tests/test_imap_cdf_manager.py | 6 ++---- 4 files changed, 5 insertions(+), 7 deletions(-) rename imap_processing/cdf/tests/{imap_instrument_global_cdf_attrs.yaml => imap_instrument1_global_cdf_attrs.yaml} (100%) rename imap_processing/cdf/tests/{imap_instrument_level_variable_attrs.yaml => imap_instrument1_level1_variable_attrs.yaml} (100%) diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index ea1744a21..0f070a86c 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -16,8 +16,8 @@ class ImapCdfAttributes(CdfAttributeManager): Parameters ---------- - source_dir : Path - Source directory. + source_dir : Path or None + Source directory. """ def __init__(self, source_dir=None): @@ -26,7 +26,7 @@ def __init__(self, source_dir=None): Parameters ---------- - source_dir + source_dir : Path or None Source directory. """ if source_dir is None: diff --git a/imap_processing/cdf/tests/imap_instrument_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml similarity index 100% rename from imap_processing/cdf/tests/imap_instrument_global_cdf_attrs.yaml rename to imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml diff --git a/imap_processing/cdf/tests/imap_instrument_level_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml similarity index 100% rename from imap_processing/cdf/tests/imap_instrument_level_variable_attrs.yaml rename to imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py index c36d778c2..0a4de8682 100644 --- a/imap_processing/cdf/tests/test_imap_cdf_manager.py +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -3,14 +3,12 @@ # from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -# Test - def test_add_instrument_global_attrs(): # Create an ImapCdfAttributes object, set to correct file path imap_cdf_manager = ImapCdfAttributes() imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" - imap_cdf_manager.add_instrument_global_attrs("instrument") + imap_cdf_manager.add_instrument_global_attrs("instrument1") # Testing data loaded in imap_instrument = imap_cdf_manager.get_global_attributes("imap_test_T1_test") @@ -38,7 +36,7 @@ def test_add_instrument_variable_attrs(): # Create an ImapCdfAttributes object imap_cdf_manager = ImapCdfAttributes() imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" - imap_cdf_manager.add_instrument_variable_attrs("instrument", "level") + imap_cdf_manager.add_instrument_variable_attrs("instrument1", "level1") # Testing the actual function imap_instrument = imap_cdf_manager.get_variable_attributes("test_field_1")