From d012a9ba1109209a5a30e105122a8cd138bb26b3 Mon Sep 17 00:00:00 2001 From: anamanica Date: Tue, 25 Jun 2024 14:30:25 -0600 Subject: [PATCH] Cdf attribute tests (#637) * Adding tests for the CdfAttributeManager class, and ImapCdfAttributes class --- imap_processing/cdf/imap_cdf_manager.py | 25 +- imap_processing/cdf/tests/__init__.py | 0 .../tests/imap_default_global_cdf_attrs.yaml | 8 + .../imap_default_global_test_cdf_attrs.yaml | 7 + .../imap_instrument1_global_cdf_attrs.yaml | 14 + ...map_instrument1_level1_variable_attrs.yaml | 23 + .../imap_instrument2_global_cdf_attrs.yaml | 23 + ...map_instrument2_level2_variable_attrs.yaml | 30 ++ .../cdf/tests/imap_test_global.yaml | 26 + .../cdf/tests/imap_test_variable.yaml | 36 ++ .../default_global_cdf_attrs_schema.yaml | 246 +++++++++ .../default_variable_cdf_attrs_schema.yaml | 466 ++++++++++++++++++ .../cdf/tests/test_cdf_attribute_manager.py | 331 ++++++++++++- .../cdf/tests/test_imap_cdf_manager.py | 60 +++ 14 files changed, 1272 insertions(+), 23 deletions(-) create mode 100644 imap_processing/cdf/tests/__init__.py create mode 100644 imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml create mode 100644 imap_processing/cdf/tests/imap_test_global.yaml create mode 100644 imap_processing/cdf/tests/imap_test_variable.yaml create mode 100644 imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml create mode 100644 imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml create mode 100644 imap_processing/cdf/tests/test_imap_cdf_manager.py diff --git a/imap_processing/cdf/imap_cdf_manager.py b/imap_processing/cdf/imap_cdf_manager.py index 1081869a8..0f070a86c 100644 --- a/imap_processing/cdf/imap_cdf_manager.py +++ b/imap_processing/cdf/imap_cdf_manager.py @@ -11,11 +11,28 @@ class ImapCdfAttributes(CdfAttributeManager): - """Contains IMAP specific tools and settings for CDF management.""" + """ + Contains IMAP specific tools and settings for CDF management. - def __init__(self): - """Set the path to the config directory.""" - super().__init__(Path(__file__).parent / "config") + Parameters + ---------- + source_dir : Path or None + Source directory. + """ + + def __init__(self, source_dir=None): + """ + Set the path to the config directory. + + Parameters + ---------- + source_dir : Path or None + Source directory. + """ + if source_dir is None: + super().__init__(Path(__file__).parent / "config") + else: + super().__init__(source_dir) def add_instrument_global_attrs(self, instrument: str): """ diff --git a/imap_processing/cdf/tests/__init__.py b/imap_processing/cdf/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml new file mode 100644 index 000000000..259f7b70e --- /dev/null +++ b/imap_processing/cdf/tests/imap_default_global_cdf_attrs.yaml @@ -0,0 +1,8 @@ +Project: STP>Solar-Terrestrial Physics +Source_name: IMAP>Interstellar Mapping and Acceleration Probe +Discipline: Solar Physics>Heliospheric Physics +# TODO: CDF docs say this value should be IMAP +Mission_group: IMAP>Interstellar Mapping and Acceleration Probe +PI_name: Dr. David J. McComas +PI_affiliation: Princeton Plasma Physics Laboratory, 100 Stellarator Road, Princeton, NJ 08540 +File_naming_convention: source_descriptor_datatype_yyyyMMdd_vNNN diff --git a/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml new file mode 100644 index 000000000..4f3f1ce9e --- /dev/null +++ b/imap_processing/cdf/tests/imap_default_global_test_cdf_attrs.yaml @@ -0,0 +1,7 @@ +Project: STP>Solar-Terrestrial Physics +Source_name: IMAP>Interstellar Mapping and Acceleration Probe +Mission_group: Dysfunctional Cats +PI_name: Ana Manica +PI_affiliation: LASP, CU +Data_version: 002 +DOI: test \ No newline at end of file diff --git a/imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml new file mode 100644 index 000000000..a5e6c5ef1 --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument1_global_cdf_attrs.yaml @@ -0,0 +1,14 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_T1_test: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. \ No newline at end of file diff --git a/imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml new file mode 100644 index 000000000..8a243fbfc --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument1_level1_variable_attrs.yaml @@ -0,0 +1,23 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: test_depend + DISPLAY_TYPE: test_display_type + FILLVAL: -10 + FORMAT: I1 + VALIDMIN: 0 + VALIDMAX: 10 + VAR_TYPE: test_var_type + +test_field_1: + <<: *default + CATDESC: test time + FIELDNAM: test_field_1 + LABLAXIS: test_labaxis + UNITS: test_units + VAR_TYPE: test_variable_type + SCALETYP: test_scaletyp + MONOTON: test_monoton + TIME_BASE: 10 + TIME_SCALE: test_time_scale + REFERENCE_POSITION: test_reference_position + NOT_REQUIRED: test_not_required \ No newline at end of file diff --git a/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml new file mode 100644 index 000000000..f28c9bc79 --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument2_global_cdf_attrs.yaml @@ -0,0 +1,23 @@ +instrument_base: &instrument_base + Descriptor: SWE>Solar Wind Electron + TEXT: > + The IMAP Solar Wind Electron (SWE) instrument measures the solar wind electrons + in the heliosphere. SWE will contribute to our understanding of the acceleration + and transport of charged particles in the heliosphere. + SWE design and assembly is led by the Princeton Plasma Physics Laboratory. See + https://imap.princeton.edu/instruments/swe for more details. + Instrument_type: "Particles (space)" + +imap_swe_l1a_sci: + <<: *instrument_base + Data_level: 1A + Data_type: L1A_SCI>Level-1A Science data + Logical_source: imap_swe_l1a_sci + Logical_source_description: SWE Instrument Level-1A Science Data + +imap_swe_l1b_sci: + <<: *instrument_base + Data_level: 1A + Data_type: L1B_SCI>Level-1B Science data + Logical_source: imap_swe_l1b_sci + Logical_source_description: SWE Instrument Level-1B Science Data diff --git a/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml new file mode 100644 index 000000000..c01deccaf --- /dev/null +++ b/imap_processing/cdf/tests/imap_instrument2_level2_variable_attrs.yaml @@ -0,0 +1,30 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: epoch + DISPLAY_TYPE: time_series + FILLVAL: -9223372036854775808 + FORMAT: I19 + VALIDMIN: 0 + VALIDMAX: 9223372036854775807 + VAR_TYPE: data + +epoch: + <<: *default + CATDESC: Time, number of nanoseconds since J2000 with leap seconds included + FIELDNAM: epoch + LABLAXIS: epoch + UNITS: ns + VAR_TYPE: support_data + SCALETYP: linear + MONOTON: INCREASE + TIME_BASE: J2000 + TIME_SCALE: Terrestrial Time + REFERENCE_POSITION: Rotating Earth Geoid + +x_front: + <<: *default + CATDESC: x front position + FIELDNAM: Event x-position on front foil + LABLAXIS: x front position + # TODO: come back to format + UNITS: mm \ No newline at end of file diff --git a/imap_processing/cdf/tests/imap_test_global.yaml b/imap_processing/cdf/tests/imap_test_global.yaml new file mode 100644 index 000000000..ae00269d6 --- /dev/null +++ b/imap_processing/cdf/tests/imap_test_global.yaml @@ -0,0 +1,26 @@ +Descriptor: &desc + TEST>Testinstrument +TEXT: &txt > + This file is nothing more than a test yamal. This description? This is a test description. + This Testinstrument will contribute to our understanding of the cdf_attribute_manager.py file. + This test file is led by me, Ana Manica. Lasp undergraduate employee. + I don't have a website. Sorry. + +imap_test_T1_test: + Descriptor: *desc + TEXT: *txt + Data_type: T1_test-one>Test-1 test one + Logical_source: imap_test_T1_test + Logical_source_description: IMAP Mission TEST one document Level-T1. + +imap_test_T2_test: + Descriptor: *desc + TEXT: *txt + Data_type: T2_test-two>Test-2 test two + Logical_source: imap_test_T2_test + Logical_source_description: IMAP Mission TEST two document Level-T2. + LINK_TEXT: Test two additional info + +imap_test_T3_test: + Descriptor: *desc + TEXT: *txt diff --git a/imap_processing/cdf/tests/imap_test_variable.yaml b/imap_processing/cdf/tests/imap_test_variable.yaml new file mode 100644 index 000000000..cbdd98cfa --- /dev/null +++ b/imap_processing/cdf/tests/imap_test_variable.yaml @@ -0,0 +1,36 @@ +default_attrs: &default + # Assumed values for all variable attrs unless overwritten + DEPEND_0: test_depend + DISPLAY_TYPE: test_display_type + FILLVAL: -10 + FORMAT: I1 + VALIDMIN: 0 + VALIDMAX: 10 + VAR_TYPE: test_var_type + +test_field_1: + <<: *default + CATDESC: test time + FIELDNAM: test_field_1 + LABLAXIS: test_labaxis + UNITS: test_units + VAR_TYPE: test_variable_type + SCALETYP: test_scaletyp + MONOTON: test_monoton + TIME_BASE: 10 + TIME_SCALE: test_time_scale + REFERENCE_POSITION: test_reference_position + NOT_REQUIRED: test_not_required + +test_field_2: + <<: *default + CATDESC: test time 2 + FIELDNAM: test_field_2 + LABLAXIS: test_labaxis_2 + UNITS: test_units_2 + VAR_TYPE: test_variable_type_2 + SCALETYP: test_scaletyp_2 + MONOTON: test_monoton_2 + TIME_BASE: 11 + TIME_SCALE: test_time_scale_2 + REFERENCE_POSITION: test_reference_position_2 \ No newline at end of file diff --git a/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml new file mode 100644 index 000000000..802c305fc --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_global_cdf_attrs_schema.yaml @@ -0,0 +1,246 @@ +DOI: + description: > + DOI is a persistent Unique Digital Identifier with the form + https://doi.org// with the identifying the DOI + registration authority and the identifying the dataset. The DOI should point to + a landing page for additional information about the dataset. DOIs are typically created by + the SPASE naming authority or archive. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true # include in validation + overwrite: false +Data_level: + description: > + This attribute is used in file name creation and records the level of processsing done + on the dataset. For HERMES the following are valid values: + - l0>Level 0 + - l1>Level 1 + - l2>Level 2 + - l3>Level 3 + - l4>Level 4 + - ql>Quicklook + default: null + required: true # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_product_descriptor: + description: > + This is an optional field that may not be needed for all products. Where it is used, identifier + should be short (e.q. 3-8 characters) descriptors that are helpful to end- users. If a + descriptor contains multiple components, underscores are used to separate those components. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_type: + description: > + This attribute is used by CDF file writing software to create a filename. It is a + combination of the following filename components: mode, data level, and optional data + product descriptor. + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +Data_version: + description: > + This attribute identifies the version of a particular CDF data file. + default: null + required: true + validate: true + overwrite: false +Descriptor: + description: > + This attribute identifies the name of the instrument or sensor that collected the data. Both + a long name and a short name are given. For any data file, only a single value is allowed. + For HERMES, the following are valid values: + - EEA>Electron Electrostatic Analyzer + - MERIT>Miniaturized Electron pRoton Telescope + - NEMISIS> Noise Eliminating Magnetometer In a Small Integrated System + - SPAN-I>Solar Probe Analyzer for Ions + default: null + required: true + validate: true + overwrite: false +Discipline: + description: > + This attribute describes both the science discipline and sub discipline. For HERMES, + this value should always be "Space Physics>Magnetospheric Science." + default: Space Physics>Magnetospheric Science + required: true + validate: true + overwrite: false +File_naming_convention: + description: > + If File_naming_convention was not set, it uses default setting: + source_datatype_descriptor_yyyyMMdd + default: source_datatype_descriptor_yyyyMMdd + required: false + validate: false + overwrite: true +Generation_date: + description: > + Date stamps the creation of the file using the syntax yyyymmdd, e.g., " + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: true +HTTP_LINK: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Instrument_mode: + description: > + TBS + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: false +Instrument_type: + description: > + This attribute is used to facilitate making choices of instrument type. More than one entry + is allowed. Acceptable values for HERMES include: + - Magnetic Fields (space) + - Particles (space) + - Plasma and Solar Wind + - Ephemeris -> Ephemeris/Attitude/Ancillary + default: null + required: true + validate: true + overwrite: false +LINK_TEXT: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +LINK_TITLE: + description: > + The 'HTTP_LINK', 'LINK_TEXT', and 'LINK_TITLE' attributes store the URL with a + description of this dataset at the HERMES SDC. The use of HTTP_LINK attribute requires + the existence and equal number of corresponding LINK_TEXT and LINK_TITLE attributes. + If text is not needed for these attributes, use an empty string "". + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Logical_file_id: + description: > + This attribute stores the name of the CDF file but without the + file extension (e.g. ".cdf"). This attribute is required to avoid loss of the original source + in the case of accidental (or intentional) renaming. + default: null + required: true + validate: true + overwrite: true +Logical_source: + description: > + This attribute determines the file naming convention in the SKT Editor and is used by + CDA Web. It is composed of the following values: + - source_name - (e.g. spacecraft identifier) + - descriptor - (e.g. instrument identifier - see Section Error! Reference source not + found.) + - data_type - (e.g. mode, data level, and optional data product descriptor - value + come from 'Data_type' attribute) + default: null + required: true + validate: true + overwrite: true +Logical_source_description: + description: > + This attribute writes out the full words associated with the encrypted Logical_source + above, e.g., "Level 1 Dual Electron Spectrometer Survey Data". Users on CDAWeb see + this value on their website. + default: null + required: true + validate: true + overwrite: true +MODS: + description: > + This attribute is an SPDF standard global attribute, which is used to denote the history of + modifications made to the CDF data set. The MODS attribute should contain a + description of all significant changes to the data set, essentially capturing a log of high- + level release notes. This attribute can have as many entries as necessary and should be + updated if the Interface Number ("X") of the version number changes. + default: null + required: false # NOT Required in ISTP Guide (Recommended) + validate: true + overwrite: false +Mission_group: + description: > + This attribute has a single value and is used to facilitate making choices of source through + CDAWeb. This value should be "HERMES." + default: HERMES + required: true + validate: true + overwrite: false +PI_affiliation: + description: > + This attribute value should include the HERMES mission PI affiliation followed by a + comma-separated list of any Co-I affiliations that are responsible for this particular + dataset. The following are valid HERMES values, of which the abbreviations should be + used exclusively within this attribute value, and the full text of the affiliation included in + the general 'text' attribute as it is used solely in plot labels. + - GSFC - Goddard Space Flight Center + - UCB - University of California, Berkeley + - SSL - Space Sciences Laboratory, UCB + - UM - University of Michigan + default: null + required: true + validate: true + overwrite: false +PI_name: + description: > + This attribute value should include first initial and last name of the HERMES mission PI + followed by a comma-separated list of any Co-Is that are responsible for this particular + dataset. + default: null + required: true + validate: true + overwrite: false +Project: + description: > + This attribute identifies the name of the project and indicates ownership. For HERMES, + this value should be "STP>Solar-Terrestrial Physics". + default: STP>Solar-Terrestrial Physics + required: true + validate: true + overwrite: false +Source_name: + description: > + This attribute identifies the observatory where the data originated. The following are + valid values for HERMES: + - HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + default: HERMES>Heliophysics Environmental and Radiation Measurement Experiment Suite + required: true + validate: true + overwrite: false +Start_time: + description: > + The start time of the contained data given in YYYYMMDD_hhmmss + default: null + required: false # NOT Required in ISTP Guide (Derived) + validate: false + overwrite: true +TEXT: + description: > + This attribute is an SPDF standard global attribute, which is a text description of the + experiment whose data is included in the CDF. A reference to a journal article(s) or to a + World Wide Web page describing the experiment is essential and constitutes the + minimum requirement. A written description of the data set is also desirable. This + attribute can have as many entries as necessary to contain the desired information. + Typically, this attribute is about a paragraph in length and is not shown on CDAWeb. + CDAWeb is the web portal for access to SPDF data, available at https://cdaweb.gsfc.nasa.gov. + default: null + required: true + validate: true + overwrite: false diff --git a/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml new file mode 100644 index 000000000..4adea4723 --- /dev/null +++ b/imap_processing/cdf/tests/shared/default_variable_cdf_attrs_schema.yaml @@ -0,0 +1,466 @@ +# Based on : https://spdf.gsfc.nasa.gov/istp_guide/vattributes.html +# HERMES CDF Format Guide +attribute_key: + # ===== EPOCH-ONLY VARIABLE ATTRIBUTES ===== + TIME_BASE: + description: > + fixed (0AD, 1900, 1970 (POSIX), J2000 (used by CDF_TIME_TT2000), + 4714 BC (Julian)) or flexible (provider-defined) + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RESOLUTION: + description: > + Using ISO8601 relative time format, for example: "1s" = 1 second. + Resolution provides the smallest change in time that is measured. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + TIME_SCALE: + description: > + TT (same as TDT, used by CDF_TIME_TT2000), TAI (same as IAT, TT-32.184s), + UTC (includes leap seconds), TDB (same as SPICE ET), EME1950 [default: UTC] + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + REFERENCE_POSITION: + description: > + Topocenter (local), Geocenter , rotating Earth geoid (used by CDF_TIME_TT2000). + Reference_Position is optional metadata to account for time variance with position in + the gravity wells and with relative velocity. While we could use a combined + TimeSystem attribute that defines mission-specific time scales where needed, such as + UTC-at-STEREO-B, it's cleaner to keep them separate as Time_Scale=UTC and + Reference_Position=STEREO-B. + required: true # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + LEAP_SECONDS_INCLUDED: + description: > + comma-delimited list (within brackets) of leap seconds included in the form of a lists + of ISO8601 times when each leap second was added, appended with the size of the leap + second in ISO8601 relative time (+/- time, most commonly: "+1s") [default: standard + list of leap seconds up to time of data]. Leap_Seconds_Included is needed to account + for time scales that don't have all 34 (in 2009) leap seconds and for the clocks in + various countries that started using leap seconds at different times. The full list is + required to handle the equally or more common case where a time scale starts at a + pecific UTC but continues on without leap seconds in TAI mode; this is basically what + missions that don't add leap seconds are doing. + $ cat tai-utc.dat | awk 'ORS="," { val = $7 - prev } {prev = $7} { print $1$2"01+" val "s" }' + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + ABSOLUTE_ERROR: + description: > + Absolute or systematic error, in same units as Units attribute. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + RELATIVE_ERROR: + description: > + Relative or random error, in same units as Units attribute - to specify the accuracy + of the time stamps relative to each other. This is usually much smaller than Absolute_Error. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + BIN_LOCATION: + description: > + relative position of time stamp to the data measurement bin, with 0.0 at the beginning + of time bin and 1.0 at the end. Default is 0.5 for the time at the center of the data + measurement. Since clock readings are usually truncated, the real value may be closer to 0.0. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + # ===== DATA VARIABLE ATTRIBUTES ===== + CATDESC: + description: > + This is a human readable description of the data variable. Generally, this is an 80- + character string which describes the variable and what it depends on. + required: true + overwrite: false + valid_values: null + alternate: null + DELTA_MINUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DELTA_PLUS_VAR: + description: > + DEPEND_i variables are typically physical values along the corresponding i-th + dimension of the parent data variable, such as energy levels or spectral frequencies. The + discreet set of values are located with respect to the sampling bin by + DELTA_PLUS_VAR and DELTA_MINUS_VAR, which hold the variable name + containing the distance from the value to the bin edge. It is strongly recommended that + HERMES DEPEND_i variables include DELTA_PLUS_VAR and + DELTA_MINUS_VAR attributes that point to the appropriate variable(s) located + elsewhere in the CDF file. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + DEPEND_0: + description: > + Explicitly ties a data variable to the time variable on which it depends. All variables + which change with time must have a DEPEND_0 attribute defined. See section 5.2.1 + which specifies the HERMES usage of DEPEND_0. + required: true + overwrite: false + valid_values: null + alternate: null + DEPEND_i: # For i'th dimensional data variables + description: > + Ties a dimensional data variable to a SUPPORT_DATA variable on which the i-th + dimension of the data variable depends. The number of DEPEND attributes must match + the dimensionality of the variable, i.e., a one-dimensional variable must have a + DEPEND_1, a two-dimensional variable must have a DEPEND_1 and a DEPEND_2 + attribute, etc. The value of the attribute must be a variable in the same CDF data set. It is + strongly recommended that DEPEND_i variables hold values in physical units. + DEPEND_i variables also require their own attributes, as described in section 5.1.4. + required: false + overwrite: false + valid_values: null + alternate: null + DISPLAY_TYPE: + description: > + This tells automated software, such as CDAWeb, how the data should be displayed. + required: true + overwrite: false + valid_values: + time_series + time_series>noerrorbars + spectrogram + stack_plot + image + alternate: null + FIELDNAM: + description: > + A shortened version of CATDESC which can be used to label a plot axis or as a data + listing heading. This is a string, up to ~30 characters in length. + required: true + overwrite: false + valid_values: null + alternate: null + FILLVAL: + description: > + Identifies the fill value used where data values are known to be bad or missing. + FILLVAL is required for time-varying variables. Fill data are always non-valid data. The + ISTP standard fill values are listed in Table 5-4. + required: true + overwrite: false + valid_values: null + alternate: null + FORMAT: # NOTE Only one of FORMAT or FORM_PTR should be present + description: > + This field allows software to properly format the associated data when displayed on a + screen or output to a file. Format can be specified using either Fortran or C format codes. + For instance, "F10.3" indicates that the data should be displayed across 10 characters + where 3 of those characters are to the right of the decimal. For a description of FORTRAN + formatting codes see the docs here: + https://docs.oracle.com/cd/E19957-01/805-4939/z40007437a2e/index.html + required: true + overwrite: false + valid_values: null + alternate: FORM_PTR + FORM_PTR: + description: > + The value of this field is a variable which stores the character string that represents the + desired output format for the associated data. + required: false + overwrite: false + valid_values: null + alternate: FORMAT + LABLAXIS: # NOTE Only one of LABLAXIS or LABL_PTR_i should be present + description: > + Used to label a plot axis or to provide a heading for a data listing. This field is generally + 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: true + overwrite: false + valid_values: null + alternate: LABL_PTR_1 + LABL_PTR_i: + description: > + Used to label a dimensional variable when one value of LABLAXIS is not sufficient to + describe the variable or to label all the axes. LABL_PTR_i is used instead of + LABLAXIS, where i can take on any value from 1 to n where n is the total number of + dimensions of the original variable. The value of LABL_PTR_1 is a variable which will + contain the short character strings which describe the first dimension of the original + variable. The value of the attribute must be a variable in the same CDF data set and is + generally 6-10 characters. Only one of LABLAXIS or LABL_PTR_i should be present. + required: false + overwrite: false + valid_values: null + alternate: LABLAXIS + SI_CONVERSION: + description: > + The conversion factor to SI units. This is the factor that the variable must be multiplied + by in order to convert it to generic SI units. This parameter contains two text fields + separated by the ">" delimiter. The first component is the conversion factor and the + second is the standard SI unit. Units are defined according to their standard SI symbols + (ie. Tesla = T, Newtons = N, Meters = m, etc.) For data variables that are inherently + unitless, and thus lack a conversion factor, this data attribute will be " > " where ' ' is + a blank space and the quotation marks are not included. Units which are not conveniently + transformed into SI should follow the blank syntax " > " described above. + required: false # NOT Required in ISTP Guide + overwrite: false + valid_values: null + alternate: null + UNITS: # NOTE Only one of UNITS or UNIT_PTR should be present + description: > + A 6-20 character string that identifies the units of the variable (e.g. nT for magnetic + field). Use a blank character, rather than "None" or "unitless", for variables that have no + units (e.g., a ratio or a direction cosine). An active list of HERMES standard UNITS and + their SI_CONVERSIONs is maintained on the mission web-pages at + https://lasp.colorado.edu/galaxy/display/HERMES/Units+of+Measure, accessible via the + HERMES Science Working Team pages. Those pages also lay out the rules for + formatting the UNITS string. + required: true + overwrite: false + valid_values: null + alternate: UNIT_PTR + UNIT_PTR: + description: > + The value of this field is a variable which stores short character strings which identify the + units of the variable. Use a blank character, rather than "None" or "unitless", for + variables that have no units (e.g., a ratio or a direction cosine). The value of this attribute + must be a variable in the same CDF data set. + required: false + overwrite: false + valid_values: null + alternate: UNITS + VALIDMIN: + description: > + The minimum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VALIDMAX: + description: > + The maximum value for a particular variable that is expected over the lifetime of the + mission. Used by application software to filter out values that are out of range. The + value must match the data type of the variable. + required: true + overwrite: false + valid_values: null + alternate: null + VAR_TYPE: + description: > + Used in CDAWeb to indicate if the data should be used directly by users. + required: true + overwrite: false + valid_values: + data + support_data + metadata + ignore_data + alternate: null + COORDINATE_SYSTEM: + description: > + All variables for which the values are dependent on the system of coordinates are + strongly recommended to have this attribute. This includes both full vectors, tensors, etc. + or individual values, e.g. of an angle with respect to some axis. The attribute is a text + string which takes the form: "XXX[>optional long name]" + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TENSOR_ORDER: + description: > + All variables which hold physical vectors, tensors, etc., or sub-parts thereof, are strongly + recommended to have their tensorial properties held by this numerical value. Vectors + have TENSOR_ORDER=1, pressure tensors have TENSOR_ORDER=2, etc. Variables + which hold single components or sub-parts of a vector or tensor, e.g., the x-component of + velocity or the three diagonal elements of a tensor, use this attribute to establish the + underlying object from which they are extracted. TENSOR_ORDER is a number, usually + held as a CDF_INT4, rather than a character string. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + REPRESENTATION_i: + description: > + This strongly recommended attribute holds the way vector or tensor variables are held, + e.g., as Cartesian or polar forms, and their sequence order in the dimension i in which + they are held. Cartesians are indicated by x,y,z; polar coordinates by r (magnitude), t + (theta - from z-axis), p (phi - longitude or azimuth around z-axis from x axis), l (lambda + = latitude). Examples follow. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + OPERATOR_TYPE: + description: > + This has been introduced to describe HERMES quaternions (see Section 5.2 below). It + has allowed values "UNIT_QUATERNION" or "ROTATION_MATRIX" although other + values could be added. Unit quaternions correspond to pure spatial rotations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + WCSAXES: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. + The value field shall contain a non-negative integer no + greater than 999, representing the number of axes in the associated + data array. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + MJDREF: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the reference time position of the time stamps along the + 0'th axis of the measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEUNIT: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a character + string giving the units of the time stamps along the 0'th axis of the measurement. + The TIMEUNIT should match the CUNITi along the time axis of the measurement + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + TIMEDEL: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. The value shall contain a floating + point number representing the resolution of the time stamps along the 0'th axis of + the measurement. The TIMEDEL should match the CRDELi along the time axis of the + measurement. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CNAMEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall contain a charachter string represnting the name of the i'th axis. + The name is used for comment/documentation purposes only and is not used as a part + of the i'th axis coordinate transformations. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CTYPEi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a character string, giving + the name of the coordinate represented by axis i. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CUNITi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value shall be the units along axis i, compatible with CTYPEi to be used for + scaling and coordinate transformations along the i'th axis. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRPIXi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + identifying the location of a reference point along axis i, in units of + the axis index. This value is based upon a counter that runs from 1 to + NAXISn with an increment of 1 per pixel. The reference point value + need not be that for the center of a pixel nor lie within the actual + data array. Use comments to indicate the location of the index point + relative to the pixel. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CRVALi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number, + giving the value of the coordinate specified by the CTYPEn keyword at + the reference point CRPIXi. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null + CDELTi: + description: > + This is a FITS WCS Keyword being repurposed for handling WCS transformations with + high-dimensional or spectral CDF data variables. This metadata attribte should be + used for the i'th dimension (1-based) and reapeated for all WCSAXES dimensions. + The value field shall contain a floating point number + giving the partial derivative of the coordinate specified by the CTYPEi + keywords with respect to the pixel index, evaluated at the reference + point CRPIXi, in units of the coordinate specified by the CTYPEi + keyword. + required: false # NOT Required in HERMES Format Guide + overwrite: false + valid_values: null + alternate: null +data: + - CATDESC + - DEPEND_0 + - DISPLAY_TYPE + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +support_data: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - LABLAXIS + - SI_CONVERSION + - UNITS + - VALIDMIN + - VALIDMAX + - VAR_TYPE +metadata: + - CATDESC + - FIELDNAM + - FILLVAL + - FORMAT + - VAR_TYPE diff --git a/imap_processing/cdf/tests/test_cdf_attribute_manager.py b/imap_processing/cdf/tests/test_cdf_attribute_manager.py index 2578f36ed..28c2dd478 100644 --- a/imap_processing/cdf/tests/test_cdf_attribute_manager.py +++ b/imap_processing/cdf/tests/test_cdf_attribute_manager.py @@ -1,58 +1,351 @@ from pathlib import Path +import pytest + from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager -# @pytest.mark.xfail(reason="Missing IMAP specific global schema") -def test_global_attribute(): +def test_default_attr_schema(): + """ + Test function that covers: + _load_default_global_attr_schema + _load_default_variable_attr_schema + """ + + # Initialize CdfAttributeManager object which loads in default schema cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") + # Default global tests + assert cdf_manager.global_attribute_schema["DOI"]["required"] is False + assert cdf_manager.global_attribute_schema["Data_level"]["required"] is True + + # Default variable tests + assert ( + cdf_manager.variable_attribute_schema["attribute_key"]["ABSOLUTE_ERROR"][ + "required" + ] + is False + ) + assert ( + cdf_manager.variable_attribute_schema["attribute_key"]["RESOLUTION"]["required"] + is True + ) + - # Expected failure: file_naming_convention is not in schema. - # for attr in cdf_manager.global_attributes.keys(): - # assert attr in cdf_manager.global_attribute_schema.keys() +def test_global_attribute(): + """ + Test function that covers: + load_global_attributes + """ + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") + + # Testing information has been loaded in + assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + assert ( + cdf_manager.global_attributes["Source_name"] + == "IMAP>Interstellar Mapping and Acceleration Probe" + ) + assert ( + cdf_manager.global_attributes["Discipline"] + == "Solar Physics>Heliospheric Physics" + ) assert ( cdf_manager.global_attributes["Mission_group"] == "IMAP>Interstellar Mapping and Acceleration Probe" ) + assert cdf_manager.global_attributes["PI_name"] == "Dr. David J. McComas" + assert ( + cdf_manager.global_attributes["PI_affiliation"] + == "Princeton Plasma Physics Laboratory, 100 Stellarator Road, " + "Princeton, NJ 08540" + ) + assert ( + cdf_manager.global_attributes["File_naming_convention"] + == "source_descriptor_datatype_yyyyMMdd_vNNN" + ) + # The following test should fail because "DOI" is not an attribute in + # imap_default_global_cdf_attrs.yaml + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["DOI"] == "test" - # Load additional global attributes - cdf_manager.load_global_attributes("imap_mag_global_cdf_attrs.yaml") + # Load in different data + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + # Testing attributes carried over assert ( - cdf_manager.global_attributes["Mission_group"] + cdf_manager.global_attributes["File_naming_convention"] + == "source_descriptor_datatype_yyyyMMdd_vNNN" + ) + assert ( + cdf_manager.global_attributes["Discipline"] + == "Solar Physics>Heliospheric Physics" + ) + + # Testing attributes newly loaded + assert cdf_manager.global_attributes["Project"] == "STP>Solar-Terrestrial Physics" + assert ( + cdf_manager.global_attributes["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" ) + assert cdf_manager.global_attributes["Mission_group"] == "Dysfunctional Cats" + assert cdf_manager.global_attributes["PI_name"] == "Ana Manica" + assert cdf_manager.global_attributes["PI_affiliation"] == "LASP, CU" + assert cdf_manager.global_attributes["Data_version"] == 2 + assert cdf_manager.global_attributes["DOI"] == "test" + + # Testing that everything loaded into the global attrs is present in + # the global attrs schema + for attr in cdf_manager.global_attributes.keys(): + assert attr in cdf_manager.global_attribute_schema.keys() - assert cdf_manager.global_attributes["Descriptor"] == "MAG>Magnetometer" - mag_l1a_global_attrs = cdf_manager.get_global_attributes("imap_mag_l1a_norm-raw") +def test_get_global_attributes(): + """ + Test function that covers: + get_global_attributes + """ + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Loading in instrument specific attributes + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + + # Testing information previously loaded into global attributes (first if statement) + assert test_get_global_attrs["Project"] == "STP>Solar-Terrestrial Physics" assert ( - mag_l1a_global_attrs["Mission_group"] + test_get_global_attrs["Source_name"] == "IMAP>Interstellar Mapping and Acceleration Probe" ) - assert mag_l1a_global_attrs["Descriptor"] == "MAG>Magnetometer" - assert mag_l1a_global_attrs["Logical_source"] == "imap_mag_l1a_norm-raw" + assert test_get_global_attrs["Mission_group"] == "Dysfunctional Cats" + # Testing instrument specific global attributes (first elif statement) + assert test_get_global_attrs["Descriptor"] == "TEST>Testinstrument" + assert test_get_global_attrs["Data_type"] == "T1_test-one>Test-1 test one" + assert test_get_global_attrs["Logical_source"] == "imap_test_T1_test" + assert ( + test_get_global_attrs["Logical_source_description"] + == "IMAP Mission TEST one document Level-T1." + ) + # Not given, and not required information + assert test_get_global_attrs["Data_level"] is None + with pytest.raises(KeyError): + assert test_get_global_attrs["bad_name"] == "false info" + + # Testing second elif statement + test_error_elif = cdf_manager.get_global_attributes("imap_test_T3_test") + with pytest.raises(KeyError): + assert test_error_elif["Data_type"] == "Does Not Exist" + + # Load in more data using get_global_attributes + test_get_global_attrs_2 = cdf_manager.get_global_attributes("imap_test_T2_test") + # Testing information previously loaded into global attributes (first if statement) + assert test_get_global_attrs_2["Project"] == "STP>Solar-Terrestrial Physics" + # Testing first elif statement + assert test_get_global_attrs_2["Descriptor"] == "TEST>Testinstrument" + # "Data_type" not required according to default schema + assert test_get_global_attrs_2["Data_type"] == "T2_test-two>Test-2 test two" + + # Testing that required schema keys are in get_global_attributes + for attr_name in cdf_manager.global_attribute_schema.keys(): + required_schema = cdf_manager.global_attribute_schema[attr_name]["required"] + if required_schema is True: + assert attr_name in test_get_global_attrs.keys() + + +def test_instrument_id_format(): + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Loading in instrument specific attributes + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + + # Testing how instrument_id operates + assert test_get_global_attrs["Project"] == cdf_manager.global_attributes["Project"] + assert ( + test_get_global_attrs["Source_name"] + == cdf_manager.global_attributes["Source_name"] + ) + assert ( + test_get_global_attrs["Data_type"] + == cdf_manager.global_attributes["imap_test_T1_test"]["Data_type"] + ) + assert ( + cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] + == "imap_test_T1_test" + ) + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["imap_test_T1_test"]["Project"] + + +def test_add_global_attribute(): + # Initialize CdfAttributeManager object which loads in default info + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_test_global.yaml") + + # Changing a dynamic global variable + cdf_manager.add_global_attribute("Project", "Test Project") + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + assert cdf_manager.global_attributes["Project"] == "Test Project" + assert test_get_global_attrs["Project"] == "Test Project" + + # Testing adding required global attribute + cdf_manager.global_attributes.__delitem__("Source_name") + # Reloading get_global_attributes to pick up deleted Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["Source_name"] + assert test_get_global_attrs["Source_name"] is None + + # Adding deleted global attribute + cdf_manager.add_global_attribute("Source_name", "anas_source") + assert cdf_manager.global_attributes["Source_name"] == "anas_source" + # Reloading get_global_attributes to pick up added Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + assert test_get_global_attrs["Source_name"] == "anas_source" + + # Testing instrument specific attribute + cdf_manager.global_attributes["imap_test_T1_test"].__delitem__("Logical_source") + # Reloading get_global_attributes to pick up deleted Source_name + test_get_global_attrs = cdf_manager.get_global_attributes("imap_test_T1_test") + with pytest.raises(KeyError): + assert cdf_manager.global_attributes["imap_test_T1_test"]["Logical_source"] + assert test_get_global_attrs["Logical_source"] is None def test_variable_attribute(): + """ + Test function that covers: + load_variable_attributes + get_variable_attributes + """ + + # Creating CdfAttributeManager object, loading in default data cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") - cdf_manager.load_global_attributes("imap_default_global_cdf_attrs.yaml") - cdf_manager.load_variable_attributes("imap_mag_l1a_variable_attrs.yaml") + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + # Loading in test data + cdf_manager.load_variable_attributes("imap_test_variable.yaml") # All variables required to have: expected_attributes = [ + "CATDESC", + "DEPEND_0", "DISPLAY_TYPE", + "FIELDNAM", "FILLVAL", "FORMAT", + "LABLAXIS", + "UNITS", "VALIDMIN", "VALIDMAX", "VAR_TYPE", ] - for variable_attrs in cdf_manager.variable_attributes.values(): - for attr in expected_attributes: - assert attr in variable_attrs.keys() + # Assuring all required attributes are loaded in + for attr_name in cdf_manager.variable_attribute_schema["attribute_key"]: + attribute = cdf_manager.variable_attribute_schema["attribute_key"][attr_name] + if attribute["required"] is True: + for exp_attr in expected_attributes: + assert ( + exp_attr in cdf_manager.variable_attribute_schema["attribute_key"] + ) + + # Testing specific attributes + assert ( + cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] + == cdf_manager.variable_attributes["default_attrs"]["DEPEND_0"] + ) + assert cdf_manager.variable_attributes["default_attrs"]["FILLVAL"] == -10 + assert cdf_manager.variable_attributes["test_field_1"]["DEPEND_0"] == "test_depend" + assert ( + cdf_manager.variable_attributes["default_attrs"]["VAR_TYPE"] == "test_var_type" + ) + with pytest.raises(KeyError): + assert cdf_manager.variable_attributes["default_attrs"]["CATDESC"] == "test" + + +def test_get_variable_attributes(): + # Creating CdfAttributeManager object, loading in default data + cdf_manager = CdfAttributeManager(Path(__file__).parent.parent / "config") + + # Change filepath to load test global attributes + cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + cdf_manager.load_global_attributes("imap_default_global_test_cdf_attrs.yaml") + cdf_manager.load_variable_attributes("imap_test_variable.yaml") + + # Loading in instrument specific attributes + imap_test_variable = cdf_manager.get_variable_attributes("test_field_1") + + # Make sure all expected attributes are present + for variable_attrs in cdf_manager.variable_attribute_schema.keys(): + required_var_attributes = cdf_manager.variable_attribute_schema[variable_attrs] + if required_var_attributes is True: + assert variable_attrs in imap_test_variable.keys() + + # Calling default attributes + assert imap_test_variable["DEPEND_0"] == "test_depend" + assert imap_test_variable["DISPLAY_TYPE"] == "test_display_type" + assert imap_test_variable["FILLVAL"] == -10 + assert imap_test_variable["FORMAT"] == "I1" + assert imap_test_variable["VALIDMIN"] == 0 + assert imap_test_variable["VALIDMAX"] == 10 + assert imap_test_variable["VAR_TYPE"] == "test_variable_type" + + # Calling required attributes + assert imap_test_variable["CATDESC"] == "test time" + assert imap_test_variable["FIELDNAM"] == "test_field_1" + assert imap_test_variable["LABLAXIS"] == "test_labaxis" + assert imap_test_variable["UNITS"] == "test_units" + assert imap_test_variable["VAR_TYPE"] == "test_variable_type" + assert imap_test_variable["SCALETYP"] == "test_scaletyp" + assert imap_test_variable["MONOTON"] == "test_monoton" + assert imap_test_variable["TIME_BASE"] == 10 + assert imap_test_variable["TIME_SCALE"] == "test_time_scale" + assert imap_test_variable["REFERENCE_POSITION"] == "test_reference_position" + + # Calling to non required attributes + assert imap_test_variable["NOT_REQUIRED"] == "test_not_required" + + # Calling attribute name that does not exist + with pytest.raises(KeyError): + assert imap_test_variable["DOES_NOT_EXIST"] == "test time" + + # Load in different data, test again + imap_test_variable_2 = cdf_manager.get_variable_attributes("test_field_2") + # Calling default attributes + assert imap_test_variable_2["DEPEND_0"] == "test_depend" + assert imap_test_variable_2["DISPLAY_TYPE"] == "test_display_type" + assert imap_test_variable_2["FILLVAL"] == -10 + assert imap_test_variable_2["FORMAT"] == "I1" + assert imap_test_variable_2["VALIDMIN"] == 0 + assert imap_test_variable_2["VALIDMAX"] == 10 + assert imap_test_variable_2["VAR_TYPE"] == "test_variable_type_2" + + # Calling required attributes + assert imap_test_variable_2["CATDESC"] == "test time 2" + assert imap_test_variable_2["FIELDNAM"] == "test_field_2" + assert imap_test_variable_2["LABLAXIS"] == "test_labaxis_2" + assert imap_test_variable_2["UNITS"] == "test_units_2" + assert imap_test_variable_2["VAR_TYPE"] == "test_variable_type_2" + assert imap_test_variable_2["SCALETYP"] == "test_scaletyp_2" + assert imap_test_variable_2["MONOTON"] == "test_monoton_2" + assert imap_test_variable_2["TIME_BASE"] == 11 + assert imap_test_variable_2["TIME_SCALE"] == "test_time_scale_2" + assert imap_test_variable_2["REFERENCE_POSITION"] == "test_reference_position_2" diff --git a/imap_processing/cdf/tests/test_imap_cdf_manager.py b/imap_processing/cdf/tests/test_imap_cdf_manager.py new file mode 100644 index 000000000..0a4de8682 --- /dev/null +++ b/imap_processing/cdf/tests/test_imap_cdf_manager.py @@ -0,0 +1,60 @@ +from pathlib import Path + +# from imap_processing.cdf.cdf_attribute_manager import CdfAttributeManager +from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes + + +def test_add_instrument_global_attrs(): + # Create an ImapCdfAttributes object, set to correct file path + imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + imap_cdf_manager.add_instrument_global_attrs("instrument1") + + # Testing data loaded in + imap_instrument = imap_cdf_manager.get_global_attributes("imap_test_T1_test") + assert imap_instrument["Data_type"] == "T1_test-one>Test-1 test one" + assert imap_instrument["Project"] == "STP>Solar-Terrestrial Physics" + + # Testing reloading data + imap_cdf_manager.add_instrument_global_attrs("instrument2") + + # Testing data carried over, and overwritten + instrument2_instrument = imap_cdf_manager.get_global_attributes("imap_swe_l1a_sci") + assert instrument2_instrument["Data_type"] == "L1A_SCI>Level-1A Science data" + assert instrument2_instrument["Project"] == "STP>Solar-Terrestrial Physics" + + +def testing_source_dir(): + # Create an ImapCdfAttributes object + imap_cdf_manager = ImapCdfAttributes(Path(__file__).parent.parent / "tests") + assert str(imap_cdf_manager.source_dir) == str( + Path(__file__).parent.parent / "tests" + ) + + +def test_add_instrument_variable_attrs(): + # Create an ImapCdfAttributes object + imap_cdf_manager = ImapCdfAttributes() + imap_cdf_manager.source_dir = Path(__file__).parent.parent / "tests" + imap_cdf_manager.add_instrument_variable_attrs("instrument1", "level1") + + # Testing the actual function + imap_instrument = imap_cdf_manager.get_variable_attributes("test_field_1") + assert imap_instrument["DEPEND_0"] == "test_depend" + assert imap_instrument["CATDESC"] == "test time" + assert imap_instrument["FIELDNAM"] == "test_field_1" + assert imap_instrument["UNITS"] == "test_units" + + # Testing reloading data + imap_cdf_manager.add_instrument_variable_attrs("instrument2", "level2") + + # Testing again + instrument2_instrument = imap_cdf_manager.get_variable_attributes("epoch") + assert instrument2_instrument["DEPEND_0"] == "epoch" + assert instrument2_instrument["DISPLAY_TYPE"] == "time_series" + assert instrument2_instrument["VALIDMAX"] == 9223372036854775807 + assert ( + instrument2_instrument["CATDESC"] + == "Time, number of nanoseconds since J2000 with leap seconds included" + ) + assert instrument2_instrument["UNITS"] == "ns"