Merge pull request #243 from NASA-IMPACT/feature-add_new_checks

Feature add new checks
NASA-IMPACT · Jul 24, 2023 · a105e27 · a105e27
2 parents baf4b47 + 7ccd425
commit a105e27
Show file tree

Hide file tree

Showing 4 changed files with 255 additions and 10 deletions.
diff --git a/pyQuARC/code/custom_validator.py b/pyQuARC/code/custom_validator.py
@@ -94,6 +94,51 @@ def one_item_presence_check(*field_values):
                 break
 
         return {"valid": validity, "value": value}
+
+    @staticmethod
+    def dif_standard_product_check(*field_values):
+        """
+        Checks if the Extended_Metadata field in the DIF schema is being 
+        utilized to specify whether or not the collection is a Standard Product.
+        This check is needed because DIF schema does not have a dedicated field
+        for Standard Product, and the Extended_Metadata field is also utilized
+        for other things.
+        """
+        validity = False
+        value = None
+
+        for field_value in field_values:
+             if field_value:
+                if 'StandardProduct' in field_value:
+                    value = field_value
+                    validity = True
+                    break
+        else:
+            pass
+        return {"valid": validity, "value": value}
+
+    @staticmethod
+    def license_url_description_check(description_field, url_field, license_text):
+        """
+        Determines if a description has been provided for the License URL if a
+        License URL has been provided in the metadata.
+
+        Args:
+            url_field (string): license URL string
+            description_field (string): string describing the URL
+        """
+        validity = True
+        value  = description_field
+
+        if not license_text and not url_field:
+            validity = False
+            return {"valid": validity, "value": value}
+        elif license_text and not url_field:
+            return {"valid": validity, "value": value}
+        else:
+            if not description_field:
+                validity = False
+            return {"valid": validity, "value": value}
 
     @staticmethod
     def granule_sensor_presence_check(sensor_values, collection_shortname=None, version=None, dataset_id=None):

diff --git a/pyQuARC/schemas/check_messages.json b/pyQuARC/schemas/check_messages.json
@@ -1,4 +1,13 @@
 {
+    "free_and_open_data_presence_check":{
+        "failure": "No FreeAndOpenData value was given.",
+        "help": {
+            "message": "",
+            "url": "https://wiki.earthdata.nasa.gov/display/CMR/Use+Constraints"
+        },
+        "remediation": "Recommend providing a FreeAndOpenData value of 'true'."
+    },
+
     "datetime_format_check": {
         "failure": "`{}` does not adhere to the ISO 1601 standard.",
         "help": {
@@ -311,6 +320,22 @@
         },
         "remediation": "Recommend providing the instrument short name."
     },
+    "standard_product_check": {
+        "failure": "The Standard Product flag is missing.",
+        "help": {
+            "message": "For information please see:",
+            "url": "https://wiki.earthdata.nasa.gov/display/CMR/StandardProduct"
+        },
+        "remediation": "Recommend indicating whether this is a StandardProduct."
+    },
+    "dif_standard_product_check": {
+        "failure": "The Standard Product flag is missing.",
+        "help": {
+            "message": "For information please see:",
+            "url": "https://wiki.earthdata.nasa.gov/display/CMR/StandardProduct"
+        },
+        "remediation": "Recommend indicating whether this is a StandardProduct."
+    },
     "validate_granule_instrument_against_collection": {
         "failure": "The instrument short name listed in the granule metadata does not match the instrument short name listed in the collection metadata.",
         "help": {
@@ -663,6 +688,14 @@
         },
         "remediation": "Recommend providing information about the license applicable to the dataset, preferably as a URL. For EOSDIS records, recommend providing the following link (unless under different usage terms): https://earthdata.nasa.gov/earth-observation-data/data-use-policy"
     },
+    "license_url_description_check": {
+        "failure": "Recommend providing a license URL description. For example: \n",
+        "help": {
+            "message": "",
+            "url": ""
+        },
+        "remediation": "'The Earth Observing System Data and Information System (EOSDIS) data use policy for NASA data.'"
+    },
     "collection_citation_presence_check": {
         "failure": "No citation information is provided.",
         "help": {
@@ -1022,5 +1055,13 @@
             "url": "https://wiki.earthdata.nasa.gov/display/CMR/Instrument"
         },
         "remediation": "Recommend updating the Number Of Instruments/Sensors value to match the numerical amount of instruments/sensors provided."
+    },
+    "horizontal_resolution_presence_check": {
+        "failure": "No horizontal resolution information is provided.",
+        "help": {
+            "message": "",
+            "url": "https://wiki.earthdata.nasa.gov/display/CMR/Spatial+Extent"
+        },
+        "remediation": "Recommend providing the horizontal pixel resolution, if applicable. If provided, this information will be indexed in the EDSC 'Horizontal Data Resolution' search facet which allows users to search by spatial resolution."
     }
-}
+}
diff --git a/pyQuARC/schemas/checks.json b/pyQuARC/schemas/checks.json
@@ -194,6 +194,11 @@
         "check_function": "one_item_presence_check",
         "available": true
     },
+    "dif_standard_product_check": {
+        "data_type": "custom",
+        "check_function": "dif_standard_product_check",
+        "available": true
+    },
     "doi_link_update": {
         "data_type": "url",
         "check_function": "doi_link_update",
@@ -229,6 +234,11 @@
         "check_function": "online_resource_type_gcmd_check",
         "available": true
     },
+    "license_url_description_check": {
+        "data_type": "custom",
+        "check_function": "license_url_description_check",
+        "available": true
+    },
     "uniqueness_check": {
         "data_type": "custom",
         "check_function": "uniqueness_check",

diff --git a/pyQuARC/schemas/rule_mapping.json b/pyQuARC/schemas/rule_mapping.json
@@ -1,4 +1,36 @@
 {
+
+    "free_and_open_data_presence_check": {
+        "rule_name": "Free and Open Data Presence Check",
+        "fields_to_apply": {
+
+            "echo-c": [
+                {
+                    "fields": [
+                        "Collection/UseConstraints/FreeAndOpenData"
+                    ]
+                }
+            ],
+            "dif10": [
+                {
+                    "fields": [
+                        "DIF/Use_Constraints/Free_And_Open_Data"
+                    ]
+                }
+            ],
+            "umm-c": [
+                {
+                    "fields": [
+                        "UseConstraints/FreeAndOpenData"
+                    ]
+                }
+            ]
+
+        },
+        "severity": "warning",
+        "check_id": "one_item_presence_check"
+    },
+
     "data_update_time_logic_check": {
         "rule_name": "Data Update Time Logic Check",
         "fields_to_apply": {
@@ -2565,8 +2597,43 @@
             ]
         },
         "severity": "info",
+        "check_id": "one_item_presence_check"    
+    },
+    "standard_product_check": {
+        "rule_name": "Standard Product Check",
+        "fields_to_apply": {
+            "echo-c": [
+                {
+                    "fields": [
+                        "Collection/StandardProduct"
+                    ]
+                }
+             ],
+            "umm-c": [
+                {
+                    "fields": [
+                        "StandardProduct"
+                    ]
+                }
+            ]
+        },
+        "severity": "warning",
         "check_id": "one_item_presence_check"
     },
+    "dif_standard_product_check": {
+        "rule_name": "Standard Product Check",
+        "fields_to_apply": {
+            "dif10": [
+                {
+                    "fields": [
+                        "DIF/Extended_Metadata/Metadata/Name"
+                    ]
+                }
+            ]
+        },
+        "severity": "warning",
+        "check_id": "dif_standard_product_check"
+    },
     "validate_granule_instrument_against_collection": {
         "rule_name": "Granule Instrument Short Name Check",
         "fields_to_apply": {
@@ -3691,7 +3758,7 @@
                 {
                     "fields": [
                         "Collection/UseConstraints/LicenseURL/URL",
-                        "Collection/UseConstraints/LicenseURL/Description"
+                        "Collection/UseConstraints/LicenseText"
                     ]
                 }
             ],
@@ -3707,14 +3774,48 @@
                 {
                     "fields": [
                         "UseConstraints/LicenseURL/Linkage",
-                        "UseConstraints/LicenseURL/Description"
+                        "UseConstraints/LicenseText"
                     ]
                 }
             ]
         },
         "severity": "error",
         "check_id": "one_item_presence_check"
     },
+    "license_url_description_check": {
+        "rule_name": "License URL Description Check",
+        "fields_to_apply": {
+            "echo-c": [
+                {
+                    "fields": [
+                        "Collection/UseConstraints/LicenseURL/Description",
+                        "Collection/UseConstraints/LicenseURL/URL",
+                        "Collection/UseConstraints/LicenseText"
+                    ]
+                }
+            ],
+            "dif10": [
+                {
+                    "fields": [
+                        "DIF/Use_Constraints/License_URL/Description",
+                        "DIF/Use_Constraints/License_URL/URL",
+                        "DIF/Use_Constraints/License_Text"
+                    ]
+                }
+            ],
+            "umm-c": [
+                {
+                    "fields": [
+                        "UseConstraints/LicenseURL/Description",
+                        "UseConstraints/LicenseURL/Linkage",
+                        "UseConstraints/LicenseText"
+                    ]
+                }
+            ]
+        },
+        "severity": "warning",
+        "check_id": "license_url_description_check"
+    },
     "collection_citation_presence_check": {
         "rule_name": "Collection Citation Presence Check",
         "fields_to_apply": {
@@ -4590,12 +4691,6 @@
                         "DIF/Multimedia_Sample/Description",
                         "DIF/Multimedia_Sample/URL"
                     ]
-                },
-                {
-                    "fields": [
-                        "DIF/Use_Constraints/License_URL/Description",
-                        "DIF/Use_Constraints/License_URL/URL"
-                    ]
                 }
             ],
             "umm-c": [
@@ -5301,5 +5396,59 @@
         },
         "severity": "warning",
         "check_id": "granule_data_format_presence_check"
+    },
+    "horizontal_resolution_presence_check": {
+        "rule_name": "Horizontal Resolution Presence Check",
+        "fields_to_apply": {
+            "echo-c": [
+                {
+                    "fields": [
+                        "Collection/SpatialInfo/HorizontalCoordinateSystem/GeographicCoordinateSystem/LongitudeResolution",
+                        "Collection/SpatialInfo/HorizontalCoordinateSystem/GeographicCoordinateSystem/LatitudeResolution",
+                        "Collection/SpatialInfo/HorizontalCoordinateSystem/GeographicCoordinateSystem/GeographicCoordinateUnits"
+                    ]
+                }
+            ],
+            "dif10": [
+                {
+                    "fields": [
+                        "DIF/Data_Resolution/Longitude_Resolution",
+                        "DIF/Data_Resolution/Latitude_Resolution",
+                        "DIF/Spatial_Coverage/Spatial_Info/Horizontal_Coordinate_System/Geographic_Coordinate_System/LongitudeResolution",
+                        "DIF/Spatial_Coverage/Spatial_Info/Horizontal_Coordinate_System/Geographic_Coordinate_System/LatitudeResolution",
+                        "DIF/Spatial_Coverage/Spatial_Info/Horizontal_Coordinate_System/Geographic_Coordinate_System/GeographicCoordinateUnits"
+                    ]
+                }
+            ],
+            "umm-c": [
+                {
+                    "fields": [
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/VariesResolution",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/PointResolution",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedResolutions/XDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedResolutions/YDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedResolutions/Unit",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GenericResolutions/XDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GenericResolutions/YDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GenericResolutions/Unit",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedResolutions/XDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedResolutions/YDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedResolutions/Unit",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedRangeResolutions/MinimumXDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedRangeResolutions/MaximumXDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedRangeResolutions/MinimumYDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedRangeResolutions/MaximumYDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/NonGriddedRangeResolutions/Unit",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedRangeResolutions/MinimumXDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedRangeResolutions/MaximumXDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedRangeResolutions/MinimumYDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedRangeResolutions/MaximumYDimension",
+                        "SpatialExtent/HorizontalSpatialDomain/ResolutionAndCoordinateSystem/HorizontalDataResolution/GriddedRangeResolutions/Unit"
+                    ]
+                }
+            ]
+        },
+        "severity": "warning",
+        "check_id": "one_item_presence_check"
     }
-}
+}