Merge pull request #49 from cedadev/radar-changes

Clarifications to NCAS-Radar

checksit/generic.py

@@ -319,10 +319,22 @@ def check_var(dct, variable, defined_attrs, rules_attrs=None, skip_spellcheck=Fa
                 attr_key = attr.split(":")[0]
                 attr_rule = ":".join(attr.split(":")[1:])
                 if attr_key not in dct["variables"][variable]:
-                    errors.append(
-                        f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
-                        f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
-                    )
+                    if not (
+                        attr_key == "standard_name" and attr_rule.split(":")[1] == "allow-proposed"
+                    ):
+                        errors.append(
+                            f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                            f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                        )
+                    else:
+                        rule_errors, rule_warnings = rules.check(
+                            attr_rule,
+                            dct["variables"][variable].get(attr_key),
+                            context=dct["variables"][variable].get("proposed_standard_name"),
+                            label=f"[variables:******:{variable}]***",
+                        )
+                        errors.extend(rule_errors)
+                        warnings.extend(rule_warnings)
                 elif is_undefined(dct["variables"][variable].get(attr_key)):
                     errors.append(
                         f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
@@ -386,14 +398,31 @@ def check_var(dct, variable, defined_attrs, rules_attrs=None, skip_spellcheck=Fa
                 attr_key = attr.split(":")[0]
                 attr_rule = ":".join(attr.split(":")[1:])
                 if attr_key not in dct["variables"][variable]:
-                    errors.append(
-                        f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
-                        f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
-                    )
-                elif is_undefined(dct["variables"][variable].get(attr_key)):
-                    errors.append(
-                        f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
-                    )
+                    if not (
+                        attr_key == "standard_name" and attr_rule.split(":")[1] == "allow-proposed"
+                    ):
+                        errors.append(
+                            f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                            f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                        )
+                    else:
+                        rule_errors, rule_warnings = rules.check(
+                            attr_rule,
+                            dct["variables"][variable].get(attr_key),
+                            context=dct["variables"][variable].get("proposed_standard_name"),
+                            label=f"[variables:******:{variable}]***",
+                        )
+                        errors.extend(rule_errors)
+                        warnings.extend(rule_warnings)
+                #if attr_key not in dct["variables"][variable]:
+                #    errors.append(
+                #        f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                #        f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                #    )
+                #elif is_undefined(dct["variables"][variable].get(attr_key)):
+                #    errors.append(
+                #        f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
+                #    )
                 elif attr_rule.startswith("rule-func:same-type-as"):
                     var_checking_against = attr_rule.split(":")[-1]
                     rule_errors, rule_warnings = rules.check(

checksit/rules/rule_funcs.py

@@ -4,6 +4,7 @@
 import requests
 from urllib.request import urlopen
 import numpy as np
+import sys
 
 from . import processors
 from ..config import get_config
@@ -384,3 +385,48 @@ def check_qc_flags(value, context, extras=None, label=""):
         )
 
     return errors
+
+
+def check_utc_date_iso_format(value, context, extras=None, label=""):
+    """
+    Check date given is in ISO 8601 format and in UTC
+    value - date string
+    """
+    errors = []
+
+    original_value = value
+    if sys.version_info < (3,11): # python datetime changed its recognition of ISO format from 3.11 onward
+        if value.endswith("Z"):
+            value = value.replace("Z", "+00:00")
+        elif re.fullmatch(r"(\+|-)\d{4}", value[-5:]):
+            value = f"{value[:-2]}:{value[-2:]}"
+    try:
+        dt = datetime.fromisoformat(value)
+        if (dt.utcoffset() != None) and (dt.utcoffset().total_seconds() != 0):
+            errors.append(f"{label} Date string '{original_value}' not in UTC.")
+    except ValueError:
+        errors.append(f"{label} Date string '{original_value}' not in ISO 8601 format.")
+    except:
+        raise
+
+    return errors
+
+
+def allow_proposed(value, context, extras=None, label=""):
+    """
+    Check for proposed_standard_name if standard_name not given
+    value - value of the standard_name attribute
+    context - value of the proposed_standard_name attribute
+    extras - value to match
+    """
+    errors = []
+
+    if extras != None and isinstance(extras, list):
+        extras = extras[0]
+
+    if value != extras and context != extras:
+        errors.append(f"{label} does not contain standard_name or proposed_standard_name with value '{extras}'")
+
+    return errors
+
+

specs/groups/ncas-radar-1.0.0/coordinate-variables.yml

@@ -17,10 +17,10 @@ var-requires1:
       - range
     defined_attrs:
       - type:float
-      - standard_name:projection_range_coordinate
       - axis:radial_range_coordinate
     rules_attrs:
       - dimension: rule-func:match-one-of:range|sweep, range
       - units: rule-func:match-one-of:metres|meters
       - long_name: rule-func:string-of-length:5+
       - spacing_is_constant: rule-func:match-one-of:true|false
+      - standard_name: rule-func:allow-proposed:projection_range_coordinate

specs/groups/ncas-radar-1.0.0/global-attrs.yml

@@ -31,14 +31,14 @@ required-gloabl-attrs:
         creator_email: regex-rule:valid-email
         creator_url: regex-rule:valid-url||rule-func-warning:validate-orcid-ID
         processing_level: rule-func:match-one-of:1|2|3
-        last_revised_date: regex-rule:datetime
+        last_revised_date: rule-func:check-utc-date-iso-format
         project_principal_investigator_email: regex-rule:valid-email
         project_principal_investigator_url: regex-rule:valid-url||rule-func-warning:validate-orcid-ID
         platform: rule-func:ceda-platform||rule-func-warning:ncas-platform
         deployment_mode: rule-func:match-one-of:land|sea|air
-        time_coverage_start: regex-rule:datetimeZ
-        time_coverage_end: regex-rule:datetimeZ
+        time_coverage_start: rule-func:check-utc-date-iso-format
+        time_coverage_end: rule-func:check-utc-date-iso-format
     regex_attrs:
-        Conventions: ^(NCAS-Radar-1\.0|CfRadial-1\.4) (NCAS-Radar-1\.0|CfRadial-1\.4) instrument_parameters radar_parameters radar_calibration$
+        Conventions: ^(NCAS-Radar-1\.0 CfRadial-1\.4|CfRadial-1\.4 NCAS-Radar-1\.0)( instrument_parameters| radar_parameters| lidar_parameters| radar_calibration| lidar_calibration| platform_velocity| geometry_correction)*$
         product_version: ^v\d+\.\d+\.\d+$
 

specs/groups/ncas-radar-1.0.0/sensor-pointing-variables.yml

@@ -7,9 +7,10 @@ var-requires0:
       - type:float
       - dimension:time
       - units:degrees
-      - standard_name:ray_azimuth_angle
       - long_name:azimuth_angle_from_true_north
       - axis:radial_azimuth_coordinate
+    rules_attrs:
+      - standard_name: rule-func:allow-proposed:ray_azimuth_angle
 var-requires1:
   func: checksit.generic.check_var
   params:
@@ -19,6 +20,7 @@ var-requires1:
       - type:float
       - dimension:time
       - units:degrees
-      - standard_name:ray_elevation_angle
       - long_name:elevation_angle_from_horizontal_plane
       - axis:radial_elevation_coordinate
+    rules_attrs:
+      - standard_name: rule-func:allow-proposed:ray_elevation_angle