Merge pull request #48 from cedadev/ncas-radar

NCAS Radar 1.0.0

checksit/check.py

@@ -17,6 +17,8 @@
 from .make_specs import make_amof_specs
 
 AMOF_CONVENTIONS = ['"CF-1.6, NCAS-AMF-2.0.0"']
+GENERAL_CONVENTION_PREFIXES = ["NCAS-AMF", "NCAS-AMOF", "NCAS-GENERAL"]
+RADAR_CONVENTION_PREFIXES = ["NCAS-RADAR"]
 IMAGE_EXTENSIONS = ["png", "jpg", "jpeg"]
 conf = get_config()
 
@@ -271,8 +273,8 @@ def _get_ncas_specs(
             )
             # NCAS-GENERAL file
             if any(
-                name in conventions
-                for name in ["NCAS-GENERAL", "NCAS-AMF", "NCAS-AMOF"]
+                name in conventions.upper()
+                for name in GENERAL_CONVENTION_PREFIXES
             ):
                 if verbose:
                     print("\nNCAS-AMOF file detected, finding correct spec files")
@@ -375,8 +377,34 @@ def _get_ncas_specs(
                 # don't need to do template check
                 template = "off"
 
-            # NCAS-RADAR (coming soon...)
-            # if "NCAS-Radar" in conventions
+            # NCAS-Radar
+            elif any(
+                name in conventions.upper()
+                for name in RADAR_CONVENTION_PREFIXES
+            ):
+                version_number = (
+                    conventions[conventions.index("NCAS-") :]
+                    .split("-")[2]
+                    .split(" ")[0]
+                    .replace('"', "")
+                )
+                if version_number.count(".") == 1:
+                    version_number = f"{version_number}.0"
+                template = "off"
+                spec_names = [
+                    "coordinate-variables",
+                    "dimensions",
+                    "global-attrs",
+                    "global-variables",
+                    "instrument-parameters",
+                    "location-variables",
+                    "moment-variables",
+                    "radar-calibration",
+                    "radar-parameters",
+                    "sensor-pointing-variables",
+                    "sweep-variables",
+                ]
+                specs = [f"ncas-radar-{version_number}/{spec}" for spec in spec_names]
 
         elif (
             file_path.split(".")[-1].lower() in IMAGE_EXTENSIONS

checksit/generic.py

@@ -204,7 +204,7 @@ def check_var_exists(dct, variables, skip_spellcheck=False):
 
 def check_dim_exists(dct, dimensions, skip_spellcheck=False):
     """
-    Check that variables exist
+    Check that dimensions exist
 
     E.g. check-dim-exists:dimensions:time|latitude
     """
@@ -229,13 +229,42 @@ def check_dim_exists(dct, dimensions, skip_spellcheck=False):
     return errors, warnings
 
 
-def check_var(dct, variable, defined_attrs, attr_rules=[], skip_spellcheck=False):
+def check_dim_regex(dct, regex_dims, skip_spellcheck=False):
+    """
+    Check dimension exists matching regex
+
+    E.g. check-dime-regex:regex-dims:^string_length[^,]*$
+    """
+    errors = []
+    warnings = []
+    for regex_dim in regex_dims:
+        if regex_dim.endswith(":__OPTIONAL__"):
+            regex_dim = ":".join(regex_dim.split(":")[:-1])
+            r = re.compile(regex_dim)
+            matches = list(filter(r.match, dct["dimensions"].keys()))
+            if len(matches) == 0:
+                warnings.append(
+                    f"[dimension**************:{regex_dim}]: No dimension matching optional regex check in file. "
+                )
+        else:
+            r = re.compile(regex_dim)
+            matches = list(filter(r.match, dct["dimensions"].keys()))
+            if len(matches) == 0:
+                errors.append(
+                    f"[dimension**************:{regex_dim}]: No dimension matching regex check in file. "
+                )
+    return errors, warnings
+
+
+def check_var(dct, variable, defined_attrs, rules_attrs=None, skip_spellcheck=False):
     """
     Check variable exists and has attributes defined.
     """
     errors = []
     warnings = []
 
+    rules_attrs = rules_attrs or {}
+
     if isinstance(variable, list):
         variable = variable[0]
     if ":__OPTIONAL__" in variable:
@@ -282,21 +311,54 @@ def check_var(dct, variable, defined_attrs, attr_rules=[], skip_spellcheck=False
                         f"[variable**************:{variable}]: Attribute '{attr_key}' must have definition '{attr_value}', "
                         f"not '{dct['variables'][variable].get(attr_key).encode('unicode_escape').decode('utf-8')}'."
                     )
-            for rule_to_check in attr_rules:
-                if rule_to_check == "rule-func:check-qc-flags":
+
+            for attr in rules_attrs:
+                if isinstance(attr, dict) and len(attr.keys()) == 1:
+                    for key, value in attr.items():
+                        attr = f"{key}:{value}"
+                attr_key = attr.split(":")[0]
+                attr_rule = ":".join(attr.split(":")[1:])
+                if attr_key not in dct["variables"][variable]:
+                    errors.append(
+                        f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                        f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                    )
+                elif is_undefined(dct["variables"][variable].get(attr_key)):
+                    errors.append(
+                        f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
+                    )
+                elif attr_rule.startswith("rule-func:same-type-as"):
+                    var_checking_against = attr_rule.split(":")[-1]
                     rule_errors, rule_warnings = rules.check(
-                        rule_to_check,
+                        attr_rule,
+                        dct["variables"][variable].get(attr_key),
+                        context=dct["variables"][var_checking_against].get("type"),
+                        label=f"[variables:******:{attr_key}]***",
+                    )
+                    errors.extend(rule_errors)
+                    warnings.extend(rule_warnings)
+                elif attr_rule.strip() == ("rule-func:check-qc-flags"):
+                    rule_errors, rule_warnings = rules.check(
+                        attr_rule,
                         dct["variables"][variable].get("flag_values"),
                         context=dct["variables"][variable].get("flag_meanings"),
                         label=f"[variable******:{variable}]: ",
                     )
                     errors.extend(rule_errors)
                     warnings.extend(rule_warnings)
+                else:
+                    rule_errors, rule_warnings = rules.check(
+                        attr_rule,
+                        dct["variables"][variable].get(attr_key),
+                        label=f"[variables:******:{variable}] Value of attribute '{attr_key}' -",
+                    )
+                    errors.extend(rule_errors)
+                    warnings.extend(rule_warnings)
 
     else:
         if variable not in dct["variables"].keys():
             errors.append(
-                f"[variable**************:{variable}]: Optional variable does not exist in file. "
+                f"[variable**************:{variable}]: Variable does not exist in file. "
                 f"{search_close_match(variable, dct['variables'].keys()) if not skip_spellcheck else ''}"
             )
         else:
@@ -317,6 +379,49 @@ def check_var(dct, variable, defined_attrs, attr_rules=[], skip_spellcheck=False
                         f"not '{dct['variables'][variable].get(attr_key)}'."
                     )
 
+            for attr in rules_attrs:
+                if isinstance(attr, dict) and len(attr.keys()) == 1:
+                    for key, value in attr.items():
+                        attr = f"{key}:{value}"
+                attr_key = attr.split(":")[0]
+                attr_rule = ":".join(attr.split(":")[1:])
+                if attr_key not in dct["variables"][variable]:
+                    errors.append(
+                        f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                        f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                    )
+                elif is_undefined(dct["variables"][variable].get(attr_key)):
+                    errors.append(
+                        f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
+                    )
+                elif attr_rule.startswith("rule-func:same-type-as"):
+                    var_checking_against = attr_rule.split(":")[-1]
+                    rule_errors, rule_warnings = rules.check(
+                        attr_rule,
+                        dct["variables"][variable].get(attr_key),
+                        context=dct["variables"][var_checking_against].get("type"),
+                        label=f"[variables:******:{attr_key}]***",
+                    )
+                    errors.extend(rule_errors)
+                    warnings.extend(rule_warnings)
+                elif attr_rule.strip() == "rule-func:check-qc-flags":
+                    rule_errors, rule_warnings = rules.check(
+                        attr_rule,
+                        dct["variables"][variable].get("flag_values"),
+                        context=dct["variables"][variable].get("flag_meanings"),
+                        label=f"[variable******:{variable}]: ",
+                    )
+                    errors.extend(rule_errors)
+                    warnings.extend(rule_warnings)
+                else:
+                    rule_errors, rule_warnings = rules.check(
+                        attr_rule,
+                        dct["variables"][variable].get(attr_key),
+                        label=f"[variables:******:{variable}] Value of attribute '{attr_key}' -",
+                    )
+                    errors.extend(rule_errors)
+                    warnings.extend(rule_warnings)
+
     return errors, warnings
 
 
@@ -414,3 +519,72 @@ def check_file_name(file_name, vocab_checks=None, rule_checks=None, **kwargs):
         )
 
     return errors, warnings
+
+
+def check_radar_moment_variables(
+    dct, exist_attrs=None, rule_attrs=None, one_of_attrs=None, skip_spellcheck=False
+):
+    """
+    Finds moment variables in radar file, runs checks against all those variables
+    """
+    exist_attrs = exist_attrs or []
+    rule_attrs = rule_attrs or {}
+    one_of_attrs = one_of_attrs or []
+
+    errors = []
+    warnings = []
+
+    moment_variables = []
+    for radarvariable, radarattributes in dct["variables"].items():
+        if (
+            isinstance(radarattributes, dict)
+            and "coordinates" in radarattributes.keys()
+        ):
+            moment_variables.append(radarvariable)
+
+    for variable in moment_variables:
+        for attr in exist_attrs:
+            if attr not in dct["variables"][variable]:
+                errors.append(
+                    f"[variable**************:{variable}]: Attribute '{attr}' does not exist. "
+                    f"{search_close_match(attr, dct['variables'][variable]) if not skip_spellcheck else ''}"
+                )
+        for attr in rule_attrs:
+            if isinstance(attr, dict) and len(attr.keys()) == 1:
+                for key, value in attr.items():
+                    attr = f"{key}:{value}"
+            attr_key = attr.split(":")[0]
+            attr_rule = ":".join(attr.split(":")[1:])
+            if attr_key not in dct["variables"][variable]:
+                errors.append(
+                    f"[variable:**************:{variable}]: Attribute '{attr_key}' does not exist. "
+                    f"{search_close_match(attr_key, dct['variables'][variable].keys()) if not skip_spellcheck else ''}"
+                )
+            elif is_undefined(dct["variables"][variable].get(attr_key)):
+                errors.append(
+                    f"[variable:**************:{variable}]: No value defined for attribute '{attr_key}'."
+                )
+            else:
+                rule_errors, rule_warnings = rules.check(
+                    attr_rule,
+                    dct["variables"][variable].get(attr_key),
+                    label=f"[variables:******:{variable}] Value of attribute '{attr_key}' -",
+                )
+                errors.extend(rule_errors)
+                warnings.extend(rule_warnings)
+        for attrs in one_of_attrs:
+            attr_options = attrs.split("|")
+            matches = 0
+            for attr in attr_options:
+                if attr in dct["variables"][variable]:
+                    matches += 1
+            if matches == 0:
+                errors.append(
+                    f"[variable:**************:{variable}]: One attribute of '{attr_options}' must be defined."
+                )
+            elif matches > 1:
+                errors.append(
+                    f"[variable:**************:{variable}]: Only one of '{attr_options}' should be defined, {matches} found."
+                )
+
+    return errors, warnings

checksit/readers/cdl.py

@@ -152,7 +152,11 @@ def _construct_variables(self, content):
                     variables[var_id] = current.copy()
 
                 var_id, dtype, dimensions = self._parse_var_dtype_dims(line)
-                current = {"type": dtype, "dimension": ", ".join(dimensions)}
+                if dimensions == [""]:
+                    dimensions = "--none--"
+                else:
+                    dimensions = ", ".join(dimensions)
+                current = {"type": dtype, "dimension": dimensions}
             else:
                 #                key, value = [x.strip() for x in line.split(":", 1)[1].split("=", 1)]
                 # Send last key and last value (from last iteration of loop) and line to get new value

checksit/rules/rule_funcs.py

@@ -362,13 +362,13 @@ def check_qc_flags(value, context, extras=None, label=""):
         )
 
     # check there are at least two values and they start with 0 and 1
-    if not len(value) > 2:
+    if not len(value) >= 2:
         errors.append(f"{label} There must be at least two QC flag values.")
     elif not (np.all(value[:2] == [0, 1]) or np.all(value[:2] == (0, 1))):
         errors.append(f"{label} First two QC flag_values must be '[0, 1]'.")
 
     # check there are at least two meanings and the first two are correct
-    if not len(meanings) > 2:
+    if not len(meanings) >= 2:
         errors.append(
             f"{label} There must be at least two QC flag meanings (space separated)."
         )

checksit/rules/rules.py

@@ -42,6 +42,10 @@ def __init__(self):
                 "regex-rule": r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?",
                 "example": "2023-11-17T15:00:00",
             },
+            "datetimeZ": {
+                "regex-rule": r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z",
+                "example": "2023-11-17T15:00:00Z"
+            },
             "datetime-or-na": {
                 "regex-rule": r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?)|"
                 + _NOT_APPLICABLE_RULES,
@@ -111,7 +115,7 @@ def check(self, rule_lookup, value, context=None, label=""):
 
         rule_lookup = re.sub(f"^{rules_prefix}:", "", rule_lookup)
 
-        rule_lookup_list = rule_lookup.split(", ")
+        rule_lookup_list = rule_lookup.split("||")
 
         for i in rule_lookup_list:
 

docs/source/conf.py

@@ -7,7 +7,7 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
 project = 'checksit'
-copyright = '2023, Ag Stephens, Hugo Ricketts, Joshua Hampton'
+copyright = '2024, Ag Stephens, Hugo Ricketts, Joshua Hampton'
 author = 'Ag Stephens, Hugo Ricketts, Joshua Hampton'
 release = '0.1'
 

docs/source/dev/ncas_standard_specifics.rst

@@ -27,6 +27,13 @@ function within ``checksit/make_specs.py``. However, if ``checksit`` cannot find
 version, or does not have permission to write into the ``specs/groups`` folder, then an error is
 raised.
 
+NCAS-Radar
+----------
+
+Similarly to the NCAS-GENERAL standard above, if ``NCAS-Radar`` is in the ``Conventions`` global attribute
+of a netCDF file, then ``checksit`` will use specs defined for the identifed version of the ``NCAS-Radar``
+data standard.
+
 NCAS-IMAGE
 ----------
 
@@ -36,4 +43,4 @@ one of ``png``, ``jpg`` or ``jpeg`` (or uppercase versions), and the file has th
 ``checksit`` will find specs related to NCAS-IMAGE. The version of the standard is identified using
 the ``Instructions`` tag, and specs relating to either the ``photo`` or ``plot`` data product are
 selected depending on the file name. The data product spec is combined with a global tags spec file
-that covers tags required by the standard regardless of which data product is used.
+that covers tags required by the standard regardless of which data product is used.

docs/source/dev/where_does_checksit_do_it.rst

@@ -118,6 +118,8 @@ checks, managed by the ``Rules`` class in ``checksit/rules/rules.py``. There are
      - ``r"v\d\.\d"``
    * - "datetime"
      - ``r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?"``
+   * - "datetimeZ"
+     - ``r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z"``
    * - "datetime-or-na"
      - ``r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(.\d+)?)" + _NOT_APPLICABLE_RULES``
    * - "number"