Omnibus 2024-11-27

.github/workflows/test.yaml

@@ -2,10 +2,10 @@ name: Test
 on:
   pull_request:
     branches:
-      - main
+      - '**'
   push:
     branches:
-      - main
+      - '**'
   workflow_dispatch:
     branches:
       - '**'

docs/sections/user_guide/cli/tools/config/realize-verbose.out

@@ -1,30 +1,30 @@
-[2024-05-23T19:39:16]    DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose
-[2024-05-23T19:39:16]    DEBUG Reading input from stdin
-[2024-05-23T19:39:16]    DEBUG Dereferencing, current value:
-[2024-05-23T19:39:16]    DEBUG   hello: '{{ recipient }}'
-[2024-05-23T19:39:16]    DEBUG   recipient: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: {{ recipient }}
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: hello
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: hello
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: recipient
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: recipient
-[2024-05-23T19:39:16]    DEBUG Dereferencing, current value:
-[2024-05-23T19:39:16]    DEBUG   hello: world
-[2024-05-23T19:39:16]    DEBUG   recipient: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: hello
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: hello
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: world
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendering: recipient
-[2024-05-23T19:39:16]    DEBUG [dereference] Rendered: recipient
-[2024-05-23T19:39:16]    DEBUG Dereferencing, final value:
-[2024-05-23T19:39:16]    DEBUG   hello: world
-[2024-05-23T19:39:16]    DEBUG   recipient: world
-[2024-05-23T19:39:16]    DEBUG Writing output to stdout
+[2024-11-27T05:24:34]    DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose
+[2024-11-27T05:24:34]    DEBUG Reading input from stdin
+[2024-11-27T05:24:34]    DEBUG Dereferencing, current value:
+[2024-11-27T05:24:34]    DEBUG   hello: '{{ recipient }}'
+[2024-11-27T05:24:34]    DEBUG   recipient: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: {{ recipient }}
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: world
+[2024-11-27T05:24:34]    DEBUG Dereferencing, current value:
+[2024-11-27T05:24:34]    DEBUG   hello: world
+[2024-11-27T05:24:34]    DEBUG   recipient: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: world
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: world
+[2024-11-27T05:24:34]    DEBUG Dereferencing, final value:
+[2024-11-27T05:24:34]    DEBUG   hello: world
+[2024-11-27T05:24:34]    DEBUG   recipient: world
+[2024-11-27T05:24:34]    DEBUG Writing output to stdout
 hello: world
 recipient: world

docs/sections/user_guide/cli/tools/config/validate-verbose.out

@@ -1,21 +1,21 @@
-[2024-08-26T22:54:28]    DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose
-[2024-08-26T22:54:28]    DEBUG Using schema file: schema.jsonschema
-[2024-08-26T22:54:28]    DEBUG Dereferencing, current value:
-[2024-08-26T22:54:28]    DEBUG   values:
-[2024-08-26T22:54:28]    DEBUG     greeting: Hello
-[2024-08-26T22:54:28]    DEBUG     recipient: World
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendering: Hello
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendered: Hello
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendering: greeting
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendered: greeting
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendering: World
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendered: World
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendering: recipient
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendered: recipient
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendering: values
-[2024-08-26T22:54:28]    DEBUG [dereference] Rendered: values
-[2024-08-26T22:54:28]    DEBUG Dereferencing, final value:
-[2024-08-26T22:54:28]    DEBUG   values:
-[2024-08-26T22:54:28]    DEBUG     greeting: Hello
-[2024-08-26T22:54:28]    DEBUG     recipient: World
-[2024-08-26T22:54:29]     INFO 0 UW schema-validation errors found in config
+[2024-11-27T05:24:34]    DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose
+[2024-11-27T05:24:34]    DEBUG Using schema file: schema.jsonschema
+[2024-11-27T05:24:34]    DEBUG Dereferencing, current value:
+[2024-11-27T05:24:34]    DEBUG   values:
+[2024-11-27T05:24:34]    DEBUG     greeting: Hello
+[2024-11-27T05:24:34]    DEBUG     recipient: World
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: values
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: values
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: greeting
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: greeting
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: Hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: Hello
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: recipient
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendering: World
+[2024-11-27T05:24:34]    DEBUG [dereference] Rendered: World
+[2024-11-27T05:24:34]    DEBUG Dereferencing, final value:
+[2024-11-27T05:24:34]    DEBUG   values:
+[2024-11-27T05:24:34]    DEBUG     greeting: Hello
+[2024-11-27T05:24:34]    DEBUG     recipient: World
+[2024-11-27T05:24:34]     INFO 0 UW schema-validation errors found in config

docs/sections/user_guide/yaml/tags.rst

@@ -26,7 +26,7 @@ Additionally, UW defines the following tags to support use cases not covered by
 ``!bool``
 ^^^^^^^^^
 
-Converts the tagged node to a Python ``boolean`` object. For example, given ``input.yaml``:
+Converts the tagged node to a Python ``bool`` object. For example, given ``input.yaml``:
 
 .. code-block:: yaml
 
@@ -35,7 +35,7 @@ Converts the tagged node to a Python ``boolean`` object. For example, given ``in
 
 .. code-block:: text
 
-   % uw config realize -i ../input.yaml --output-format yaml
+   $ uw config realize -i ../input.yaml --output-format yaml
    flag1: True
    flag2: True
 
@@ -52,7 +52,7 @@ Converts the tagged node to a Python ``datetime`` object. For example, given ``i
 
 .. code-block:: text
 
-   % uw config realize -i ../input.yaml --output-format yaml
+   $ uw config realize -i ../input.yaml --output-format yaml
    date1: 2024-09-01
    date2: 2024-09-01 00:00:00
 
@@ -69,49 +69,76 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input
 
 .. code-block:: text
 
-   % uw config realize --input-file input.yaml --output-format yaml
+   $ uw config realize --input-file input.yaml --output-format yaml
    f2: 5.859
 
-``!int``
-^^^^^^^^
+``!include``
+^^^^^^^^^^^^
 
-Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``:
+Load and parse the files specified in the tagged sequence value and insert their contents here. For example, given ``numbers.yaml``:
 
 .. code-block:: yaml
 
-   f1: 3
-   f2: 11
-   f3: !int "{{ (f1 + f2) * 10 }}"
+   values: !include [constants.yaml]
+
+and ``constants.yaml``:
+
+.. code-block:: yaml
+
+   e: 2.718
+   pi: 3.141
 
 .. code-block:: text
 
-   % uw config realize --input-file input.yaml --output-format yaml
-   f1: 3
-   f2: 11
-   f2: 140
+   $ uw config realize --input-file numbers.yaml --output-format yaml
+   values:
+     e: 2.718
+     pi: 3.141
 
-``!include``
-^^^^^^^^^^^^
+Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, given ``numbers.yaml``:
+
+.. code-block:: yaml
+
+   values: !include [e.yaml, pi.yaml]
 
-Parse the tagged file and include its tags. For example, given ``input.yaml``:
+``e.yaml``:
 
 .. code-block:: yaml
 
-   values: !include [./supplemental.yaml]
+   constants:
+     e: 2.718
 
-and ``supplemental.yaml``:
+and ``pi.yaml``:
 
 .. code-block:: yaml
 
-   e: 2.718
-   pi: 3.141
+   constants:
+     pi: 3.141
 
 .. code-block:: text
 
-   % uw config realize --input-file input.yaml --output-format yaml
+   $ uw config realize --input-file numbers.yaml --output-format yaml
    values:
-      e: 2.718
-      pi: 3.141
+     constants:
+       pi: 3.141
+
+``!int``
+^^^^^^^^
+
+Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``:
+
+.. code-block:: yaml
+
+   f1: 3
+   f2: 11
+   f3: !int "{{ (f1 + f2) * 10 }}"
+
+.. code-block:: text
+
+   $ uw config realize --input-file input.yaml --output-format yaml
+   f1: 3
+   f2: 11
+   f2: 140
 
 ``!remove``
 ^^^^^^^^^^^
@@ -131,5 +158,5 @@ and ``update.yaml``:
 
 .. code-block:: text
 
-   % uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml
+   $ uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml
    pi: 3.141

recipe/meta.json

@@ -21,7 +21,8 @@
       "pytest-cov =5.0.*",
       "pytest-xdist =3.6.*",
       "python >=3.9,<3.13",
-      "pyyaml =6.0.*"
+      "pyyaml =6.0.*",
+      "setuptools"
     ],
     "run": [
       "f90nml =1.4.*",

recipe/meta.yaml

@@ -12,6 +12,7 @@ build:
 requirements:
   build:
     - pip
+    - setuptools
   run:
     - f90nml 1.4.*
     - iotaa 0.8.*

src/uwtools/config/formats/yaml.py

@@ -10,9 +10,9 @@
 from uwtools.config.support import (
     INCLUDE_TAG,
     UWYAMLConvert,
-    UWYAMLRemove,
     from_od,
     log_and_error,
+    uw_yaml_loader,
     yaml_to_str,
 )
 from uwtools.exceptions import UWConfigError
@@ -94,10 +94,9 @@ def _load(self, config_file: Optional[Path]) -> dict:
 
         :param config_file: Path to config file to load.
         """
-        loader = self._yaml_loader
         with readable(config_file) as f:
             try:
-                config = yaml.load(f.read(), Loader=loader)
+                config = yaml.load(f.read(), Loader=self._yaml_loader)
                 if isinstance(config, dict):
                     return config
                 t = type(config).__name__
@@ -157,13 +156,10 @@ def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict:
     @property
     def _yaml_loader(self) -> type[yaml.SafeLoader]:
         """
-        The loader, with appropriate constructors added.
+        A loader with all UW constructors added.
         """
-        loader = yaml.SafeLoader
+        loader = uw_yaml_loader()
         loader.add_constructor(INCLUDE_TAG, self._yaml_include)
-        for tag_class in (UWYAMLConvert, UWYAMLRemove):
-            for tag in getattr(tag_class, "TAGS"):
-                loader.add_constructor(tag, tag_class)
         return loader
 
     # Public methods

src/uwtools/config/jinja2.py

@@ -8,10 +8,12 @@
 from pathlib import Path
 from typing import Optional, Union
 
+import yaml
 from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta
 from jinja2.exceptions import UndefinedError
 
-from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config
+from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, uw_yaml_loader
+from uwtools.exceptions import UWConfigRealizeError
 from uwtools.logging import INDENT, MSGWIDTH, log
 from uwtools.utils.file import get_file_format, readable, writable
 
@@ -122,19 +124,19 @@ def dereference(
     :param keys: The dict keys leading to this value.
     :return: The input value, with Jinja2 syntax rendered.
     """
-    rendered: _ConfigVal = val  # fall-back value
+    rendered: _ConfigVal
     if isinstance(val, dict):
         keys = keys or []
-        new = {}
+        rendered = {}
         for k, v in val.items():
             if isinstance(v, UWYAMLRemove):
-                _deref_debug("Removing value at", " > ".join([*keys, k]))
+                _deref_debug("Removing value at", ".".join([*keys, k]))
             else:
-                new[dereference(k, context)] = dereference(v, context, local=val, keys=[*keys, k])
-        return new
-    if isinstance(val, list):
-        return [dereference(v, context) for v in val]
-    if isinstance(val, str):
+                kd, vd = [dereference(x, context, val, [*keys, k]) for x in (k, v)]
+                rendered[kd] = vd
+    elif isinstance(val, list):
+        rendered = [dereference(v, context) for v in val]
+    elif isinstance(val, str):
         _deref_debug("Rendering", val)
         rendered = _deref_render(val, context, local)
     elif isinstance(val, UWYAMLConvert):
@@ -143,6 +145,7 @@ def dereference(
         rendered = _deref_convert(val)
     else:
         _deref_debug("Accepting", val)
+        rendered = val
     return rendered
 
 
@@ -266,6 +269,9 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str:
     context = {**(local or {}), **context}
     try:
         rendered = _register_filters(env).from_string(val).render(context)
+        if isinstance(yaml.load(rendered, Loader=uw_yaml_loader()), UWYAMLConvert):
+            _deref_debug("Held", rendered)
+            raise UWConfigRealizeError()
         _deref_debug("Rendered", rendered)
     except Exception as e:  # pylint: disable=broad-exception-caught
         rendered = val

src/uwtools/config/support.py

@@ -68,6 +68,17 @@ def log_and_error(msg: str) -> Exception:
     return UWConfigError(msg)
 
 
+def uw_yaml_loader() -> type[yaml.SafeLoader]:
+    """
+    A loader with basic UW constructors added.
+    """
+    loader = yaml.SafeLoader
+    for tag_class in (UWYAMLConvert, UWYAMLRemove):
+        for tag in getattr(tag_class, "TAGS"):
+            loader.add_constructor(tag, tag_class)
+    return loader
+
+
 def yaml_to_str(cfg: dict) -> str:
     """
     Return a uwtools-conventional YAML representation of the given dict.