diff --git a/docs/sections/user_guide/cli/tools/config/realize-verbose.out b/docs/sections/user_guide/cli/tools/config/realize-verbose.out index cac8e0711..707d30d2a 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/realize-verbose.out @@ -1,30 +1,30 @@ -[2024-11-27T05:24:34] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose -[2024-11-27T05:24:34] DEBUG Reading input from stdin -[2024-11-27T05:24:34] DEBUG Dereferencing, current value: -[2024-11-27T05:24:34] DEBUG hello: '{{ recipient }}' -[2024-11-27T05:24:34] DEBUG recipient: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: {{ recipient }} -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world -[2024-11-27T05:24:34] DEBUG Dereferencing, current value: -[2024-11-27T05:24:34] DEBUG hello: world -[2024-11-27T05:24:34] DEBUG recipient: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world -[2024-11-27T05:24:34] DEBUG Dereferencing, final value: -[2024-11-27T05:24:34] DEBUG hello: world -[2024-11-27T05:24:34] DEBUG recipient: world -[2024-11-27T05:24:34] DEBUG Writing output to stdout +[2025-01-05T21:15:07] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose +[2025-01-05T21:15:07] DEBUG Reading input from stdin +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, current value: +[2025-01-05T21:15:07] DEBUG [dereference] hello: '{{ recipient }}' +[2025-01-05T21:15:07] DEBUG [dereference] recipient: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: {{ recipient }} +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: world +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, current value: +[2025-01-05T21:15:07] DEBUG [dereference] hello: world +[2025-01-05T21:15:07] DEBUG [dereference] recipient: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: world +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: world +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, final value: +[2025-01-05T21:15:07] DEBUG [dereference] hello: world +[2025-01-05T21:15:07] DEBUG [dereference] recipient: world +[2025-01-05T21:15:07] DEBUG Writing output to stdout hello: world recipient: world diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index fb76c5dc8..cf60c3196 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2024-11-27T05:24:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2024-11-27T05:24:34] DEBUG Using schema file: schema.jsonschema -[2024-11-27T05:24:34] DEBUG Dereferencing, current value: -[2024-11-27T05:24:34] DEBUG values: -[2024-11-27T05:24:34] DEBUG greeting: Hello -[2024-11-27T05:24:34] DEBUG recipient: World -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: values -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: values -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: greeting -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: greeting -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: Hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: Hello -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient -[2024-11-27T05:24:34] DEBUG [dereference] Rendering: World -[2024-11-27T05:24:34] DEBUG [dereference] Rendered: World -[2024-11-27T05:24:34] DEBUG Dereferencing, final value: -[2024-11-27T05:24:34] DEBUG values: -[2024-11-27T05:24:34] DEBUG greeting: Hello -[2024-11-27T05:24:34] DEBUG recipient: World -[2024-11-27T05:24:34] INFO 0 schema-validation errors found in config +[2025-01-05T21:15:07] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2025-01-05T21:15:07] DEBUG Using schema file: schema.jsonschema +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, current value: +[2025-01-05T21:15:07] DEBUG [dereference] values: +[2025-01-05T21:15:07] DEBUG [dereference] greeting: Hello +[2025-01-05T21:15:07] DEBUG [dereference] recipient: World +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: values +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: values +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: greeting +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: greeting +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: Hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: Hello +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: recipient +[2025-01-05T21:15:07] DEBUG [dereference] Rendering: World +[2025-01-05T21:15:07] DEBUG [dereference] Rendered: World +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, final value: +[2025-01-05T21:15:07] DEBUG [dereference] values: +[2025-01-05T21:15:07] DEBUG [dereference] greeting: Hello +[2025-01-05T21:15:07] DEBUG [dereference] recipient: World +[2025-01-05T21:15:07] INFO 0 schema-validation errors found in config diff --git a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out index 70aad1c6e..b5f00ea45 100644 --- a/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out +++ b/docs/sections/user_guide/cli/tools/rocoto/realize-exec-stdout-verbose.out @@ -1,21 +1,21 @@ -[2024-08-26T23:39:19] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose -[2024-08-26T23:39:19] DEBUG Dereferencing, current value: -[2024-08-26T23:39:19] DEBUG workflow: -[2024-08-26T23:39:19] DEBUG attrs: -[2024-08-26T23:39:19] DEBUG realtime: false -[2024-08-26T23:39:19] DEBUG scheduler: slurm -[2024-08-26T23:39:19] DEBUG cycledef: -[2024-08-26T23:39:19] DEBUG - attrs: -[2024-08-26T23:39:19] DEBUG group: howdy -[2024-08-26T23:39:19] DEBUG spec: 202209290000 202209300000 06:00:00 +[2025-01-05T21:15:07] DEBUG Command: uw rocoto realize --config-file rocoto.yaml --verbose +[2025-01-05T21:15:07] DEBUG [dereference] Dereferencing, current value: +[2025-01-05T21:15:07] DEBUG [dereference] workflow: +[2025-01-05T21:15:07] DEBUG [dereference] attrs: +[2025-01-05T21:15:07] DEBUG [dereference] realtime: false +[2025-01-05T21:15:07] DEBUG [dereference] scheduler: slurm +[2025-01-05T21:15:07] DEBUG [dereference] cycledef: +[2025-01-05T21:15:07] DEBUG [dereference] - attrs: +[2025-01-05T21:15:07] DEBUG [dereference] group: howdy +[2025-01-05T21:15:07] DEBUG [dereference] spec: 202209290000 202209300000 06:00:00 ... -[2024-08-26T23:39:20] DEBUG cycledefs: howdy -[2024-08-26T23:39:20] DEBUG account: '&ACCOUNT;' -[2024-08-26T23:39:20] DEBUG command: echo hello $person -[2024-08-26T23:39:20] DEBUG jobname: hello -[2024-08-26T23:39:20] DEBUG native: --reservation my_reservation -[2024-08-26T23:39:20] DEBUG nodes: 1:ppn=1 -[2024-08-26T23:39:20] DEBUG walltime: 00:01:00 -[2024-08-26T23:39:20] DEBUG envars: -[2024-08-26T23:39:20] DEBUG person: siri -[2024-08-26T23:39:20] INFO 0 Rocoto XML validation errors found +[2025-01-05T21:15:07] DEBUG [dereference] cycledefs: howdy +[2025-01-05T21:15:07] DEBUG [dereference] account: '&ACCOUNT;' +[2025-01-05T21:15:07] DEBUG [dereference] command: echo hello $person +[2025-01-05T21:15:07] DEBUG [dereference] jobname: hello +[2025-01-05T21:15:07] DEBUG [dereference] native: --reservation my_reservation +[2025-01-05T21:15:07] DEBUG [dereference] nodes: 1:ppn=1 +[2025-01-05T21:15:07] DEBUG [dereference] walltime: 00:01:00 +[2025-01-05T21:15:07] DEBUG [dereference] envars: +[2025-01-05T21:15:07] DEBUG [dereference] person: siri +[2025-01-05T21:15:07] INFO 0 Rocoto XML validation errors found diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index e768d74e5..8df1ab617 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -21,7 +21,9 @@ Or explicit: integer: !!int "3" float: !!float "3.14" -Additionally, UW defines the following tags to support use cases not covered by standard tags: +Additionally, UW defines the following tags to support use cases not covered by standard tags. Where standard YAML tags are applied to their values immediately, application of UW YAML tags is delayed until after Jinja2 expressions in tagged values are dereferenced. + +**NB** Values tagged with UW YAML tags must be strings. Use quotes as necessary to ensure that they are. ``!bool`` ^^^^^^^^^ @@ -32,12 +34,14 @@ Converts the tagged node to a Python ``bool`` object. For example, given ``input flag1: True flag2: !bool "{{ flag1 }}" + flag3: !bool "0" .. code-block:: text $ uw config realize -i ../input.yaml --output-format yaml flag1: True flag2: True + flag3: False ``!datetime`` @@ -58,6 +62,26 @@ Converts the tagged node to a Python ``datetime`` object. For example, given ``i The value provided to the tag must be in :python:`ISO 8601 format` to be interpreted correctly by the ``!datetime`` tag. +``!dict`` +^^^^^^^^^ + +Converts the tagged node to a Python ``dict`` value. For example, given ``input.yaml``: + +.. code-block:: yaml + + d1: {'k0': 0, 'k1': 1, 'k2': 2} + d2: !dict "{ k0: 0, k1: 1, k2: 2 }" + d3: !dict "{{ '{' }}{% for n in range(3) %} k{{ n }}:{{ n }},{% endfor %}{{ '}' }}" + d4: !dict "[{% for n in range(3) %}[k{{ n }},{{ n }}],{% endfor %}]" + +.. code-block:: text + + $ uw config realize --input-file input.yaml --output-format yaml + d1: {'k0': 0, 'k1': 1, 'k2': 2} + d2: {'k0': 0, 'k1': 1, 'k2': 2} + d3: {'k0': 0, 'k1': 1, 'k2': 2} + d4: {'k0': 0, 'k1': 1, 'k2': 2} + ``!float`` ^^^^^^^^^^ @@ -140,6 +164,24 @@ Converts the tagged node to a Python ``int`` value. For example, given ``input.y f2: 11 f2: 140 +``!list`` +^^^^^^^^^ + +Converts the tagged node to a Python ``list`` value. For example, given ``input.yaml``: + +.. code-block:: yaml + + l1: [1, 2, 3] + l2: !list "[{% for n in range(3) %} a{{ n }},{% endfor %} ]" + l3: !list "[ a0, a1, a2, ]" + +.. code-block:: text + + $ uw config realize --input-file input.yaml --output-format yaml + l1: [1, 2, 3] + l2: ['a0', 'a1', 'a2'] + l3: ['a0', 'a1', 'a2'] + ``!remove`` ^^^^^^^^^^^ diff --git a/notebooks/config.ipynb b/notebooks/config.ipynb index 5a23040b0..e49a44ae1 100644 --- a/notebooks/config.ipynb +++ b/notebooks/config.ipynb @@ -336,7 +336,7 @@ "text": [ "Help on function realize in module uwtools.api.config:\n", "\n", - "realize(input_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, input_format: Optional[str] = None, update_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, update_format: Optional[str] = None, output_file: Union[str, pathlib.Path, NoneType] = None, output_format: Optional[str] = None, key_path: Optional[list[Union[str, int]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> dict\n", + "realize(input_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, input_format: Optional[str] = None, update_config: Union[uwtools.config.formats.base.Config, pathlib.Path, dict, str, NoneType] = None, update_format: Optional[str] = None, output_file: Union[str, pathlib.Path, NoneType] = None, output_format: Optional[str] = None, key_path: Optional[list[Union[bool, float, int, str]]] = None, values_needed: bool = False, total: bool = False, dry_run: bool = False, stdin_ok: bool = False) -> dict\n", " Realize a config based on a base input config and an optional update config.\n", "\n", " The input config may be specified as a filesystem path, a ``dict``, or a ``Config`` object. When it\n", @@ -1491,6 +1491,12 @@ " | :param src: The dictionary with new data to use.\n", " |\n", " | ----------------------------------------------------------------------\n", + " | Readonly properties inherited from uwtools.config.formats.base.Config:\n", + " |\n", + " | config_file\n", + " | Return the path to the config file from which this object was instantiated, if applicable.\n", + " |\n", + " | ----------------------------------------------------------------------\n", " | Data descriptors inherited from uwtools.config.formats.base.Config:\n", " |\n", " | __dict__\n", @@ -1555,7 +1561,7 @@ " |\n", " | update(self, other=(), /, **kwds)\n", " | D.update([E, ]**F) -> None. Update D from mapping/iterable E and F.\n", - " | If E present and has a .keys() method, does: for k in E: D[k] = E[k]\n", + " | If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k]\n", " | If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v\n", " | In either case, this is followed by: for k, v in F.items(): D[k] = v\n", " |\n", diff --git a/notebooks/rocoto.ipynb b/notebooks/rocoto.ipynb index 92f29c3da..f8c9af3f4 100644 --- a/notebooks/rocoto.ipynb +++ b/notebooks/rocoto.ipynb @@ -139,8 +139,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] INFO 0 schema-validation errors found in Rocoto config\n", - "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" + "[2025-01-05T21:26:43] INFO 0 schema-validation errors found in Rocoto config\n", + "[2025-01-05T21:26:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -256,13 +256,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] ERROR 3 schema-validation errors found in Rocoto config\n", - "[2024-11-19T23:15:43] ERROR Error at workflow -> attrs:\n", - "[2024-11-19T23:15:43] ERROR 'realtime' is a required property\n", - "[2024-11-19T23:15:43] ERROR Error at workflow -> tasks -> task_greet:\n", - "[2024-11-19T23:15:43] ERROR 'command' is a required property\n", - "[2024-11-19T23:15:43] ERROR Error at workflow:\n", - "[2024-11-19T23:15:43] ERROR 'log' is a required property\n" + "[2025-01-05T21:26:43] ERROR 3 schema-validation errors found in Rocoto config\n", + "[2025-01-05T21:26:43] ERROR Error at workflow.attrs:\n", + "[2025-01-05T21:26:43] ERROR 'realtime' is a required property\n", + "[2025-01-05T21:26:43] ERROR Error at workflow.tasks.task_greet:\n", + "[2025-01-05T21:26:43] ERROR 'command' is a required property\n", + "[2025-01-05T21:26:43] ERROR Error at workflow:\n", + "[2025-01-05T21:26:43] ERROR 'log' is a required property\n" ] }, { @@ -388,8 +388,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] INFO 0 schema-validation errors found in Rocoto config\n", - "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" + "[2025-01-05T21:26:43] INFO 0 schema-validation errors found in Rocoto config\n", + "[2025-01-05T21:26:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -577,8 +577,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] INFO 0 schema-validation errors found in Rocoto config\n", - "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" + "[2025-01-05T21:26:43] INFO 0 schema-validation errors found in Rocoto config\n", + "[2025-01-05T21:26:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -722,8 +722,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] INFO 0 schema-validation errors found in Rocoto config\n", - "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" + "[2025-01-05T21:26:43] INFO 0 schema-validation errors found in Rocoto config\n", + "[2025-01-05T21:26:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -925,7 +925,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] INFO 0 Rocoto XML validation errors found\n" + "[2025-01-05T21:26:43] INFO 0 Rocoto XML validation errors found\n" ] }, { @@ -1001,22 +1001,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "[2024-11-19T23:15:43] ERROR 4 Rocoto XML validation errors found\n", - "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", - "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", - "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", - "[2024-11-19T23:15:43] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", - "[2024-11-19T23:15:43] ERROR Invalid Rocoto XML:\n", - "[2024-11-19T23:15:43] ERROR 1 \n", - "[2024-11-19T23:15:43] ERROR 2 \n", - "[2024-11-19T23:15:43] ERROR 3 logs/test.log\n", - "[2024-11-19T23:15:43] ERROR 4 \n", - "[2024-11-19T23:15:43] ERROR 5 1\n", - "[2024-11-19T23:15:43] ERROR 6 00:00:10\n", - "[2024-11-19T23:15:43] ERROR 7 echo Hello, World!\n", - "[2024-11-19T23:15:43] ERROR 8 greet\n", - "[2024-11-19T23:15:43] ERROR 9 \n", - "[2024-11-19T23:15:43] ERROR 10 \n" + "[2025-01-05T21:26:44] ERROR 4 Rocoto XML validation errors found\n", + "[2025-01-05T21:26:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_ATTRVALID: Element workflow failed to validate attributes\n", + "[2025-01-05T21:26:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_NOELEM: Expecting an element cycledef, got nothing\n", + "[2025-01-05T21:26:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_INTERSEQ: Invalid sequence in interleave\n", + "[2025-01-05T21:26:44] ERROR :2:0:ERROR:RELAXNGV:RELAXNG_ERR_CONTENTVALID: Element workflow failed to validate content\n", + "[2025-01-05T21:26:44] ERROR Invalid Rocoto XML:\n", + "[2025-01-05T21:26:44] ERROR 1 \n", + "[2025-01-05T21:26:44] ERROR 2 \n", + "[2025-01-05T21:26:44] ERROR 3 logs/test.log\n", + "[2025-01-05T21:26:44] ERROR 4 \n", + "[2025-01-05T21:26:44] ERROR 5 1\n", + "[2025-01-05T21:26:44] ERROR 6 00:00:10\n", + "[2025-01-05T21:26:44] ERROR 7 echo Hello, World!\n", + "[2025-01-05T21:26:44] ERROR 8 greet\n", + "[2025-01-05T21:26:44] ERROR 9 \n", + "[2025-01-05T21:26:44] ERROR 10 \n" ] }, { diff --git a/src/uwtools/config/formats/base.py b/src/uwtools/config/formats/base.py index 37b736226..ff3f18671 100644 --- a/src/uwtools/config/formats/base.py +++ b/src/uwtools/config/formats/base.py @@ -5,14 +5,13 @@ from collections import UserDict from copy import deepcopy from io import StringIO -from math import inf from pathlib import Path from typing import Optional, Union import yaml from uwtools.config import jinja2 -from uwtools.config.support import INCLUDE_TAG, depth, log_and_error, yaml_to_str +from uwtools.config.support import INCLUDE_TAG, depth, dict_to_yaml_str, log_and_error from uwtools.exceptions import UWConfigError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import str2path @@ -87,8 +86,8 @@ def _compare_config_get_lines(d: dict) -> list[str]: :param d: A dict object. """ sio = StringIO() - yaml.safe_dump(d, stream=sio, default_flow_style=False, indent=2, width=inf) - return sio.getvalue().splitlines(keepends=True) + sio.write(dict_to_yaml_str(d, sort=True)) + return sio.getvalue().splitlines(keepends=False) @staticmethod def _compare_config_log_header() -> None: @@ -224,7 +223,7 @@ def dereference(self, context: Optional[dict] = None) -> None: def logstate(state: str) -> None: jinja2.deref_debug("Dereferencing, %s value:" % state) - for line in yaml_to_str(self.data).split("\n"): + for line in dict_to_yaml_str(self.data).split("\n"): jinja2.deref_debug("%s%s" % (INDENT, line)) while True: diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 71bc870f8..c755d6f2b 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -10,10 +10,10 @@ from uwtools.config.support import ( INCLUDE_TAG, UWYAMLConvert, + dict_to_yaml_str, from_od, log_and_error, uw_yaml_loader, - yaml_to_str, ) from uwtools.exceptions import UWConfigError from uwtools.strings import FORMAT @@ -70,7 +70,7 @@ def _dict_to_str(cls, cfg: dict) -> str: :param cfg: The in-memory config object. """ cls._add_yaml_representers() - return yaml_to_str(cfg) + return dict_to_yaml_str(cfg) @staticmethod def _get_depth_threshold() -> Optional[int]: diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 0bce1dea2..82d75081e 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -148,14 +148,16 @@ def dereference( return rendered -def deref_debug(action: str, val: Optional[_ConfigVal] = "") -> None: +def deref_debug(action: str, val: Optional[_ConfigVal] = None) -> None: """ Log a debug-level message related to dereferencing. :param action: The dereferencing activity being performed. :param val: The value being dereferenced. """ - log.debug("[dereference] %s: %s", action, val) + tag = "[dereference]" + args = ("%s %s", tag, action) if val is None else ("%s %s: %s", tag, action, val) + log.debug(*args) def render( @@ -224,9 +226,9 @@ def unrendered(s: str) -> bool: """ try: Environment(undefined=StrictUndefined).from_string(s).render({}) - return False except UndefinedError: return True + return False # Private functions @@ -245,10 +247,11 @@ def _deref_convert(val: UWYAMLConvert) -> _ConfigVal: converted: _ConfigVal = val # fall-back value deref_debug("Converting", val.value) try: - converted = val.convert() - deref_debug("Converted", converted) + converted = val.converted except Exception as e: # pylint: disable=broad-exception-caught deref_debug("Conversion failed", str(e)) + else: + deref_debug("Converted", converted) return converted @@ -264,16 +267,18 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: :param local: Local sibling values to use if a match is not found in context. :return: The rendered value (potentially unchanged). """ - env = Environment(undefined=StrictUndefined) + env = _register_filters(Environment(undefined=StrictUndefined)) + template = env.from_string(val) context = {**(local or {}), **context} try: - rendered = _register_filters(env).from_string(val).render(context) - deref_debug("Rendered", rendered) + rendered = template.render(context) except Exception as e: # pylint: disable=broad-exception-caught rendered = val deref_debug("Rendering failed", val) for line in str(e).split("\n"): deref_debug(line) + else: + deref_debug("Rendered", rendered) try: loaded = yaml.load(rendered, Loader=uw_yaml_loader()) except Exception as e: # pylint: disable=broad-exception-caught diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index 713c12419..9dae32268 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -3,6 +3,7 @@ import math from collections import OrderedDict from datetime import datetime +from functools import partial from importlib import import_module from typing import Callable, Type, Union @@ -79,13 +80,14 @@ def uw_yaml_loader() -> type[yaml.SafeLoader]: return loader -def yaml_to_str(cfg: dict) -> str: +def dict_to_yaml_str(d: dict, sort: bool = False) -> str: """ Return a uwtools-conventional YAML representation of the given dict. - :param cfg: A dict object. + :param d: A dict object. + :param sort: Sort dict/mapping keys? """ - return yaml.dump(cfg, default_flow_style=False, sort_keys=False, width=math.inf).strip() + return yaml.dump(d, default_flow_style=False, indent=2, sort_keys=sort, width=math.inf).strip() class UWYAMLTag: @@ -119,23 +121,45 @@ class UWYAMLConvert(UWYAMLTag): method. See the pyyaml documentation for details. """ - TAGS = ("!bool", "!datetime", "!float", "!int") + TAGS = ("!bool", "!datetime", "!dict", "!float", "!int", "!list") + TaggedValT = Union[bool, datetime, dict, float, int, list] - def convert(self) -> Union[datetime, float, int]: + def __init__(self, loader: yaml.SafeLoader, node: yaml.nodes.ScalarNode) -> None: + super().__init__(loader, node) + if not isinstance(self.value, str): + hint = ( + "%s %s" % (node.tag, node.value) + if node.start_mark is None + else node.start_mark.buffer.replace("\n\x00", "") + ) + raise UWConfigError( + "Value tagged %s must be type 'str' (not '%s') in: %s" + % (node.tag, node.value.__class__.__name__, hint) + ) + + def __repr__(self) -> str: + return "%s %s" % (self.tag, self.converted) + + def __str__(self) -> str: + return str(self.converted) + + @property + def converted(self) -> UWYAMLConvert.TaggedValT: """ - Return the original YAML value converted to the specified type. + Return the original YAML value converted to the type speficied by the tag. - Will raise an exception if the value cannot be represented as the specified type. + :raises: Appropriate exception if the value cannot be represented as the required type. """ - converters: dict[ - str, Union[Callable[[str], bool], Callable[[str], datetime], type[float], type[int]] - ] = dict( - zip( - self.TAGS, - [lambda x: {"True": True, "False": False}[x], datetime.fromisoformat, float, int], - ) - ) - return converters[self.tag](self.value) + load_as = lambda t, v: t(yaml.safe_load(v)) + converters: list[Callable[..., UWYAMLConvert.TaggedValT]] = [ + partial(load_as, bool), + datetime.fromisoformat, + partial(load_as, dict), + float, + int, + partial(load_as, list), + ] + return dict(zip(UWYAMLConvert.TAGS, converters))[self.tag](self.value) class UWYAMLRemove(UWYAMLTag): diff --git a/src/uwtools/tests/config/formats/test_base.py b/src/uwtools/tests/config/formats/test_base.py index c3557d3cc..4bb3baaad 100644 --- a/src/uwtools/tests/config/formats/test_base.py +++ b/src/uwtools/tests/config/formats/test_base.py @@ -210,20 +210,29 @@ def test_dereference(tmp_path): yaml = """ a: !int '{{ b.c + 11 }}' b: - c: !int '{{ N | int + 11 }}' + c: !int '{{ l | int + 11 }}' d: '{{ X }}' e: - - !int '42' - - !float '3.14' - - !datetime '{{ D }}' - !bool "False" + - !datetime '{{ i }}' + - !dict "{ b0: 0, b1: 1, b2: 2,}" + - !dict "[ ['c0',0], ['c1',1], ['c2',2], ]" + - !float '3.14' + - !int '42' + - !list "[ a0, a1, a2, ]" f: - f1: !int '42' + f1: True f2: !float '3.14' - f3: True + f3: !dict "{ b0: 0, b1: 1, b2: 2,}" + f4: !dict "[ ['c0',0], ['c1',1], ['c2',2], ]" + f5: !int '42' + f6: !list "[ 0, 1, 2, ]" g: !bool '{{ f.f3 }}' -D: 2024-10-10 00:19:00 -N: "22" +h: !bool 0 +i: 2024-10-10 00:19:00 +j: !dict "{ b0: 0, b1: 1, b2: 2,}" +k: !list "[ a0, a1, a2, ]" +l: "22" """.strip() path = tmp_path / "config.yaml" @@ -237,11 +246,29 @@ def test_dereference(tmp_path): "a": 44, "b": {"c": 33}, "d": "{{ X }}", - "e": [42, 3.14, datetime.fromisoformat("2024-10-10 00:19:00"), False], - "f": {"f1": 42, "f2": 3.14, "f3": True}, + "e": [ + False, + datetime.fromisoformat("2024-10-10 00:19:00"), + {"b0": 0, "b1": 1, "b2": 2}, + {"c0": 0, "c1": 1, "c2": 2}, + 3.14, + 42, + ["a0", "a1", "a2"], + ], + "f": { + "f1": True, + "f2": 3.14, + "f3": {"b0": 0, "b1": 1, "b2": 2}, + "f4": {"c0": 0, "c1": 1, "c2": 2}, + "f5": 42, + "f6": [0, 1, 2], + }, "g": True, - "D": datetime.fromisoformat("2024-10-10 00:19:00"), - "N": "22", + "h": False, + "i": datetime.fromisoformat("2024-10-10 00:19:00"), + "j": {"b0": 0, "b1": 1, "b2": 2}, + "k": ["a0", "a1", "a2"], + "l": "22", } diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 10c3c0f7c..9d4f69a7b 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -287,11 +287,20 @@ def test_unrendered(s, status): assert jinja2.unrendered(s) is status -@mark.parametrize("tag", ["!bool", "!datetime", "!float", "!int"]) -def test__deref_convert_no(caplog, tag): +@mark.parametrize( + "tag,value", + [ + ("!datetime", "foo"), + ("!dict", "foo"), + ("!float", "foo"), + ("!int", "foo"), + ("!list", "null"), + ], +) +def test__deref_convert_no(caplog, tag, value): log.setLevel(logging.DEBUG) loader = yaml.SafeLoader(os.devnull) - val = UWYAMLConvert(loader, yaml.ScalarNode(tag=tag, value="foo")) + val = UWYAMLConvert(loader, yaml.ScalarNode(tag=tag, value=value)) assert jinja2._deref_convert(val=val) == val assert not regex_logged(caplog, "Converted") assert regex_logged(caplog, "Conversion failed") @@ -301,9 +310,15 @@ def test__deref_convert_no(caplog, tag): "converted,tag,value", [ (True, "!bool", "True"), + (False, "!bool", "0"), (datetime(2024, 9, 9, 0, 0), "!datetime", "2024-09-09 00:00:00"), + ({"a": 0, "b": 1}, "!dict", "{a: 0, b: 1}"), + ({"a": 0, "b": 1}, "!dict", "[[a, 0], [b, 1]]"), (3.14, "!float", "3.14"), (42, "!int", "42"), + ([0, 1, 2], "!list", "[0, 1, 2]"), + (["f", "o", "o"], "!list", "foo"), + ([0, 1, 2], "!list", "{0: a, 1: b, 2: c}"), ], ) def test__deref_convert_ok(caplog, converted, tag, value): @@ -317,7 +332,7 @@ def test__deref_convert_ok(caplog, converted, tag, value): def test__deref_render_held(caplog): log.setLevel(logging.DEBUG) - val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=uw_yaml_loader()) + val, context = "!int '{{ a }}'", yaml.load("a: !int '42'", Loader=uw_yaml_loader()) assert jinja2._deref_render(val=val, context=context) == val assert regex_logged(caplog, "Rendered") assert regex_logged(caplog, "Held") diff --git a/src/uwtools/tests/config/test_support.py b/src/uwtools/tests/config/test_support.py index c04b27f41..841b094e6 100644 --- a/src/uwtools/tests/config/test_support.py +++ b/src/uwtools/tests/config/test_support.py @@ -23,10 +23,20 @@ @mark.parametrize("d,n", [({1: 42}, 1), ({1: {2: 42}}, 2), ({1: {2: {3: 42}}}, 3), ({1: {}}, 2)]) -def test_depth(d, n): +def test_config_support_depth(d, n): assert support.depth(d) == n +def test_config_support_dict_to_yaml_str(capsys): + xs = " ".join("x" * 999) + expected = f"xs: {xs}" + cfgobj = YAMLConfig({"xs": xs}) + assert repr(cfgobj) == expected + assert str(cfgobj) == expected + cfgobj.dump() + assert capsys.readouterr().out.strip() == expected + + @mark.parametrize( "cfgtype,fmt", [ @@ -37,22 +47,22 @@ def test_depth(d, n): (YAMLConfig, FORMAT.yaml), ], ) -def test_format_to_config(cfgtype, fmt): +def test_config_support_format_to_config(cfgtype, fmt): assert support.format_to_config(fmt) is cfgtype -def test_format_to_config_fail(): +def test_config_support_format_to_config_fail(): with raises(UWConfigError): support.format_to_config("no-such-config-type") -def test_from_od(): +def test_config_support_from_od(): assert support.from_od(d=OrderedDict([("example", OrderedDict([("key", "value")]))])) == { "example": {"key": "value"} } -def test_log_and_error(caplog): +def test_config_support_log_and_error(caplog): log.setLevel(logging.ERROR) msg = "Something bad happened" with raises(UWConfigError) as e: @@ -61,16 +71,6 @@ def test_log_and_error(caplog): assert logged(caplog, msg) -def test_yaml_to_str(capsys): - xs = " ".join("x" * 999) - expected = f"xs: {xs}" - cfgobj = YAMLConfig({"xs": xs}) - assert repr(cfgobj) == expected - assert str(cfgobj) == expected - cfgobj.dump() - assert capsys.readouterr().out.strip() == expected - - class Test_UWYAMLConvert: """ Tests for class uwtools.config.support.UWYAMLConvert. @@ -84,55 +84,90 @@ def loader(self): yaml.add_representer(support.UWYAMLConvert, support.UWYAMLTag.represent) return yaml.SafeLoader("") + @mark.parametrize( + "tag,val,val_type", + [ + ("!bool", True, "bool"), + ("!dict", {1: 2}, "dict"), + ("!float", 3.14, "float"), + ("!int", 42, "int"), + ("!list", [1, 2], "list"), + ], + ) + def test_UWYAMLConvert_bad_non_str(self, loader, tag, val, val_type): + with raises(UWConfigError) as e: + support.UWYAMLConvert(loader, yaml.ScalarNode(tag=tag, value=val)) + msg = "Value tagged %s must be type 'str' (not '%s') in: %s %s" + assert str(e.value) == msg % (tag, val_type, tag, val) + # These tests bypass YAML parsing, constructing nodes with explicit string values. They then - # demonstrate that those nodes' convert() methods return representations in type type specified + # demonstrate that those nodes' convert() methods return representations in the type specified # by the tag. - def test_bool_bad(self, loader): - ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!bool", value="foo")) - with raises(KeyError): - ts.convert() - @mark.parametrize("value, expected", [("False", False), ("True", True)]) - def test_bool_values(self, expected, loader, value): + def test_UWYAMLConvert_bool_values(self, expected, loader, value): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!bool", value=value)) - assert ts.convert() == expected + assert ts.converted == expected - def test_datetime_no(self, loader): + def test_UWYAMLConvert_datetime_no(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!datetime", value="foo")) with raises(ValueError): - ts.convert() + assert ts.converted - def test_datetime_ok(self, loader): + def test_UWYAMLConvert_datetime_ok(self, loader): ts = support.UWYAMLConvert( loader, yaml.ScalarNode(tag="!datetime", value="2024-08-09 12:22:42") ) - assert ts.convert() == datetime(2024, 8, 9, 12, 22, 42) + assert ts.converted == datetime(2024, 8, 9, 12, 22, 42) self.comp(ts, "!datetime '2024-08-09 12:22:42'") - def test_float_no(self, loader): + def test_UWYAMLConvert_dict_no(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!dict", value="42")) + with raises(TypeError): + assert ts.converted + + def test_UWYAMLConvert_dict_ok(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!dict", value="{a0: 0,a1: 1,}")) + assert ts.converted == {"a0": 0, "a1": 1} + self.comp(ts, "!dict '{a0: 0,a1: 1,}'") + + def test_UWYAMLConvert_float_no(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!float", value="foo")) with raises(ValueError): - ts.convert() + assert ts.converted - def test_float_ok(self, loader): + def test_UWYAMLConvert_float_ok(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!float", value="3.14")) - assert ts.convert() == 3.14 + assert ts.converted == 3.14 self.comp(ts, "!float '3.14'") - def test_int_no(self, loader): + def test_UWYAMLConvert_int_no(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="foo")) with raises(ValueError): - ts.convert() + assert ts.converted - def test_int_ok(self, loader): + def test_UWYAMLConvert_int_ok(self, loader): ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="42")) - assert ts.convert() == 42 + assert ts.converted == 42 self.comp(ts, "!int '42'") - def test___repr__(self, loader): - ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!int", value="42")) - assert str(ts) == "!int 42" + def test_UWYAMLConvert_list_no(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!list", value="null")) + with raises(TypeError): + assert ts.converted + + def test_UWYAMLConvert_list_ok(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!list", value="[1,2,3,]")) + assert ts.converted == [1, 2, 3] + self.comp(ts, "!list '[1,2,3,]'") + + def test_UWYAMLConvert___repr__(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!list", value="[ 1,2,3, ]")) + assert repr(ts) == "!list [1, 2, 3]" + + def test_UWYAMLConvert___str__(self, loader): + ts = support.UWYAMLConvert(loader, yaml.ScalarNode(tag="!list", value="[ 1,2,3, ]")) + assert str(ts) == "[1, 2, 3]" class Test_UWYAMLRemove: @@ -140,7 +175,7 @@ class Test_UWYAMLRemove: Tests for class uwtools.config.support.UWYAMLRemove. """ - def test___repr__(self): + def test_UWYAMLRemove___str__(self): yaml.add_representer(support.UWYAMLRemove, support.UWYAMLTag.represent) node = support.UWYAMLRemove(yaml.SafeLoader(""), yaml.ScalarNode(tag="!remove", value="")) assert str(node) == "!remove" diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 6747b6f0b..0bbfea7be 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -279,7 +279,7 @@ def test_realize_config_double_tag_nest(tmp_path): help_realize_config_double_tag(config, expected, tmp_path) -def test_realize_config_double_tag_nest_forwrad_reference(tmp_path): +def test_realize_config_double_tag_nest_forward_reference(tmp_path): config = """ a: true b: false