From cb494cea6be7ddf49a9ec3349ed6de819ff01087 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 26 Nov 2024 22:58:37 +0000 Subject: [PATCH 01/31] Add failing test --- src/uwtools/tests/config/test_tools.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 69e5c77e7..7ff949e94 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -235,6 +235,27 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): ) +def test_realize_config_double_tag(tmp_path): + config = """ + a: 2 + b: 7 + foo: !int "{{ a * b }}" + bar: !int "{{ foo }}" + """ + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + expected = """ + a: 2 + b: 7 + foo: 14 + bar: 14 + """ + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. From 6fbfd3a36c8ab92364a18aaa87b8ba5d21bbf59e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Tue, 26 Nov 2024 23:36:43 +0000 Subject: [PATCH 02/31] Fix whitespace --- src/uwtools/tests/config/test_tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 7ff949e94..4434b960e 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -255,7 +255,8 @@ def test_realize_config_double_tag(tmp_path): """ with open(path_out, "r", encoding="utf-8") as f: assert f.read().strip() == dedent(expected).strip() - + + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. From 4a71bf31524c87501f0ba4c33bfdbafa762017f3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 01:15:11 +0000 Subject: [PATCH 03/31] WIP --- src/uwtools/config/jinja2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 8115e248b..a613604be 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -130,7 +130,9 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", " > ".join([*keys, k])) else: - new[dereference(k, context)] = dereference(v, context, local=val, keys=[*keys, k]) + k_deref = dereference(k, context) + v_deref = dereference(v, context, local=val, keys=[*keys, k]) + new[k_deref] = v_deref return new if isinstance(val, list): return [dereference(v, context) for v in val] From 04336dfc2625a00a249f8015da7f12b690c5bdbf Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 01:41:37 +0000 Subject: [PATCH 04/31] More keypath dot notation --- src/uwtools/config/jinja2.py | 2 +- src/uwtools/tests/config/test_jinja2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index a613604be..3a9399106 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -128,7 +128,7 @@ def dereference( new = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): - _deref_debug("Removing value at", " > ".join([*keys, k])) + _deref_debug("Removing value at", ".".join([*keys, k])) else: k_deref = dereference(k, context) v_deref = dereference(v, context, local=val, keys=[*keys, k]) diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 407e49f9c..160828aa4 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -141,7 +141,7 @@ def test_dereference_remove(caplog): remove = UWYAMLRemove(yaml.SafeLoader(""), yaml.ScalarNode(tag="!remove", value="")) val = {"a": {"b": {"c": "cherry", "d": remove}}} assert jinja2.dereference(val=val, context={}) == {"a": {"b": {"c": "cherry"}}} - assert regex_logged(caplog, "Removing value at: a > b > d") + assert regex_logged(caplog, "Removing value at: a.b.d") def test_dereference_str_expression_rendered(): From e154f1a0a90ddedad958b89f9c3b97625a11c486 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 03:12:10 +0000 Subject: [PATCH 05/31] WIP --- src/uwtools/config/jinja2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 3a9399106..12fa0f733 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -125,13 +125,13 @@ def dereference( rendered: _ConfigVal = val # fall-back value if isinstance(val, dict): keys = keys or [] - new = {} + new: dict = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: k_deref = dereference(k, context) - v_deref = dereference(v, context, local=val, keys=[*keys, k]) + v_deref = dereference(v, context, local={**val, **new}, keys=[*keys, k]) new[k_deref] = v_deref return new if isinstance(val, list): From 3f9ae1b385de2574922d55e425ada573e8f67dcc Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 03:24:34 +0000 Subject: [PATCH 06/31] WIP --- src/uwtools/config/jinja2.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 12fa0f733..4b59dea3f 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -130,9 +130,8 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - k_deref = dereference(k, context) - v_deref = dereference(v, context, local={**val, **new}, keys=[*keys, k]) - new[k_deref] = v_deref + kd, vd = [dereference(x, context, {**val, **new}, [*keys, k]) for x in (k, v)] + new[kd] = vd return new if isinstance(val, list): return [dereference(v, context) for v in val] From 31f66979c06f8c531b1f490f8978ed4b1eca9786 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 03:28:56 +0000 Subject: [PATCH 07/31] WIP --- src/uwtools/config/jinja2.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 4b59dea3f..d3db6f233 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -122,17 +122,17 @@ def dereference( :param keys: The dict keys leading to this value. :return: The input value, with Jinja2 syntax rendered. """ - rendered: _ConfigVal = val # fall-back value + rendered: _ConfigVal if isinstance(val, dict): keys = keys or [] - new: dict = {} + rendered = {} for k, v in val.items(): if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - kd, vd = [dereference(x, context, {**val, **new}, [*keys, k]) for x in (k, v)] - new[kd] = vd - return new + kd, vd = [dereference(x, context, {**val, **rendered}, [*keys, k]) for x in (k, v)] + rendered[kd] = vd + return rendered if isinstance(val, list): return [dereference(v, context) for v in val] if isinstance(val, str): @@ -144,6 +144,7 @@ def dereference( rendered = _deref_convert(val) else: _deref_debug("Accepting", val) + rendered = val return rendered From 8b77713fb81dcf32f8fb17b4551cd036e71aae02 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 03:30:58 +0000 Subject: [PATCH 08/31] WIP --- src/uwtools/config/jinja2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index d3db6f233..4a75cde17 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -132,10 +132,9 @@ def dereference( else: kd, vd = [dereference(x, context, {**val, **rendered}, [*keys, k]) for x in (k, v)] rendered[kd] = vd - return rendered - if isinstance(val, list): - return [dereference(v, context) for v in val] - if isinstance(val, str): + elif isinstance(val, list): + rendered = [dereference(v, context) for v in val] + elif isinstance(val, str): _deref_debug("Rendering", val) rendered = _deref_render(val, context, local) elif isinstance(val, UWYAMLConvert): From 105ce1e662b4b0cdfadeda942ac6ee27aab72e92 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 03:56:32 +0000 Subject: [PATCH 09/31] Move walk_key_path() to uwtools.config.support --- src/uwtools/config/jinja2.py | 2 +- src/uwtools/config/support.py | 23 ++++++++++++++++++++++ src/uwtools/config/tools.py | 25 +----------------------- src/uwtools/drivers/driver.py | 2 +- src/uwtools/tests/config/test_support.py | 19 +++++++++++++++++- src/uwtools/tests/config/test_tools.py | 17 ---------------- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 4a75cde17..9b8b83c80 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -11,7 +11,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, walk_key_path from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index f6337853c..c6fad9743 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -68,6 +68,29 @@ def log_and_error(msg: str) -> Exception: return UWConfigError(msg) +def walk_key_path(config: dict, key_path: list[str]) -> tuple[dict, str]: + """ + Navigate to the sub-config at the end of the path of given keys. + + :param config: A config. + :param key_path: Path of keys to subsection of config file. + :return: The sub-config and a string representation of the key path. + """ + keys = [] + pathstr = "" + for key in key_path: + keys.append(key) + pathstr = ".".join(keys) + try: + subconfig = config[key] + except KeyError as e: + raise log_and_error(f"Bad config path: {pathstr}") from e + if not isinstance(subconfig, dict): + raise log_and_error(f"Value at {pathstr} must be a dictionary") + config = subconfig + return config, pathstr + + def yaml_to_str(cfg: dict) -> str: """ Return a uwtools-conventional YAML representation of the given dict. diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index 305e4808a..d4ed8d1e1 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -7,7 +7,7 @@ from uwtools.config.formats.base import Config from uwtools.config.jinja2 import unrendered -from uwtools.config.support import depth, format_to_config, log_and_error +from uwtools.config.support import depth, format_to_config, log_and_error, walk_key_path from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError from uwtools.logging import log from uwtools.strings import FORMAT @@ -109,29 +109,6 @@ def realize_config( return input_obj.data -def walk_key_path(config: dict, key_path: list[str]) -> tuple[dict, str]: - """ - Navigate to the sub-config at the end of the path of given keys. - - :param config: A config. - :param key_path: Path of keys to subsection of config file. - :return: The sub-config and a string representation of the key path. - """ - keys = [] - pathstr = "" - for key in key_path: - keys.append(key) - pathstr = ".".join(keys) - try: - subconfig = config[key] - except KeyError as e: - raise log_and_error(f"Bad config path: {pathstr}") from e - if not isinstance(subconfig, dict): - raise log_and_error(f"Value at {pathstr} must be a dictionary") - config = subconfig - return config, pathstr - - # Private functions diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 878e44e8b..0f844e26d 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -18,7 +18,7 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig -from uwtools.config.tools import walk_key_path +from uwtools.config.support import walk_key_path from uwtools.config.validator import ( bundle, internal_schema_file, diff --git a/src/uwtools/tests/config/test_support.py b/src/uwtools/tests/config/test_support.py index c04b27f41..d43d2c776 100644 --- a/src/uwtools/tests/config/test_support.py +++ b/src/uwtools/tests/config/test_support.py @@ -16,7 +16,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.config.formats.sh import SHConfig from uwtools.config.formats.yaml import YAMLConfig -from uwtools.exceptions import UWConfigError +from uwtools.exceptions import UWConfigError, UWError from uwtools.logging import log from uwtools.tests.support import logged from uwtools.utils.file import FORMAT @@ -61,6 +61,23 @@ def test_log_and_error(caplog): assert logged(caplog, msg) +def test_walk_key_path_fail_bad_key_path(): + with raises(UWError) as e: + support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "x"]) + assert str(e.value) == "Bad config path: a.x" + + +def test_walk_key_path_fail_bad_leaf_value(): + with raises(UWError) as e: + support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b", "c"]) + assert str(e.value) == "Value at a.b.c must be a dictionary" + + +def test_walk_key_path_pass(): + expected = ({"c": "cherry"}, "a.b") + assert support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected + + def test_yaml_to_str(capsys): xs = " ".join("x" * 999) expected = f"xs: {xs}" diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 0adf76500..cf589fd6f 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -575,23 +575,6 @@ def test_realize_config_values_needed_yaml(caplog): assert actual.strip() == dedent(expected).strip() -def test_walk_key_path_fail_bad_key_path(): - with raises(UWError) as e: - tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "x"]) - assert str(e.value) == "Bad config path: a.x" - - -def test_walk_key_path_fail_bad_leaf_value(): - with raises(UWError) as e: - tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b", "c"]) - assert str(e.value) == "Value at a.b.c must be a dictionary" - - -def test_walk_key_path_pass(): - expected = ({"c": "cherry"}, "a.b") - assert tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected - - def test__ensure_format_bad_no_path_no_format(): with raises(UWError) as e: tools._ensure_format(desc="foo") From 40c1655391b805fe3bbc91b618bb264e29a3284f Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 04:01:50 +0000 Subject: [PATCH 10/31] Tests pass, but trivially --- src/uwtools/config/jinja2.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 9b8b83c80..545121923 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -122,6 +122,7 @@ def dereference( :param keys: The dict keys leading to this value. :return: The input value, with Jinja2 syntax rendered. """ + assert walk_key_path is not None # PM RM rendered: _ConfigVal if isinstance(val, dict): keys = keys or [] @@ -130,7 +131,15 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - kd, vd = [dereference(x, context, {**val, **rendered}, [*keys, k]) for x in (k, v)] + kd, vd = [ + dereference( + val=x, + context={**context, **rendered} if context == val else context, + local={**val, **rendered}, + keys=[*keys, k], + ) + for x in (k, v) + ] rendered[kd] = vd elif isinstance(val, list): rendered = [dereference(v, context) for v in val] From 509406d3214aa6c7e5408ff5075490c4cea28fc3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 04:28:32 +0000 Subject: [PATCH 11/31] WIP --- src/uwtools/config/jinja2.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 545121923..4c1320e68 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -11,7 +11,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, walk_key_path +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -122,7 +122,6 @@ def dereference( :param keys: The dict keys leading to this value. :return: The input value, with Jinja2 syntax rendered. """ - assert walk_key_path is not None # PM RM rendered: _ConfigVal if isinstance(val, dict): keys = keys or [] From e7f9212063ec0733382e4af292a24c91e45eb357 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 04:29:28 +0000 Subject: [PATCH 12/31] WIP --- src/uwtools/tests/config/test_tools.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index cf589fd6f..90cf357db 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -237,9 +237,9 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): def test_realize_config_double_tag(tmp_path): config = """ - a: 2 - b: 7 - foo: !int "{{ a * b }}" + a: 1 + b: 2 + foo: !int "{{ a + b }}" bar: !int "{{ foo }}" """ path_in = tmp_path / "in.yaml" @@ -248,10 +248,10 @@ def test_realize_config_double_tag(tmp_path): print(dedent(config).strip(), file=f) tools.realize_config(input_config=path_in, output_file=path_out) expected = """ - a: 2 - b: 7 - foo: 14 - bar: 14 + a: 1 + b: 2 + foo: 3 + bar: 3 """ with open(path_out, "r", encoding="utf-8") as f: assert f.read().strip() == dedent(expected).strip() From 10f4743797a646fac28ed618363f8fe6f7b08694 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 04:33:49 +0000 Subject: [PATCH 13/31] Add test --- src/uwtools/tests/config/test_tools.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 90cf357db..cc913cdac 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -235,7 +235,7 @@ def test_realize_config_depth_mismatch_to_sh(realize_config_yaml_input): ) -def test_realize_config_double_tag(tmp_path): +def test_realize_config_double_tag_flat(tmp_path): config = """ a: 1 b: 2 @@ -257,6 +257,30 @@ def test_realize_config_double_tag(tmp_path): assert f.read().strip() == dedent(expected).strip() +def test_realize_config_double_tag_nest(tmp_path): + config = """ + a: 1 + b: 2 + qux: + foo: !int "{{ a + b }}" + bar: !int "{{ foo }}" + """ + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + expected = """ + a: 1 + b: 2 + qux: + foo: 3 + bar: 3 + """ + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. From 246e0503ff639b97015a8bd427f7613f3da5e26e Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:10:55 +0000 Subject: [PATCH 14/31] Raise exception to defer render --- src/uwtools/config/jinja2.py | 4 ++++ src/uwtools/tests/config/test_tools.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 4c1320e68..ccb0a3634 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -8,10 +8,12 @@ from pathlib import Path from typing import Optional, Union +import yaml from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config +from uwtools.exceptions import UWConfigRealizeError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -275,6 +277,8 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: context = {**(local or {}), **context} try: rendered = _register_filters(env).from_string(val).render(context) + if isinstance(yaml.safe_load(rendered), UWYAMLConvert): + raise UWConfigRealizeError(f"Rendering deferred: {rendered}") _deref_debug("Rendered", rendered) except Exception as e: # pylint: disable=broad-exception-caught rendered = val diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index cc913cdac..24c9b7901 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -281,6 +281,30 @@ def test_realize_config_double_tag_nest(tmp_path): assert f.read().strip() == dedent(expected).strip() +def test_realize_config_double_tag_nest_reverse(tmp_path): + config = """ + a: 1 + b: 2 + bar: !int "{{ qux.foo }}" + qux: + foo: !int "{{ a + b }}" + """ + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + expected = """ + a: 1 + b: 2 + bar: 3 + qux: + foo: 3 + """ + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + + def test_realize_config_dry_run(caplog): """ Test that providing a YAML base file with a dry-run flag will print an YAML config file. From 51f0bf01e5cdcf523e06367dcd6ec215adcc6b55 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:15:39 +0000 Subject: [PATCH 15/31] DRY out tests --- src/uwtools/config/jinja2.py | 2 +- src/uwtools/tests/config/test_tools.py | 36 ++++++++++---------------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index ccb0a3634..bd04d221a 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -278,7 +278,7 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: try: rendered = _register_filters(env).from_string(val).render(context) if isinstance(yaml.safe_load(rendered), UWYAMLConvert): - raise UWConfigRealizeError(f"Rendering deferred: {rendered}") + raise UWConfigRealizeError(f"Rendering delayed: {rendered}") _deref_debug("Rendered", rendered) except Exception as e: # pylint: disable=broad-exception-caught rendered = val diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 24c9b7901..0ca4a45fa 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -59,6 +59,16 @@ def realize_config_yaml_input(tmp_path): # Helpers +def help_realize_config_double_tag(config, expected, tmp_path): + path_in = tmp_path / "in.yaml" + path_out = tmp_path / "out.yaml" + with open(path_in, "w", encoding="utf-8") as f: + print(dedent(config).strip(), file=f) + tools.realize_config(input_config=path_in, output_file=path_out) + with open(path_out, "r", encoding="utf-8") as f: + assert f.read().strip() == dedent(expected).strip() + + def help_realize_config_fmt2fmt(input_file, input_format, update_file, update_format, tmpdir): input_file = fixture_path(input_file) update_file = fixture_path(update_file) @@ -242,19 +252,13 @@ def test_realize_config_double_tag_flat(tmp_path): foo: !int "{{ a + b }}" bar: !int "{{ foo }}" """ - path_in = tmp_path / "in.yaml" - path_out = tmp_path / "out.yaml" - with open(path_in, "w", encoding="utf-8") as f: - print(dedent(config).strip(), file=f) - tools.realize_config(input_config=path_in, output_file=path_out) expected = """ a: 1 b: 2 foo: 3 bar: 3 """ - with open(path_out, "r", encoding="utf-8") as f: - assert f.read().strip() == dedent(expected).strip() + help_realize_config_double_tag(config, expected, tmp_path) def test_realize_config_double_tag_nest(tmp_path): @@ -265,11 +269,6 @@ def test_realize_config_double_tag_nest(tmp_path): foo: !int "{{ a + b }}" bar: !int "{{ foo }}" """ - path_in = tmp_path / "in.yaml" - path_out = tmp_path / "out.yaml" - with open(path_in, "w", encoding="utf-8") as f: - print(dedent(config).strip(), file=f) - tools.realize_config(input_config=path_in, output_file=path_out) expected = """ a: 1 b: 2 @@ -277,11 +276,10 @@ def test_realize_config_double_tag_nest(tmp_path): foo: 3 bar: 3 """ - with open(path_out, "r", encoding="utf-8") as f: - assert f.read().strip() == dedent(expected).strip() + help_realize_config_double_tag(config, expected, tmp_path) -def test_realize_config_double_tag_nest_reverse(tmp_path): +def test_realize_config_double_tag_nest_forwrad_reference(tmp_path): config = """ a: 1 b: 2 @@ -289,11 +287,6 @@ def test_realize_config_double_tag_nest_reverse(tmp_path): qux: foo: !int "{{ a + b }}" """ - path_in = tmp_path / "in.yaml" - path_out = tmp_path / "out.yaml" - with open(path_in, "w", encoding="utf-8") as f: - print(dedent(config).strip(), file=f) - tools.realize_config(input_config=path_in, output_file=path_out) expected = """ a: 1 b: 2 @@ -301,8 +294,7 @@ def test_realize_config_double_tag_nest_reverse(tmp_path): qux: foo: 3 """ - with open(path_out, "r", encoding="utf-8") as f: - assert f.read().strip() == dedent(expected).strip() + help_realize_config_double_tag(config, expected, tmp_path) def test_realize_config_dry_run(caplog): From 5123c0943678db0eedffeb54b36c73ec1423d509 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:17:46 +0000 Subject: [PATCH 16/31] Simplify --- src/uwtools/config/jinja2.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index bd04d221a..389f39b3e 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -132,15 +132,7 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - kd, vd = [ - dereference( - val=x, - context={**context, **rendered} if context == val else context, - local={**val, **rendered}, - keys=[*keys, k], - ) - for x in (k, v) - ] + kd, vd = [dereference(x, context, {**val, **rendered}, [*keys, k]) for x in (k, v)] rendered[kd] = vd elif isinstance(val, list): rendered = [dereference(v, context) for v in val] From cff7910ab2741fd7446cbeda25b599cb42849196 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:19:52 +0000 Subject: [PATCH 17/31] Formatting --- src/uwtools/config/jinja2.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 389f39b3e..0f4b03c1d 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -132,7 +132,15 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - kd, vd = [dereference(x, context, {**val, **rendered}, [*keys, k]) for x in (k, v)] + kd, vd = [ + dereference( + val=x, + context=context, + local={**val, **rendered}, + keys=[*keys, k], + ) + for x in (k, v) + ] rendered[kd] = vd elif isinstance(val, list): rendered = [dereference(v, context) for v in val] From 9c5eefef2a4e31e88bcc95dfe9dd794df68c1eb1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:21:29 +0000 Subject: [PATCH 18/31] Add setuptools to build requirements --- recipe/meta.json | 3 ++- recipe/meta.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/recipe/meta.json b/recipe/meta.json index 8913b93a0..80b698fac 100644 --- a/recipe/meta.json +++ b/recipe/meta.json @@ -21,7 +21,8 @@ "pytest-cov =5.0.*", "pytest-xdist =3.6.*", "python >=3.9,<3.13", - "pyyaml =6.0.*" + "pyyaml =6.0.*", + "setuptools" ], "run": [ "f90nml =1.4.*", diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 479b49330..9a9dfd0d9 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -12,6 +12,7 @@ build: requirements: build: - pip + - setuptools run: - f90nml 1.4.* - iotaa 0.8.* From 2a95ec9fc04073f2d330048cea37be49e36a836b Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:23:54 +0000 Subject: [PATCH 19/31] Put walk_key_path() back --- src/uwtools/config/support.py | 23 ---------------------- src/uwtools/config/tools.py | 25 +++++++++++++++++++++++- src/uwtools/drivers/driver.py | 2 +- src/uwtools/tests/config/test_support.py | 19 +----------------- src/uwtools/tests/config/test_tools.py | 17 ++++++++++++++++ 5 files changed, 43 insertions(+), 43 deletions(-) diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index c6fad9743..f6337853c 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -68,29 +68,6 @@ def log_and_error(msg: str) -> Exception: return UWConfigError(msg) -def walk_key_path(config: dict, key_path: list[str]) -> tuple[dict, str]: - """ - Navigate to the sub-config at the end of the path of given keys. - - :param config: A config. - :param key_path: Path of keys to subsection of config file. - :return: The sub-config and a string representation of the key path. - """ - keys = [] - pathstr = "" - for key in key_path: - keys.append(key) - pathstr = ".".join(keys) - try: - subconfig = config[key] - except KeyError as e: - raise log_and_error(f"Bad config path: {pathstr}") from e - if not isinstance(subconfig, dict): - raise log_and_error(f"Value at {pathstr} must be a dictionary") - config = subconfig - return config, pathstr - - def yaml_to_str(cfg: dict) -> str: """ Return a uwtools-conventional YAML representation of the given dict. diff --git a/src/uwtools/config/tools.py b/src/uwtools/config/tools.py index d4ed8d1e1..305e4808a 100644 --- a/src/uwtools/config/tools.py +++ b/src/uwtools/config/tools.py @@ -7,7 +7,7 @@ from uwtools.config.formats.base import Config from uwtools.config.jinja2 import unrendered -from uwtools.config.support import depth, format_to_config, log_and_error, walk_key_path +from uwtools.config.support import depth, format_to_config, log_and_error from uwtools.exceptions import UWConfigError, UWConfigRealizeError, UWError from uwtools.logging import log from uwtools.strings import FORMAT @@ -109,6 +109,29 @@ def realize_config( return input_obj.data +def walk_key_path(config: dict, key_path: list[str]) -> tuple[dict, str]: + """ + Navigate to the sub-config at the end of the path of given keys. + + :param config: A config. + :param key_path: Path of keys to subsection of config file. + :return: The sub-config and a string representation of the key path. + """ + keys = [] + pathstr = "" + for key in key_path: + keys.append(key) + pathstr = ".".join(keys) + try: + subconfig = config[key] + except KeyError as e: + raise log_and_error(f"Bad config path: {pathstr}") from e + if not isinstance(subconfig, dict): + raise log_and_error(f"Value at {pathstr} must be a dictionary") + config = subconfig + return config, pathstr + + # Private functions diff --git a/src/uwtools/drivers/driver.py b/src/uwtools/drivers/driver.py index 0f844e26d..878e44e8b 100644 --- a/src/uwtools/drivers/driver.py +++ b/src/uwtools/drivers/driver.py @@ -18,7 +18,7 @@ from uwtools.config.formats.base import Config from uwtools.config.formats.yaml import YAMLConfig -from uwtools.config.support import walk_key_path +from uwtools.config.tools import walk_key_path from uwtools.config.validator import ( bundle, internal_schema_file, diff --git a/src/uwtools/tests/config/test_support.py b/src/uwtools/tests/config/test_support.py index d43d2c776..c04b27f41 100644 --- a/src/uwtools/tests/config/test_support.py +++ b/src/uwtools/tests/config/test_support.py @@ -16,7 +16,7 @@ from uwtools.config.formats.nml import NMLConfig from uwtools.config.formats.sh import SHConfig from uwtools.config.formats.yaml import YAMLConfig -from uwtools.exceptions import UWConfigError, UWError +from uwtools.exceptions import UWConfigError from uwtools.logging import log from uwtools.tests.support import logged from uwtools.utils.file import FORMAT @@ -61,23 +61,6 @@ def test_log_and_error(caplog): assert logged(caplog, msg) -def test_walk_key_path_fail_bad_key_path(): - with raises(UWError) as e: - support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "x"]) - assert str(e.value) == "Bad config path: a.x" - - -def test_walk_key_path_fail_bad_leaf_value(): - with raises(UWError) as e: - support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b", "c"]) - assert str(e.value) == "Value at a.b.c must be a dictionary" - - -def test_walk_key_path_pass(): - expected = ({"c": "cherry"}, "a.b") - assert support.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected - - def test_yaml_to_str(capsys): xs = " ".join("x" * 999) expected = f"xs: {xs}" diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index 0ca4a45fa..ee666e9bb 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -615,6 +615,23 @@ def test_realize_config_values_needed_yaml(caplog): assert actual.strip() == dedent(expected).strip() +def test_walk_key_path_fail_bad_key_path(): + with raises(UWError) as e: + tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "x"]) + assert str(e.value) == "Bad config path: a.x" + + +def test_walk_key_path_fail_bad_leaf_value(): + with raises(UWError) as e: + tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b", "c"]) + assert str(e.value) == "Value at a.b.c must be a dictionary" + + +def test_walk_key_path_pass(): + expected = ({"c": "cherry"}, "a.b") + assert tools.walk_key_path({"a": {"b": {"c": "cherry"}}}, ["a", "b"]) == expected + + def test__ensure_format_bad_no_path_no_format(): with raises(UWError) as e: tools._ensure_format(desc="foo") From c1528fd777f3758f02e59a9e3e127875df758638 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:27:43 +0000 Subject: [PATCH 20/31] Trivial doc changes --- .../cli/tools/config/realize-verbose.out | 56 +++++++++---------- .../cli/tools/config/validate-verbose.out | 42 +++++++------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/sections/user_guide/cli/tools/config/realize-verbose.out b/docs/sections/user_guide/cli/tools/config/realize-verbose.out index 3371e2070..cac8e0711 100644 --- a/docs/sections/user_guide/cli/tools/config/realize-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/realize-verbose.out @@ -1,30 +1,30 @@ -[2024-05-23T19:39:16] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose -[2024-05-23T19:39:16] DEBUG Reading input from stdin -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: '{{ recipient }}' -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: {{ recipient }} -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, current value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: hello -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: world -[2024-05-23T19:39:16] DEBUG [dereference] Rendering: recipient -[2024-05-23T19:39:16] DEBUG [dereference] Rendered: recipient -[2024-05-23T19:39:16] DEBUG Dereferencing, final value: -[2024-05-23T19:39:16] DEBUG hello: world -[2024-05-23T19:39:16] DEBUG recipient: world -[2024-05-23T19:39:16] DEBUG Writing output to stdout +[2024-11-27T05:24:34] DEBUG Command: uw config realize --input-format yaml --output-format yaml --verbose +[2024-11-27T05:24:34] DEBUG Reading input from stdin +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: '{{ recipient }}' +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: {{ recipient }} +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: world +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: world +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG hello: world +[2024-11-27T05:24:34] DEBUG recipient: world +[2024-11-27T05:24:34] DEBUG Writing output to stdout hello: world recipient: world diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index 17dc2a651..d48c7ea14 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2024-08-26T22:54:28] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2024-08-26T22:54:28] DEBUG Using schema file: schema.jsonschema -[2024-08-26T22:54:28] DEBUG Dereferencing, current value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: Hello -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: greeting -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: World -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: recipient -[2024-08-26T22:54:28] DEBUG [dereference] Rendering: values -[2024-08-26T22:54:28] DEBUG [dereference] Rendered: values -[2024-08-26T22:54:28] DEBUG Dereferencing, final value: -[2024-08-26T22:54:28] DEBUG values: -[2024-08-26T22:54:28] DEBUG greeting: Hello -[2024-08-26T22:54:28] DEBUG recipient: World -[2024-08-26T22:54:29] INFO 0 UW schema-validation errors found in config +[2024-11-27T05:24:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2024-11-27T05:24:34] DEBUG Using schema file: schema.jsonschema +[2024-11-27T05:24:34] DEBUG Dereferencing, current value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: values +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: greeting +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: Hello +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: recipient +[2024-11-27T05:24:34] DEBUG [dereference] Rendering: World +[2024-11-27T05:24:34] DEBUG [dereference] Rendered: World +[2024-11-27T05:24:34] DEBUG Dereferencing, final value: +[2024-11-27T05:24:34] DEBUG values: +[2024-11-27T05:24:34] DEBUG greeting: Hello +[2024-11-27T05:24:34] DEBUG recipient: World +[2024-11-27T05:24:34] INFO 0 UW schema-validation errors found in config From 976d78f895e8cb527058047b85bdee65e6680f69 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:29:22 +0000 Subject: [PATCH 21/31] Run test workflow on all branches --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3351ba97d..c34875b49 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -2,10 +2,10 @@ name: Test on: pull_request: branches: - - main + - '**' push: branches: - - main + - '**' workflow_dispatch: branches: - '**' From 5dacba736a346fb920ae229b15996059d74213b1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:41:56 +0000 Subject: [PATCH 22/31] Formatting --- src/uwtools/config/jinja2.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 0f4b03c1d..fadb9d3c4 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -133,12 +133,7 @@ def dereference( _deref_debug("Removing value at", ".".join([*keys, k])) else: kd, vd = [ - dereference( - val=x, - context=context, - local={**val, **rendered}, - keys=[*keys, k], - ) + dereference(val=x, context=context, local={**val, **rendered}, keys=[*keys, k]) for x in (k, v) ] rendered[kd] = vd From 804e82b552594437001465fed7120cef0b02e2d1 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:42:48 +0000 Subject: [PATCH 23/31] Revert "Formatting" This reverts commit 5dacba736a346fb920ae229b15996059d74213b1. --- src/uwtools/config/jinja2.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index fadb9d3c4..0f4b03c1d 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -133,7 +133,12 @@ def dereference( _deref_debug("Removing value at", ".".join([*keys, k])) else: kd, vd = [ - dereference(val=x, context=context, local={**val, **rendered}, keys=[*keys, k]) + dereference( + val=x, + context=context, + local={**val, **rendered}, + keys=[*keys, k], + ) for x in (k, v) ] rendered[kd] = vd From 15a5905330b2359498b4e9da827b00eea961c429 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 05:46:21 +0000 Subject: [PATCH 24/31] Simplify --- src/uwtools/config/jinja2.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 0f4b03c1d..e383e8ca0 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -132,15 +132,7 @@ def dereference( if isinstance(v, UWYAMLRemove): _deref_debug("Removing value at", ".".join([*keys, k])) else: - kd, vd = [ - dereference( - val=x, - context=context, - local={**val, **rendered}, - keys=[*keys, k], - ) - for x in (k, v) - ] + kd, vd = [dereference(x, context, val, [*keys, k]) for x in (k, v)] rendered[kd] = vd elif isinstance(val, list): rendered = [dereference(v, context) for v in val] From c7abe77d450ec06af5bf7263816231bad4292182 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 06:05:37 +0000 Subject: [PATCH 25/31] Vary tags --- src/uwtools/tests/config/test_tools.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/uwtools/tests/config/test_tools.py b/src/uwtools/tests/config/test_tools.py index ee666e9bb..793af23db 100644 --- a/src/uwtools/tests/config/test_tools.py +++ b/src/uwtools/tests/config/test_tools.py @@ -263,36 +263,36 @@ def test_realize_config_double_tag_flat(tmp_path): def test_realize_config_double_tag_nest(tmp_path): config = """ - a: 1 - b: 2 + a: 1.0 + b: 2.0 qux: - foo: !int "{{ a + b }}" - bar: !int "{{ foo }}" + foo: !float "{{ a + b }}" + bar: !float "{{ foo }}" """ expected = """ - a: 1 - b: 2 + a: 1.0 + b: 2.0 qux: - foo: 3 - bar: 3 + foo: 3.0 + bar: 3.0 """ help_realize_config_double_tag(config, expected, tmp_path) def test_realize_config_double_tag_nest_forwrad_reference(tmp_path): config = """ - a: 1 - b: 2 - bar: !int "{{ qux.foo }}" + a: true + b: false + bar: !bool "{{ qux.foo }}" qux: - foo: !int "{{ a + b }}" + foo: !bool "{{ a or b }}" """ expected = """ - a: 1 - b: 2 - bar: 3 + a: true + b: false + bar: true qux: - foo: 3 + foo: true """ help_realize_config_double_tag(config, expected, tmp_path) From 5a5314845a221dfb714407618d0243fbb73580e3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 06:28:57 +0000 Subject: [PATCH 26/31] WIP --- src/uwtools/config/formats/yaml.py | 21 ++++++++++++++------- src/uwtools/config/jinja2.py | 3 ++- src/uwtools/tests/config/test_jinja2.py | 8 ++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 99c5b32a6..059d194db 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -94,10 +94,9 @@ def _load(self, config_file: Optional[Path]) -> dict: :param config_file: Path to config file to load. """ - loader = self._yaml_loader with readable(config_file) as f: try: - config = yaml.load(f.read(), Loader=loader) + config = yaml.load(f.read(), Loader=self._yaml_loader) if isinstance(config, dict): return config t = type(config).__name__ @@ -157,13 +156,10 @@ def _yaml_include(self, loader: yaml.Loader, node: yaml.SequenceNode) -> dict: @property def _yaml_loader(self) -> type[yaml.SafeLoader]: """ - The loader, with appropriate constructors added. + A loader with all UW constructors added. """ - loader = yaml.SafeLoader + loader = self.loader() loader.add_constructor(INCLUDE_TAG, self._yaml_include) - for tag_class in (UWYAMLConvert, UWYAMLRemove): - for tag in getattr(tag_class, "TAGS"): - loader.add_constructor(tag, tag_class) return loader # Public methods @@ -193,6 +189,17 @@ def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: with writable(path) as f: print(cls._dict_to_str(cfg), file=f) + @staticmethod + def loader() -> type[yaml.SafeLoader]: + """ + A loader with basic UW constructors added. + """ + loader = yaml.SafeLoader + for tag_class in (UWYAMLConvert, UWYAMLRemove): + for tag in getattr(tag_class, "TAGS"): + loader.add_constructor(tag, tag_class) + return loader + def _write_plain_open_ended(self, *args, **kwargs) -> None: """ diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index e383e8ca0..72726560a 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -270,7 +270,8 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: try: rendered = _register_filters(env).from_string(val).render(context) if isinstance(yaml.safe_load(rendered), UWYAMLConvert): - raise UWConfigRealizeError(f"Rendering delayed: {rendered}") + _deref_debug("Held", rendered) + raise UWConfigRealizeError() _deref_debug("Rendered", rendered) except Exception as e: # pylint: disable=broad-exception-caught rendered = val diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 160828aa4..889182d9c 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -16,6 +16,7 @@ from pytest import fixture, mark, raises from uwtools.config import jinja2 +from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.jinja2 import J2Template from uwtools.config.support import UWYAMLConvert, UWYAMLRemove from uwtools.logging import log @@ -315,6 +316,13 @@ def test__deref_debug(caplog): assert logged(caplog, "[dereference] Frobnicated: foo") +def test__deref_render_held(caplog): + val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=YAMLConfig.loader()) + assert jinja2._deref_render(val=val, context=context) == val + assert not regex_logged(caplog, "Rendered") + assert regex_logged(caplog, "Held") + + def test__deref_render_no(caplog, deref_render_assets): val, context, _ = deref_render_assets assert jinja2._deref_render(val=val, context=context) == val From e00b3660a26df302c85639aca249afc2f77e29a3 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 06:33:31 +0000 Subject: [PATCH 27/31] WIP --- src/uwtools/config/formats/yaml.py | 15 ++------------- src/uwtools/config/support.py | 11 +++++++++++ src/uwtools/tests/config/test_jinja2.py | 5 ++--- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/uwtools/config/formats/yaml.py b/src/uwtools/config/formats/yaml.py index 059d194db..71bc870f8 100644 --- a/src/uwtools/config/formats/yaml.py +++ b/src/uwtools/config/formats/yaml.py @@ -10,9 +10,9 @@ from uwtools.config.support import ( INCLUDE_TAG, UWYAMLConvert, - UWYAMLRemove, from_od, log_and_error, + uw_yaml_loader, yaml_to_str, ) from uwtools.exceptions import UWConfigError @@ -158,7 +158,7 @@ def _yaml_loader(self) -> type[yaml.SafeLoader]: """ A loader with all UW constructors added. """ - loader = self.loader() + loader = uw_yaml_loader() loader.add_constructor(INCLUDE_TAG, self._yaml_include) return loader @@ -189,17 +189,6 @@ def dump_dict(cls, cfg: dict, path: Optional[Path] = None) -> None: with writable(path) as f: print(cls._dict_to_str(cfg), file=f) - @staticmethod - def loader() -> type[yaml.SafeLoader]: - """ - A loader with basic UW constructors added. - """ - loader = yaml.SafeLoader - for tag_class in (UWYAMLConvert, UWYAMLRemove): - for tag in getattr(tag_class, "TAGS"): - loader.add_constructor(tag, tag_class) - return loader - def _write_plain_open_ended(self, *args, **kwargs) -> None: """ diff --git a/src/uwtools/config/support.py b/src/uwtools/config/support.py index f6337853c..393ed1a44 100644 --- a/src/uwtools/config/support.py +++ b/src/uwtools/config/support.py @@ -68,6 +68,17 @@ def log_and_error(msg: str) -> Exception: return UWConfigError(msg) +def uw_yaml_loader() -> type[yaml.SafeLoader]: + """ + A loader with basic UW constructors added. + """ + loader = yaml.SafeLoader + for tag_class in (UWYAMLConvert, UWYAMLRemove): + for tag in getattr(tag_class, "TAGS"): + loader.add_constructor(tag, tag_class) + return loader + + def yaml_to_str(cfg: dict) -> str: """ Return a uwtools-conventional YAML representation of the given dict. diff --git a/src/uwtools/tests/config/test_jinja2.py b/src/uwtools/tests/config/test_jinja2.py index 889182d9c..2985addc1 100644 --- a/src/uwtools/tests/config/test_jinja2.py +++ b/src/uwtools/tests/config/test_jinja2.py @@ -16,9 +16,8 @@ from pytest import fixture, mark, raises from uwtools.config import jinja2 -from uwtools.config.formats.yaml import YAMLConfig from uwtools.config.jinja2 import J2Template -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, uw_yaml_loader from uwtools.logging import log from uwtools.tests.support import logged, regex_logged @@ -317,7 +316,7 @@ def test__deref_debug(caplog): def test__deref_render_held(caplog): - val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=YAMLConfig.loader()) + val, context = "!int '{{ a }}'", yaml.load("a: !int '{{ 42 }}'", Loader=uw_yaml_loader()) assert jinja2._deref_render(val=val, context=context) == val assert not regex_logged(caplog, "Rendered") assert regex_logged(caplog, "Held") From 834b42f7ebe71acdfbbc8e4def5b74f4efedafd0 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 06:34:38 +0000 Subject: [PATCH 28/31] WIP --- src/uwtools/config/jinja2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uwtools/config/jinja2.py b/src/uwtools/config/jinja2.py index 72726560a..cba03bae0 100644 --- a/src/uwtools/config/jinja2.py +++ b/src/uwtools/config/jinja2.py @@ -12,7 +12,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined, Undefined, meta from jinja2.exceptions import UndefinedError -from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config +from uwtools.config.support import UWYAMLConvert, UWYAMLRemove, format_to_config, uw_yaml_loader from uwtools.exceptions import UWConfigRealizeError from uwtools.logging import INDENT, MSGWIDTH, log from uwtools.utils.file import get_file_format, readable, writable @@ -269,7 +269,7 @@ def _deref_render(val: str, context: dict, local: Optional[dict] = None) -> str: context = {**(local or {}), **context} try: rendered = _register_filters(env).from_string(val).render(context) - if isinstance(yaml.safe_load(rendered), UWYAMLConvert): + if isinstance(yaml.load(rendered, Loader=uw_yaml_loader()), UWYAMLConvert): _deref_debug("Held", rendered) raise UWConfigRealizeError() _deref_debug("Rendered", rendered) From 59dd711fefe88fc9bfc69826f7fc78ef0e6e3ab4 Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 15:41:06 +0000 Subject: [PATCH 29/31] Update docs --- docs/sections/user_guide/yaml/tags.rst | 75 +++++++++++++++++--------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index 8acd99caa..c622e85f2 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -35,7 +35,7 @@ Converts the tagged node to a Python ``boolean`` object. For example, given ``in .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml flag1: True flag2: True @@ -52,7 +52,7 @@ Converts the tagged node to a Python ``datetime`` object. For example, given ``i .. code-block:: text - % uw config realize -i ../input.yaml --output-format yaml + $ uw config realize -i ../input.yaml --output-format yaml date1: 2024-09-01 date2: 2024-09-01 00:00:00 @@ -69,49 +69,76 @@ Converts the tagged node to a Python ``float`` value. For example, given ``input .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file input.yaml --output-format yaml f2: 5.859 -``!int`` -^^^^^^^^ +``!include`` +^^^^^^^^^^^^ -Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: +Load and parse the files specified in the tagged sequence value and insert their contents here. For example, given ``numbers.yaml``: .. code-block:: yaml - f1: 3 - f2: 11 - f3: !int "{{ (f1 + f2) * 10 }}" + values: !include [constants.yaml] + +and ``constants.yaml``: + +.. code-block:: yaml + + e: 2.718 + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml - f1: 3 - f2: 11 - f2: 140 + $ uw config realize --input-file numbers.yaml --output-format yaml + values: + e: 2.718 + pi: 3.141 -``!include`` -^^^^^^^^^^^^ +Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, giben ``numbers.yaml``: + +.. code-block:: yaml + + values: !include [e.yaml, pi.yaml] -Parse the tagged file and include its tags. For example, given ``input.yaml``: +``e.yaml``: .. code-block:: yaml - values: !include [./supplemental.yaml] + constants: + e: 2.718 -and ``supplemental.yaml``: +and ``pi.yaml``: .. code-block:: yaml - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 .. code-block:: text - % uw config realize --input-file input.yaml --output-format yaml + $ uw config realize --input-file numbers.yaml --output-format yaml values: - e: 2.718 - pi: 3.141 + constants: + pi: 3.141 + +``!int`` +^^^^^^^^ + +Converts the tagged node to a Python ``int`` value. For example, given ``input.yaml``: + +.. code-block:: yaml + + f1: 3 + f2: 11 + f3: !int "{{ (f1 + f2) * 10 }}" + +.. code-block:: text + + $ uw config realize --input-file input.yaml --output-format yaml + f1: 3 + f2: 11 + f2: 140 ``!remove`` ^^^^^^^^^^^ @@ -131,5 +158,5 @@ and ``update.yaml``: .. code-block:: text - % uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml + $ uw config realize --input-file input.yaml --update-file update.yaml --output-format yaml pi: 3.141 From fc4502596728bfde59ea0d0818b99403ac531bec Mon Sep 17 00:00:00 2001 From: Paul Madden Date: Wed, 27 Nov 2024 16:13:41 +0000 Subject: [PATCH 30/31] Doc fix --- docs/sections/user_guide/yaml/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index c622e85f2..f589e09a1 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -26,7 +26,7 @@ Additionally, UW defines the following tags to support use cases not covered by ``!bool`` ^^^^^^^^^ -Converts the tagged node to a Python ``boolean`` object. For example, given ``input.yaml``: +Converts the tagged node to a Python ``bool`` object. For example, given ``input.yaml``: .. code-block:: yaml From d2aa479c60f60d889f35b0b87abca15895938ba5 Mon Sep 17 00:00:00 2001 From: Paul Madden <136389411+maddenp-noaa@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:48:49 -0700 Subject: [PATCH 31/31] Update docs/sections/user_guide/yaml/tags.rst Co-authored-by: Emily Carpenter <137525341+elcarpenterNOAA@users.noreply.github.com> --- docs/sections/user_guide/yaml/tags.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sections/user_guide/yaml/tags.rst b/docs/sections/user_guide/yaml/tags.rst index f589e09a1..e768d74e5 100644 --- a/docs/sections/user_guide/yaml/tags.rst +++ b/docs/sections/user_guide/yaml/tags.rst @@ -95,7 +95,7 @@ and ``constants.yaml``: e: 2.718 pi: 3.141 -Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, giben ``numbers.yaml``: +Values from files later in the sequence overwrite their predecessors, and full-value replacement, not structural merging, is performed. For example, given ``numbers.yaml``: .. code-block:: yaml