From 839494e9f633c7670fdf3b4cd486fb91f213adec Mon Sep 17 00:00:00 2001 From: Vibhu-gslab <109593615+Vibhu-gslab@users.noreply.github.com> Date: Mon, 10 Jun 2024 19:09:19 +0530 Subject: [PATCH] Refactor(plugins): Move jinja filter code for arista.avd.convert_dicts to PyAVD (#4069) Co-authored-by: Vibhu Tripathi --- .../avd/plugins/filter/convert_dicts.py | 108 +++-------------- .../tests/unit/filters/test_convert_dicts.py | 114 ------------------ python-avd/Makefile | 1 + python-avd/pyavd/j2filters/convert_dicts.py | 97 +++++++++++++++ python-avd/pyavd/templater.py | 2 +- .../pyavd/j2filters/test_convert_dict.py | 96 +++++++++++++++ 6 files changed, 210 insertions(+), 208 deletions(-) delete mode 100644 ansible_collections/arista/avd/tests/unit/filters/test_convert_dicts.py create mode 100644 python-avd/pyavd/j2filters/convert_dicts.py create mode 100644 python-avd/tests/pyavd/j2filters/test_convert_dict.py diff --git a/ansible_collections/arista/avd/plugins/filter/convert_dicts.py b/ansible_collections/arista/avd/plugins/filter/convert_dicts.py index 5bf990da439..3aa30b17574 100644 --- a/ansible_collections/arista/avd/plugins/filter/convert_dicts.py +++ b/ansible_collections/arista/avd/plugins/filter/convert_dicts.py @@ -8,8 +8,21 @@ __metaclass__ = type +from ansible.errors import AnsibleFilterError -import os +from ansible_collections.arista.avd.plugins.plugin_utils.pyavd_wrappers import RaiseOnUse, wrap_filter + +PLUGIN_NAME = "arista.avd.convert_dicts" + +try: + from pyavd.j2filters.convert_dicts import convert_dicts +except ImportError as e: + convert_dicts = RaiseOnUse( + AnsibleFilterError( + f"The '{PLUGIN_NAME}' plugin requires the 'pyavd' Python library. Got import error", + orig_exc=e, + ) + ) DOCUMENTATION = r""" --- @@ -87,99 +100,8 @@ """ -def convert_dicts(dictionary, primary_key="name", secondary_key=None): - """ - The `arista.avd.convert_dicts` filter will convert a dictionary containing nested dictionaries to a list of - dictionaries. It inserts the outer dictionary keys into each list item using the primary_key `name` (key name is - configurable) and if there is a non-dictionary value,it inserts this value to - secondary key (key name is configurable), if secondary key is provided. - - This filter is intended for: - - - Seamless data model migration from dictionaries to lists. - - Improve Ansible's processing performance when dealing with large dictionaries by converting them to lists of dictionaries. - - Note: If there is a non-dictionary value with no secondary key provided, it will pass through untouched - - To use this filter: - - ```jinja - {# convert list of dictionary with default `name:` as the primary key and None secondary key #} - {% set example_list = example_dictionary | arista.avd.convert_dicts %} - {% for example_item in example_list %} - item primary key is {{ example_item.name }} - {% endfor %} - - {# convert list of dictionary with `id:` set as the primary key and `types:` set as the secondary key #} - {% set example_list = example_dictionary | arista.avd.convert_dicts('id','types') %} - {% for example_item in example_list %} - item primary key is {{ example_item.id }} - item secondary key is {{ example_item.types }} - {% endfor %} - ``` - - Parameters - ---------- - dictionary : any - Nested Dictionary to convert - returned untouched if not a nested dictionary and list - primary_key : str, optional - Name of primary key used when inserting outer dictionary keys into items. - secondary_key : str, optional - Name of secondary key used when inserting dictionary values which are list into items. - - Returns - ------- - any - Returns list of dictionaries or input variable untouched if not a nested dictionary/list. - """ - if not isinstance(dictionary, (dict, list)) or os.environ.get("AVD_DISABLE_CONVERT_DICTS"): - # Not a dictionary/list, return the original - return dictionary - elif isinstance(dictionary, list): - output = [] - for element in dictionary: - if not isinstance(element, dict): - output.append({primary_key: element}) - elif primary_key not in element and secondary_key is not None: - # if element of nested dictionary is a dictionary but primary key is missing, insert primary and secondary keys. - for key in element: - output.append( - { - primary_key: key, - secondary_key: element[key], - } - ) - else: - output.append(element) - return output - else: - output = [] - for key in dictionary: - if secondary_key is not None: - # Add secondary key for the values if secondary key is provided - output.append( - { - primary_key: key, - secondary_key: dictionary[key], - } - ) - else: - if not isinstance(dictionary[key], dict): - # Not a nested dictionary - output.append({primary_key: key}) - else: - # Nested dictionary - output.append( - { - primary_key: key, - **dictionary[key], - } - ) - return output - - class FilterModule(object): def filters(self): return { - "convert_dicts": convert_dicts, + "convert_dicts": wrap_filter(PLUGIN_NAME)(convert_dicts), } diff --git a/ansible_collections/arista/avd/tests/unit/filters/test_convert_dicts.py b/ansible_collections/arista/avd/tests/unit/filters/test_convert_dicts.py deleted file mode 100644 index 33916b20287..00000000000 --- a/ansible_collections/arista/avd/tests/unit/filters/test_convert_dicts.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) 2023-2024 Arista Networks, Inc. -# Use of this source code is governed by the Apache License 2.0 -# that can be found in the LICENSE file. -from __future__ import absolute_import, division, print_function - -__metaclass__ = type - -from ansible_collections.arista.avd.plugins.filter.convert_dicts import FilterModule, convert_dicts - -nested_list_of_dict = { - "TEST1": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}], - "TEST2": [{"type": "deny", "extcommunities": "65001:65001"}], -} -nested_dict = {"TEST1": {"action": "permit 1000:1000"}, "TEST2": {"action": "permit 2000:3000"}} -list = ["Test1", "Test2", "Test3"] -list_of_dict = [{"type": "permit"}, {"extcommunities": "65000:65000"}] -dict_with_string = {"dict": "test_string"} - -f = FilterModule() - - -class TestConvertDicts: - def test_convert_dicts_with_nested_dict_default(self): - resp = convert_dicts(nested_dict) - assert resp == [{"action": "permit 1000:1000", "name": "TEST1"}, {"action": "permit 2000:3000", "name": "TEST2"}] - - def test_convert_dicts_with_nested_dict_primary_key(self): - resp = convert_dicts(nested_dict, "id") - assert resp == [{"action": "permit 1000:1000", "id": "TEST1"}, {"action": "permit 2000:3000", "id": "TEST2"}] - - def test_convert_dicts_with_nested_dict_secondary_key(self): - resp = convert_dicts(nested_dict, secondary_key="types") - assert resp == [{"name": "TEST1", "types": {"action": "permit 1000:1000"}}, {"name": "TEST2", "types": {"action": "permit 2000:3000"}}] - - def test_convert_dicts_with_nested_dict_primary_and_secondary_key(self): - resp = convert_dicts(nested_dict, "id", "types") - assert resp == [{"id": "TEST1", "types": {"action": "permit 1000:1000"}}, {"id": "TEST2", "types": {"action": "permit 2000:3000"}}] - - def test_convert_dicts_with_listofdict_default(self): - resp = convert_dicts(nested_list_of_dict) - assert resp == [{"name": "TEST1"}, {"name": "TEST2"}] - - def test_convert_dicts_with_listofdict_primary_key(self): - resp = convert_dicts(nested_list_of_dict, "test") - assert resp == [{"test": "TEST1"}, {"test": "TEST2"}] - - def test_convert_dicts_with_listofdict_secondary_key(self): - resp = convert_dicts(nested_list_of_dict, secondary_key="types") - assert resp == [ - {"name": "TEST1", "types": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}]}, - {"name": "TEST2", "types": [{"type": "deny", "extcommunities": "65001:65001"}]}, - ] - - def test_convert_dicts_with_listofdict_primary_and_secondary_key(self): - resp = convert_dicts(nested_list_of_dict, "id", "types") - assert resp == [ - {"id": "TEST1", "types": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}]}, - {"id": "TEST2", "types": [{"type": "deny", "extcommunities": "65001:65001"}]}, - ] - - def test_convert_dicts_with_list_default(self): - resp = convert_dicts(list) - assert resp == [{"name": "Test1"}, {"name": "Test2"}, {"name": "Test3"}] - - def test_convert_dicts_with_list_primary_key(self): - resp = convert_dicts(list, "test") - assert resp == [{"test": "Test1"}, {"test": "Test2"}, {"test": "Test3"}] - - def test_convert_dicts_with_list_secondary_key(self): - resp = convert_dicts(list, secondary_key="id") - assert resp == [{"name": "Test1"}, {"name": "Test2"}, {"name": "Test3"}] - - def test_convert_dicts_with_list_primary_and_secondary_key(self): - resp = convert_dicts(list, "test", "types") - assert resp == [{"test": "Test1"}, {"test": "Test2"}, {"test": "Test3"}] - - def test_convert_dicts_with_string_value_default(self): - resp = convert_dicts(dict_with_string) - assert resp == [{"name": "dict"}] - - def test_convert_dicts_with_string_value_primary_key(self): - resp = convert_dicts(dict_with_string, "test") - assert resp == [{"test": "dict"}] - - def test_convert_dicts_with_string_value_secondary_key(self): - resp = convert_dicts(dict_with_string, secondary_key="str") - assert resp == [{"name": "dict", "str": "test_string"}] - - def test_convert_dicts_with_string_value_primary_key_and_secondary_key(self): - resp = convert_dicts(dict_with_string, "test", "str") - assert resp == [{"test": "dict", "str": "test_string"}] - - def test_convert_dicts_with_list_of_dict_default(self): - resp = convert_dicts(list_of_dict) - assert resp == list_of_dict - - def test_convert_dicts_with_list_of_dict_primary_key(self): - resp = convert_dicts(list_of_dict, "test") - assert resp == list_of_dict - - def test_convert_dicts_with_list_of_dict_secondary_key(self): - # We convert a list-of-dict input if primary_key is found in element and secondary_key is set - resp = convert_dicts(list_of_dict, secondary_key="id") - assert resp == [{"name": "type", "id": "permit"}, {"name": "extcommunities", "id": "65000:65000"}] - - def test_convert_dicts_with_list_of_dict_primary_key_and_secondary_key(self): - # We convert a list-of-dict input if primary_key is found in element and secondary_key is set - resp = convert_dicts(list_of_dict, "test", "id") - assert resp == [{"test": "type", "id": "permit"}, {"test": "extcommunities", "id": "65000:65000"}] - - def test_convert_dicts_filter(self): - resp = f.filters() - assert isinstance(resp, dict) - assert "convert_dicts" in resp.keys() diff --git a/python-avd/Makefile b/python-avd/Makefile index 43e17929b6f..3b453fe4bbf 100644 --- a/python-avd/Makefile +++ b/python-avd/Makefile @@ -92,6 +92,7 @@ fix-libs: ## Fix/remove various Ansible specifics things from python files # For each moved filter make sure to override the default translation. find $(PACKAGE_DIR) -name '*.py' -exec sed -i -e 's/ansible_collections\.arista\.avd\.plugins\.filter.natural_sort/$(PYAVD_FILTER_IMPORT)\.natural_sort/g' {} + + find $(PACKAGE_DIR) -name '*.py' -exec sed -i -e 's/ansible_collections\.arista\.avd\.plugins\.filter.convert_dicts/$(PYAVD_FILTER_IMPORT)\.convert_dicts/g' {} + find $(PACKAGE_DIR) -name '*.py' -exec sed -i -e 's/ansible_collections\.arista\.avd\.plugins\.filter/$(VENDOR_IMPORT)\.j2\.filter/g' {} + find $(PACKAGE_DIR) -name '*.py' -exec sed -i -e 's/ansible_collections\.arista\.avd\.roles\.eos_designs\.python_modules/$(VENDOR_IMPORT)\.eos_designs/g' {} + diff --git a/python-avd/pyavd/j2filters/convert_dicts.py b/python-avd/pyavd/j2filters/convert_dicts.py new file mode 100644 index 00000000000..f78243d7d35 --- /dev/null +++ b/python-avd/pyavd/j2filters/convert_dicts.py @@ -0,0 +1,97 @@ +# Copyright (c) 2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +from __future__ import annotations + +import os + + +def convert_dicts(dictionary: dict | list, primary_key: str = "name", secondary_key: str | None = None) -> list: + """ + The `arista.avd.convert_dicts` filter will convert a dictionary containing nested dictionaries to a list of + dictionaries. It inserts the outer dictionary keys into each list item using the primary_key `name` (key name is + configurable) and if there is a non-dictionary value,it inserts this value to + secondary key (key name is configurable), if secondary key is provided. + + This filter is intended for: + + - Seamless data model migration from dictionaries to lists. + - Improve processing performance when dealing with large dictionaries by converting them to lists of dictionaries. + + Note: If there is a non-dictionary value with no secondary key provided, it will pass through untouched + + To use this filter: + + ```jinja + {# convert list of dictionary with default `name:` as the primary key and None secondary key #} + {% set example_list = example_dictionary | arista.avd.convert_dicts %} + {% for example_item in example_list %} + item primary key is {{ example_item.name }} + {% endfor %} + + {# convert list of dictionary with `id:` set as the primary key and `types:` set as the secondary key #} + {% set example_list = example_dictionary | arista.avd.convert_dicts('id','types') %} + {% for example_item in example_list %} + item primary key is {{ example_item.id }} + item secondary key is {{ example_item.types }} + {% endfor %} + ``` + + Parameters + ---------- + dictionary : any + Nested Dictionary to convert - returned untouched if not a nested dictionary and list + primary_key : str, optional + Name of primary key used when inserting outer dictionary keys into items. + secondary_key : str, optional + Name of secondary key used when inserting dictionary values which are list into items. + + Returns + ------- + any + Returns list of dictionaries or input variable untouched if not a nested dictionary/list. + """ + if not isinstance(dictionary, (dict, list)) or os.environ.get("AVD_DISABLE_CONVERT_DICTS"): + # Not a dictionary/list, return the original + return dictionary + if isinstance(dictionary, list): + output = [] + for element in dictionary: + if not isinstance(element, dict): + output.append({primary_key: element}) + elif primary_key not in element and secondary_key is not None: + # if element of nested dictionary is a dictionary but primary key is missing, insert primary and secondary keys. + for key in element: + output.append( + { + primary_key: key, + secondary_key: element[key], + } + ) + else: + output.append(element) + return output + # This is now a dict + output = [] + for key in dictionary: + if secondary_key is not None: + # Add secondary key for the values if secondary key is provided + output.append( + { + primary_key: key, + secondary_key: dictionary[key], + } + ) + else: + if not isinstance(dictionary[key], dict): + # Not a nested dictionary + output.append({primary_key: key}) + else: + # Nested dictionary + output.append( + { + primary_key: key, + **dictionary[key], + } + ) + return output diff --git a/python-avd/pyavd/templater.py b/python-avd/pyavd/templater.py index ebc16ee704c..400bf62827f 100644 --- a/python-avd/pyavd/templater.py +++ b/python-avd/pyavd/templater.py @@ -4,6 +4,7 @@ from jinja2 import ChoiceLoader, Environment, FileSystemLoader, ModuleLoader, StrictUndefined from .constants import JINJA2_EXTENSIONS, JINJA2_PRECOMPILED_TEMPLATE_PATH +from .j2filters.convert_dicts import convert_dicts from .j2filters.default import default from .j2filters.natural_sort import natural_sort @@ -53,7 +54,6 @@ def __init__(self, searchpaths: list[str] = None): def import_filters_and_tests(self) -> None: # pylint: disable=import-outside-toplevel - from .vendor.j2.filter.convert_dicts import convert_dicts from .vendor.j2.filter.decrypt import decrypt from .vendor.j2.filter.encrypt import encrypt from .vendor.j2.filter.hide_passwords import hide_passwords diff --git a/python-avd/tests/pyavd/j2filters/test_convert_dict.py b/python-avd/tests/pyavd/j2filters/test_convert_dict.py new file mode 100644 index 00000000000..450c22d83de --- /dev/null +++ b/python-avd/tests/pyavd/j2filters/test_convert_dict.py @@ -0,0 +1,96 @@ +# Copyright (c) 2023-2024 Arista Networks, Inc. +# Use of this source code is governed by the Apache License 2.0 +# that can be found in the LICENSE file. +from __future__ import absolute_import, division, print_function + +import pytest +from pyavd.j2filters.convert_dicts import convert_dicts + +DEFAULT_PRIMARY_KEY = "name" +NESTED_LIST_OF_DICT = { + "TEST1": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}], + "TEST2": [{"type": "deny", "extcommunities": "65001:65001"}], +} +NESTED_DICT = {"TEST1": {"action": "permit 1000:1000"}, "TEST2": {"action": "permit 2000:3000"}} +LIST_OF_STRING = ["Test1", "Test2", "Test3"] +LIST_OF_DICT = [{"type": "permit"}, {"extcommunities": "65000:65000"}] +DICT_WITH_STRING = {"dict": "test_string"} + + +class TestConvertDicts: + @pytest.mark.parametrize( + "test_dict, primary_key, secondary_key, converted_value", + [ + ( + NESTED_DICT, + "", + "", + [{"action": "permit 1000:1000", "name": "TEST1"}, {"action": "permit 2000:3000", "name": "TEST2"}], + ), # test_convert_dicts_with_nested_dict_default + ( + NESTED_DICT, + "id", + "", + [{"action": "permit 1000:1000", "id": "TEST1"}, {"action": "permit 2000:3000", "id": "TEST2"}], + ), # test_convert_dicts_with_nested_dict_primary_key + ( + NESTED_DICT, + "", + "types", + [{"name": "TEST1", "types": {"action": "permit 1000:1000"}}, {"name": "TEST2", "types": {"action": "permit 2000:3000"}}], + ), # test_convert_dicts_with_nested_dict_secondary_key + ( + NESTED_DICT, + "id", + "types", + [{"id": "TEST1", "types": {"action": "permit 1000:1000"}}, {"id": "TEST2", "types": {"action": "permit 2000:3000"}}], + ), # test_convert_dicts_with_listofdict_default + (NESTED_LIST_OF_DICT, "", "", [{"name": "TEST1"}, {"name": "TEST2"}]), # test_convert_dicts_with_listofdict_default + (NESTED_LIST_OF_DICT, "test", "", [{"test": "TEST1"}, {"test": "TEST2"}]), # test_convert_dicts_with_listofdict_primary_key + ( + NESTED_LIST_OF_DICT, + "", + "types", + [ + {"name": "TEST1", "types": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}]}, + {"name": "TEST2", "types": [{"type": "deny", "extcommunities": "65001:65001"}]}, + ], + ), # test_convert_dicts_with_listofdict_secondary_key + ( + NESTED_LIST_OF_DICT, + "id", + "types", + [ + {"id": "TEST1", "types": [{"type": "permit", "extcommunities": "65000:65000"}, {"type": "deny", "extcommunities": "65002:65002"}]}, + {"id": "TEST2", "types": [{"type": "deny", "extcommunities": "65001:65001"}]}, + ], + ), # test_convert_dicts_with_listofdict_primary_and_secondary_key + (LIST_OF_STRING, "", "", [{"name": "Test1"}, {"name": "Test2"}, {"name": "Test3"}]), # test_convert_dicts_with_list_default + (LIST_OF_STRING, "test", "", [{"test": "Test1"}, {"test": "Test2"}, {"test": "Test3"}]), # test_convert_dicts_with_list_primary_key + (LIST_OF_STRING, "", "id", [{"name": "Test1"}, {"name": "Test2"}, {"name": "Test3"}]), # test_convert_dicts_with_list_secondary_key + ( + LIST_OF_STRING, + "test", + "types", + [{"test": "Test1"}, {"test": "Test2"}, {"test": "Test3"}], + ), # test_convert_dicts_with_list_primary_and_secondary_key + (DICT_WITH_STRING, "", "", [{"name": "dict"}]), # test_convert_dicts_with_string_value_default + (DICT_WITH_STRING, "test", "", [{"test": "dict"}]), # test_convert_dicts_with_string_value_primary_key + (DICT_WITH_STRING, "", "str", [{"name": "dict", "str": "test_string"}]), # test_convert_dicts_with_string_value_secondary_key + (DICT_WITH_STRING, "test", "str", [{"test": "dict", "str": "test_string"}]), # test_convert_dicts_with_string_value_primary_key_and_secondary_key + (LIST_OF_DICT, "", "", LIST_OF_DICT), # test_convert_dicts_with_list_of_dict_default + (LIST_OF_DICT, "test", "", LIST_OF_DICT), # test_convert_dicts_with_list_of_dict_primary_key + ( + LIST_OF_DICT, + "", + "id", + [{"name": "type", "id": "permit"}, {"name": "extcommunities", "id": "65000:65000"}], + ), # test_convert_dicts_with_list_of_dict_secondary_key + (LIST_OF_DICT, "test", "id", [{"test": "type", "id": "permit"}, {"test": "extcommunities", "id": "65000:65000"}]), + ], + ) # test_convert_dicts_with_list_of_dict_primary_key_and_secondary_key + def test_convert_dicts(self, test_dict, primary_key, secondary_key, converted_value): + primary_key = primary_key if primary_key else DEFAULT_PRIMARY_KEY + secondary_key = secondary_key if secondary_key else None + resp = convert_dicts(test_dict, primary_key, secondary_key) + assert resp == converted_value