From 85f82ddff4c3720cceea6dc67cf64b1b41be5aac Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Thu, 1 Aug 2024 08:36:25 -0500 Subject: [PATCH 01/29] add custom relationship sorting --- nautobot_ssot/contrib/adapter.py | 51 ++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index f83bf056c..696eabaa9 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -34,6 +34,54 @@ ParameterSet = FrozenSet[Tuple[str, Hashable]] +def sort_relationships(diffsync: DiffSync): + """Helper function for SSoT adapters for sorting relationships entries to avoid false updates actions. + + This function checks the `_sorted_relationships` attribute in the DiffSync object/adpater. If present, it will + loop through all entries of the attribute and sort the objects accordingly. + + The `_sorted_relationships` should be a list or tuple of lists/tuples. Each entry must have three strings: + - Name of the DiffSync model with attribute to be sorted + - Name of the attribute to be sorted + - Name of the key within the attribute to be sorted by + + NOTE: The listed attribute MUST be a list of dictionaries indicating many-to-many relationships on either side + or a one-to-many relationship from the many side. + + """ + if not hasattr(diffsync, "_sorted_relationships"): + # Nothing to do + return + + for entry in diffsync._sorted_relationships: + if not isinstance(entry, tuple) and not isinstance(entry, list): + diffsync.job.logger.error(f"{type(entry)} not allowed for sorting. Skipping entry.") + continue + + if len(entry) != 3: + diffsync.job.logger.error( + "Sort Error: Invalid input. Defined objects sort must be a tuple with three entries " + "(object name, attribute name, and sort by key). Skipping entry." + ) + continue + + obj_name = entry[0] + attr_name = entry[1] + sort_by_key = entry[2] + + if not isinstance(obj_name, str) or not isinstance(attr_name, str) or not isinstance(sort_by_key, str): + diffsync.job.logger.error("Paramaters for `_sorted_relationship` entries must all be strings.") + continue + + for obj in diffsync.get_all(obj_name): + setattr( + obj, + attr_name, + sorted(getattr(obj, attr_name), key=lambda x: x[sort_by_key]), + ) + diffsync.update(obj) + + class NautobotAdapter(DiffSync): """ Adapter for loading data from Nautobot through the ORM. @@ -44,6 +92,7 @@ class NautobotAdapter(DiffSync): # This dictionary acts as an ORM cache. _cache: DefaultDict[str, Dict[ParameterSet, Model]] _cache_hits: DefaultDict[str, int] = defaultdict(int) + _sorted_relationships = () def __init__(self, *args, job, sync=None, **kwargs): """Instantiate this class, but do not load data immediately from the local system.""" @@ -174,6 +223,8 @@ def load(self): # for this specific model class as well as its children without returning anything. self._load_objects(diffsync_model) + sort_relationships(self) + def _get_diffsync_class(self, model_name): """Given a model name, return the diffsync class.""" try: From aa44338012b5e7dcd26e925b91e2dde769cd41c6 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Thu, 1 Aug 2024 14:49:19 -0500 Subject: [PATCH 02/29] fixes and add testing --- nautobot_ssot/contrib/__init__.py | 3 +- nautobot_ssot/contrib/adapter.py | 35 ++++++++------- nautobot_ssot/tests/contrib_base_classes.py | 20 ++++++++- nautobot_ssot/tests/test_contrib_adapter.py | 50 ++++++++++++++++++++- 4 files changed, 90 insertions(+), 18 deletions(-) diff --git a/nautobot_ssot/contrib/__init__.py b/nautobot_ssot/contrib/__init__.py index 6e7f6b275..35e00f249 100644 --- a/nautobot_ssot/contrib/__init__.py +++ b/nautobot_ssot/contrib/__init__.py @@ -1,6 +1,6 @@ """SSoT Contrib.""" -from nautobot_ssot.contrib.adapter import NautobotAdapter +from nautobot_ssot.contrib.adapter import NautobotAdapter, sort_relationships from nautobot_ssot.contrib.model import NautobotModel from nautobot_ssot.contrib.types import ( RelationshipSideEnum, @@ -14,4 +14,5 @@ "NautobotAdapter", "NautobotModel", "RelationshipSideEnum", + "sort_relationships", ) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 696eabaa9..a8b6d3607 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -47,40 +47,45 @@ def sort_relationships(diffsync: DiffSync): NOTE: The listed attribute MUST be a list of dictionaries indicating many-to-many relationships on either side or a one-to-many relationship from the many side. - """ if not hasattr(diffsync, "_sorted_relationships"): # Nothing to do return + if not isinstance(diffsync, DiffSync): + raise TypeError("Parameter for `sort_relationships()` must be of type DiffSync.") + if not diffsync._sorted_relationships: + return for entry in diffsync._sorted_relationships: - if not isinstance(entry, tuple) and not isinstance(entry, list): - diffsync.job.logger.error(f"{type(entry)} not allowed for sorting. Skipping entry.") - continue - if len(entry) != 3: - diffsync.job.logger.error( - "Sort Error: Invalid input. Defined objects sort must be a tuple with three entries " - "(object name, attribute name, and sort by key). Skipping entry." - ) - continue + if not isinstance(entry, tuple) and not isinstance(entry, list): + raise TypeError(f"Invalid type: {type(entry)}. Valid types include tuples or lists.") obj_name = entry[0] attr_name = entry[1] sort_by_key = entry[2] - if not isinstance(obj_name, str) or not isinstance(attr_name, str) or not isinstance(sort_by_key, str): - diffsync.job.logger.error("Paramaters for `_sorted_relationship` entries must all be strings.") - continue + # if not isinstance(obj_name, str) or not isinstance(attr_name, str) or not isinstance(sort_by_key, str): + # diffsync.job.logger.error("Paramaters for `_sorted_relationship` entries must all be strings.") + # continue for obj in diffsync.get_all(obj_name): + sorted_data = sorted( + getattr(obj, attr_name), + key=lambda x: x[sort_by_key], + ) + setattr(obj, attr_name, sorted_data) + setattr( obj, attr_name, - sorted(getattr(obj, attr_name), key=lambda x: x[sort_by_key]), + sorted( + getattr(obj, attr_name), + key=lambda x: x[sort_by_key] + ), ) diffsync.update(obj) - + class NautobotAdapter(DiffSync): """ diff --git a/nautobot_ssot/tests/contrib_base_classes.py b/nautobot_ssot/tests/contrib_base_classes.py index c694e85a4..ec535e6d0 100644 --- a/nautobot_ssot/tests/contrib_base_classes.py +++ b/nautobot_ssot/tests/contrib_base_classes.py @@ -1,6 +1,6 @@ """Base classes for contrib testing.""" -from typing import Optional, List +from typing import Optional, List, TypedDict from unittest import skip from unittest.mock import MagicMock from diffsync.exceptions import ObjectNotCreated, ObjectNotUpdated, ObjectNotDeleted @@ -527,3 +527,21 @@ class ProviderModelCustomRelationship(NautobotModel): tenants: Annotated[ List[TenantDict], CustomRelationshipAnnotation(name="Test Relationship", side=RelationshipSideEnum.SOURCE) ] = [] + + +class CustomRelationshipTypedDict(TypedDict): + """Typed dictionary for testing custom many to many relationships.""" + + name: str + + +class TenantModelCustomManyTomanyRelationship(NautobotModel): + """""" + + _model = tenancy_models.Tenant + _modelname = "tenant" + _identifiers = ("name",) + _attributes = ("tenants",) + + name: str + tenants: list[TenantDict] = [] \ No newline at end of file diff --git a/nautobot_ssot/tests/test_contrib_adapter.py b/nautobot_ssot/tests/test_contrib_adapter.py index cf4c7b05a..eeee3b880 100644 --- a/nautobot_ssot/tests/test_contrib_adapter.py +++ b/nautobot_ssot/tests/test_contrib_adapter.py @@ -19,7 +19,7 @@ from nautobot.tenancy import models as tenancy_models from typing_extensions import Annotated, TypedDict -from nautobot_ssot.contrib import NautobotAdapter, NautobotModel, CustomFieldAnnotation +from nautobot_ssot.contrib import NautobotAdapter, NautobotModel, CustomFieldAnnotation, sort_relationships from nautobot_ssot.tests.contrib_base_classes import ( TestCaseWithDeviceData, NautobotDevice, @@ -29,6 +29,7 @@ NautobotTenant, TenantModelCustomRelationship, ProviderModelCustomRelationship, + TenantModelCustomManyTomanyRelationship, ) @@ -354,3 +355,50 @@ class Adapter(NautobotAdapter): self.assertEqual(amount_of_vlans, len(diffsync_vlan_group.vlans)) for vlan in diffsync_vlan_group.vlans: self.assertEqual(location.name, vlan["location__name"]) + + +class AdapterCustomRelationshipSortingTest(NautobotAdapter): + """Adapter for testing custom many-to-many relationship sorting.""" + + top_level = ["tenant"] + tenant = TenantModelCustomManyTomanyRelationship + _sorted_relationships = (("tenant", "tenants", "name",),) + + +class TestSortedRelationships(TestCase): + """Tests for `sort_relationships` function.""" + + def setUp(self): + self.adapter = AdapterCustomRelationshipSortingTest( + job=MagicMock() + ) + + self.adapter.add(self.adapter.tenant( + name="Tenant 1", + tenants=[ + {"name": "C"}, + {"name": "B"}, + {"name": "A"}, + ] + )) + + def test_valid_sorting(self): + """Test to ensure the function properly sorts basic information.""" + # Validate before settings + before = self.adapter.get("tenant", identifier="Tenant 1") + self.assertEqual(before.tenants[0]["name"], "C") + self.assertEqual(before.tenants[1]["name"], "B") + self.assertEqual(before.tenants[2]["name"], "A") + + sort_relationships(self.adapter) + + after = self.adapter.get("tenant", identifier="Tenant 1") + self.assertEqual(after.tenants[0]["name"], "A") + self.assertEqual(after.tenants[1]["name"], "B") + self.assertEqual(after.tenants[2]["name"], "C") + + def test_invalid_type(self): + """Test passing invalid type to function.""" + self.adapter._sorted_relationships = {"Entry 1": "Value 1"} + with self.assertRaises(TypeError): + sort_relationships(self.adapter) From 97a6cdb47415868a4b703196b7b0bdfb81e467f9 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 08:15:59 -0500 Subject: [PATCH 03/29] flix black errors --- nautobot_ssot/contrib/adapter.py | 7 ++--- nautobot_ssot/tests/contrib_base_classes.py | 2 +- nautobot_ssot/tests/test_contrib_adapter.py | 30 ++++++++++++--------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index a8b6d3607..697b28a3a 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -36,7 +36,7 @@ def sort_relationships(diffsync: DiffSync): """Helper function for SSoT adapters for sorting relationships entries to avoid false updates actions. - + This function checks the `_sorted_relationships` attribute in the DiffSync object/adpater. If present, it will loop through all entries of the attribute and sort the objects accordingly. @@ -79,10 +79,7 @@ def sort_relationships(diffsync: DiffSync): setattr( obj, attr_name, - sorted( - getattr(obj, attr_name), - key=lambda x: x[sort_by_key] - ), + sorted(getattr(obj, attr_name), key=lambda x: x[sort_by_key]), ) diffsync.update(obj) diff --git a/nautobot_ssot/tests/contrib_base_classes.py b/nautobot_ssot/tests/contrib_base_classes.py index ec535e6d0..58492b510 100644 --- a/nautobot_ssot/tests/contrib_base_classes.py +++ b/nautobot_ssot/tests/contrib_base_classes.py @@ -544,4 +544,4 @@ class TenantModelCustomManyTomanyRelationship(NautobotModel): _attributes = ("tenants",) name: str - tenants: list[TenantDict] = [] \ No newline at end of file + tenants: list[TenantDict] = [] diff --git a/nautobot_ssot/tests/test_contrib_adapter.py b/nautobot_ssot/tests/test_contrib_adapter.py index eeee3b880..832b9ecfb 100644 --- a/nautobot_ssot/tests/test_contrib_adapter.py +++ b/nautobot_ssot/tests/test_contrib_adapter.py @@ -362,26 +362,32 @@ class AdapterCustomRelationshipSortingTest(NautobotAdapter): top_level = ["tenant"] tenant = TenantModelCustomManyTomanyRelationship - _sorted_relationships = (("tenant", "tenants", "name",),) + _sorted_relationships = ( + ( + "tenant", + "tenants", + "name", + ), + ) class TestSortedRelationships(TestCase): """Tests for `sort_relationships` function.""" def setUp(self): - self.adapter = AdapterCustomRelationshipSortingTest( - job=MagicMock() + self.adapter = AdapterCustomRelationshipSortingTest(job=MagicMock()) + + self.adapter.add( + self.adapter.tenant( + name="Tenant 1", + tenants=[ + {"name": "C"}, + {"name": "B"}, + {"name": "A"}, + ], + ) ) - self.adapter.add(self.adapter.tenant( - name="Tenant 1", - tenants=[ - {"name": "C"}, - {"name": "B"}, - {"name": "A"}, - ] - )) - def test_valid_sorting(self): """Test to ensure the function properly sorts basic information.""" # Validate before settings From 0a8ccb9be6c16e8d6b687cf3eaf136c765ca34f7 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 08:16:43 -0500 Subject: [PATCH 04/29] fix flake8 --- nautobot_ssot/tests/contrib_base_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/tests/contrib_base_classes.py b/nautobot_ssot/tests/contrib_base_classes.py index 58492b510..a2ca9def4 100644 --- a/nautobot_ssot/tests/contrib_base_classes.py +++ b/nautobot_ssot/tests/contrib_base_classes.py @@ -1,6 +1,6 @@ """Base classes for contrib testing.""" -from typing import Optional, List, TypedDict +from typing import Optional, List from unittest import skip from unittest.mock import MagicMock from diffsync.exceptions import ObjectNotCreated, ObjectNotUpdated, ObjectNotDeleted From 92c874992ba283971e0f14624393b9c0d715d2db Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 10:23:31 -0500 Subject: [PATCH 05/29] update docs --- docs/dev/jobs.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/docs/dev/jobs.md b/docs/dev/jobs.md index efab418e5..d27445e9d 100644 --- a/docs/dev/jobs.md +++ b/docs/dev/jobs.md @@ -167,7 +167,7 @@ The methods [`calculate_diff`][nautobot_ssot.jobs.base.DataSyncBaseJob.calculate Optionally, on your Job class, also implement the [`lookup_object`][nautobot_ssot.jobs.base.DataSyncBaseJob.lookup_object], [`data_mapping`][nautobot_ssot.jobs.base.DataSyncBaseJob.data_mappings], and/or [`config_information`][nautobot_ssot.jobs.base.DataSyncBaseJob.config_information] APIs (to provide more information to the end user about the details of this Job), as well as the various metadata properties on your Job's Meta inner class. Refer to the example Jobs provided in this Nautobot app for examples and further details. Install your Job via any of the supported Nautobot methods (installation into the `JOBS_ROOT` directory, inclusion in a Git repository, or packaging as part of an app) and it should automatically become available! -### Extra Step: Implementing `create`, `update` and `delete` +### Extra Step 1: Implementing `create`, `update` and `delete` If you are synchronizing data _to_ Nautobot and not _from_ Nautobot, you can entirely skip this step. The `nautobot_ssot.contrib.NautobotModel` class provides this functionality automatically. @@ -177,4 +177,52 @@ If you need to perform the `create`, `update` and `delete` operations on the rem You still want your models to adhere to the [modeling guide](../user/modeling.md), since it provides you the auto-generated `load` function for the diffsync adapter on the Nautobot side. !!! warning - Special care should be taken when synchronizing new Devices with children Interfaces into a Nautobot instance that also defines Device Types with Interface components of the same name. When the new Device is created in Nautobot, its Interfaces will also be created as defined in the respective Device Type. As a result, when SSoT will attempt to create the children Interfaces loaded by the remote adapter, these will already exist in the target Nautobot system. In this scenario, if not properly handled, the sync will fail! Possible remediation steps may vary depending on the specific use-case, therefore this is left as an exercise to the reader/developer to solve for their specific context. \ No newline at end of file + Special care should be taken when synchronizing new Devices with children Interfaces into a Nautobot instance that also defines Device Types with Interface components of the same name. When the new Device is created in Nautobot, its Interfaces will also be created as defined in the respective Device Type. As a result, when SSoT will attempt to create the children Interfaces loaded by the remote adapter, these will already exist in the target Nautobot system. In this scenario, if not properly handled, the sync will fail! Possible remediation steps may vary depending on the specific use-case, therefore this is left as an exercise to the reader/developer to solve for their specific context. + +### Extra Step 2: Sorting Many-to-Many and Many-to-One Relationships + +If you are not syncing any many-to-many relationships (M2M) or many-to-one (N:1) relationships from the many side, you can skip this step. + +Loading M2M and N:1 relationship data from source and target destinations are typically not in the same order as each other. For example, the order of a device's interfaces from the source data may differ compared to the order Nautobot loads the data. + +To resolve this, each relationships must be properly sorted before the source and target are compared against eachater. An additional attribute called `_sorted_relationships` must be defined in both the source and target adapters. This attribute must be identical between both adapters. + +M2M and N:1 relationships are stored in the DiffSync store as a list of dictionaries. To sort a list of dictionaries, we must specify a dictionary key to sort by and do so by using code similar to the following: + +``` +for obj in diffsync.get_all("model_name"): + sorted_data = sorted( + obj.attribute_name, + key=lamda x: x["sort_by_key_name"] + ) + obj.attribute_name = sorted_data + diffsync.update(obj) +``` + +The `_sorted_relationships` attribute was added is a tuple of tuples. Each entry must have three string values indicating: + +1. Name of the model with attribute to be sorted +2. Attribute within the model to sort +3. Dictionary key name to sort by + +The helper function `sort_relationships` has been added to contrib to assist in sorting relationships. The `NautobotAdapter` will automatically call this function and process any entries added to `_sorted_relationships`. + +For integrations other than the `NautobotAdapter`, you must also import and add the `sort_relationships` into into the `load()` method and simply pass the DiffSync/Adapter object through using `self`. This must be done after all other loading logic is completed. + +Example: +``` +from nautobot_ssot.contrib import sort_relationships + +class SourceAdapter(DiffSync): + + _sorted_relationships = ( + ("tenant", "relationship", "name"), + ) + + def load(self): + ... + # Primary load logic + ... + sort_relationships(self) +``` + From fa6ebee55270f44e505b2d69ecd0076c2e72d0f2 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 11:17:09 -0500 Subject: [PATCH 06/29] add changelog --- changes/457.added | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/457.added diff --git a/changes/457.added b/changes/457.added new file mode 100644 index 000000000..f67ad91f2 --- /dev/null +++ b/changes/457.added @@ -0,0 +1,3 @@ +Added `sort_relationships()` helper function +Added tests for `sort_relationships()` helper function +Added call to `sort_relationships()` function in contrib `NautobotAdapter` \ No newline at end of file From 99a266a4a0fc62cb6eb7fc68bf27e470680214be Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 11:35:45 -0500 Subject: [PATCH 07/29] add docstring --- nautobot_ssot/tests/contrib_base_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/tests/contrib_base_classes.py b/nautobot_ssot/tests/contrib_base_classes.py index a2ca9def4..aa55de269 100644 --- a/nautobot_ssot/tests/contrib_base_classes.py +++ b/nautobot_ssot/tests/contrib_base_classes.py @@ -536,7 +536,7 @@ class CustomRelationshipTypedDict(TypedDict): class TenantModelCustomManyTomanyRelationship(NautobotModel): - """""" + """Model for testing sorting custom relationships.""" _model = tenancy_models.Tenant _modelname = "tenant" From f5d5bc35b5dbef76dbbc633b3a7eb9f718a9d77b Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 11:59:12 -0500 Subject: [PATCH 08/29] fixes --- docs/dev/jobs.md | 8 ++++---- nautobot_ssot/contrib/adapter.py | 16 ++++++---------- nautobot_ssot/tests/test_contrib_adapter.py | 4 ++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/dev/jobs.md b/docs/dev/jobs.md index d27445e9d..e61a056f0 100644 --- a/docs/dev/jobs.md +++ b/docs/dev/jobs.md @@ -185,7 +185,7 @@ If you are not syncing any many-to-many relationships (M2M) or many-to-one (N:1) Loading M2M and N:1 relationship data from source and target destinations are typically not in the same order as each other. For example, the order of a device's interfaces from the source data may differ compared to the order Nautobot loads the data. -To resolve this, each relationships must be properly sorted before the source and target are compared against eachater. An additional attribute called `_sorted_relationships` must be defined in both the source and target adapters. This attribute must be identical between both adapters. +To resolve this, each relationships must be properly sorted before the source and target are compared against eachater. An additional attribute called `sorted_relationships` must be defined in both the source and target adapters. This attribute must be identical between both adapters. M2M and N:1 relationships are stored in the DiffSync store as a list of dictionaries. To sort a list of dictionaries, we must specify a dictionary key to sort by and do so by using code similar to the following: @@ -199,13 +199,13 @@ for obj in diffsync.get_all("model_name"): diffsync.update(obj) ``` -The `_sorted_relationships` attribute was added is a tuple of tuples. Each entry must have three string values indicating: +The `sorted_relationships` attribute was added is a tuple of tuples. Each entry must have three string values indicating: 1. Name of the model with attribute to be sorted 2. Attribute within the model to sort 3. Dictionary key name to sort by -The helper function `sort_relationships` has been added to contrib to assist in sorting relationships. The `NautobotAdapter` will automatically call this function and process any entries added to `_sorted_relationships`. +The helper function `sort_relationships` has been added to contrib to assist in sorting relationships. The `NautobotAdapter` will automatically call this function and process any entries added to `sorted_relationships`. For integrations other than the `NautobotAdapter`, you must also import and add the `sort_relationships` into into the `load()` method and simply pass the DiffSync/Adapter object through using `self`. This must be done after all other loading logic is completed. @@ -215,7 +215,7 @@ from nautobot_ssot.contrib import sort_relationships class SourceAdapter(DiffSync): - _sorted_relationships = ( + sorted_relationships = ( ("tenant", "relationship", "name"), ) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 697b28a3a..87b046f87 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -37,10 +37,10 @@ def sort_relationships(diffsync: DiffSync): """Helper function for SSoT adapters for sorting relationships entries to avoid false updates actions. - This function checks the `_sorted_relationships` attribute in the DiffSync object/adpater. If present, it will + This function checks the `sorted_relationships` attribute in the DiffSync object/adpater. If present, it will loop through all entries of the attribute and sort the objects accordingly. - The `_sorted_relationships` should be a list or tuple of lists/tuples. Each entry must have three strings: + The `sorted_relationships` should be a list or tuple of lists/tuples. Each entry must have three strings: - Name of the DiffSync model with attribute to be sorted - Name of the attribute to be sorted - Name of the key within the attribute to be sorted by @@ -48,15 +48,15 @@ def sort_relationships(diffsync: DiffSync): NOTE: The listed attribute MUST be a list of dictionaries indicating many-to-many relationships on either side or a one-to-many relationship from the many side. """ - if not hasattr(diffsync, "_sorted_relationships"): + if not hasattr(diffsync, "sorted_relationships"): # Nothing to do return if not isinstance(diffsync, DiffSync): raise TypeError("Parameter for `sort_relationships()` must be of type DiffSync.") - if not diffsync._sorted_relationships: + if not diffsync.sorted_relationships: return - for entry in diffsync._sorted_relationships: + for entry in diffsync.sorted_relationships: if not isinstance(entry, tuple) and not isinstance(entry, list): raise TypeError(f"Invalid type: {type(entry)}. Valid types include tuples or lists.") @@ -65,10 +65,6 @@ def sort_relationships(diffsync: DiffSync): attr_name = entry[1] sort_by_key = entry[2] - # if not isinstance(obj_name, str) or not isinstance(attr_name, str) or not isinstance(sort_by_key, str): - # diffsync.job.logger.error("Paramaters for `_sorted_relationship` entries must all be strings.") - # continue - for obj in diffsync.get_all(obj_name): sorted_data = sorted( getattr(obj, attr_name), @@ -94,7 +90,7 @@ class NautobotAdapter(DiffSync): # This dictionary acts as an ORM cache. _cache: DefaultDict[str, Dict[ParameterSet, Model]] _cache_hits: DefaultDict[str, int] = defaultdict(int) - _sorted_relationships = () + sorted_relationships = () def __init__(self, *args, job, sync=None, **kwargs): """Instantiate this class, but do not load data immediately from the local system.""" diff --git a/nautobot_ssot/tests/test_contrib_adapter.py b/nautobot_ssot/tests/test_contrib_adapter.py index 832b9ecfb..7dd092763 100644 --- a/nautobot_ssot/tests/test_contrib_adapter.py +++ b/nautobot_ssot/tests/test_contrib_adapter.py @@ -362,7 +362,7 @@ class AdapterCustomRelationshipSortingTest(NautobotAdapter): top_level = ["tenant"] tenant = TenantModelCustomManyTomanyRelationship - _sorted_relationships = ( + sorted_relationships = ( ( "tenant", "tenants", @@ -405,6 +405,6 @@ def test_valid_sorting(self): def test_invalid_type(self): """Test passing invalid type to function.""" - self.adapter._sorted_relationships = {"Entry 1": "Value 1"} + self.adapter.sorted_relationships = {"Entry 1": "Value 1"} with self.assertRaises(TypeError): sort_relationships(self.adapter) From d7804f0558289e1861837292ebec3edd0ca69be1 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 12:16:57 -0500 Subject: [PATCH 09/29] fix pylint --- nautobot_ssot/contrib/adapter.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 87b046f87..8f1358cc1 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -68,15 +68,9 @@ def sort_relationships(diffsync: DiffSync): for obj in diffsync.get_all(obj_name): sorted_data = sorted( getattr(obj, attr_name), - key=lambda x: x[sort_by_key], + key=lambda x: x[sort_by_key], # pylint-ignore=cell-var-from-loop ) setattr(obj, attr_name, sorted_data) - - setattr( - obj, - attr_name, - sorted(getattr(obj, attr_name), key=lambda x: x[sort_by_key]), - ) diffsync.update(obj) From 4670ba1c53ce3d8d36eb2ea4b6d121c90915d17d Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 12:56:47 -0500 Subject: [PATCH 10/29] add lint ignore --- nautobot_ssot/contrib/adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 8f1358cc1..87d789a8f 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -68,7 +68,7 @@ def sort_relationships(diffsync: DiffSync): for obj in diffsync.get_all(obj_name): sorted_data = sorted( getattr(obj, attr_name), - key=lambda x: x[sort_by_key], # pylint-ignore=cell-var-from-loop + key=lambda x: x[sort_by_key], # pylint: disable=cell-var-from-loop ) setattr(obj, attr_name, sorted_data) diffsync.update(obj) From 2b6e1e08137ff48c5ab955aa34778d15addef07b Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 2 Aug 2024 13:17:47 -0500 Subject: [PATCH 11/29] fix error --- nautobot_ssot/tests/contrib_base_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/tests/contrib_base_classes.py b/nautobot_ssot/tests/contrib_base_classes.py index aa55de269..60bebd70b 100644 --- a/nautobot_ssot/tests/contrib_base_classes.py +++ b/nautobot_ssot/tests/contrib_base_classes.py @@ -544,4 +544,4 @@ class TenantModelCustomManyTomanyRelationship(NautobotModel): _attributes = ("tenants",) name: str - tenants: list[TenantDict] = [] + tenants: List[TenantDict] = [] From 440577b55960d81033ea92bd8602c2507b57e655 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Thu, 26 Dec 2024 09:22:26 -0600 Subject: [PATCH 12/29] remove unneeded if statement --- nautobot_ssot/contrib/adapter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 48beaaba6..8a069df46 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -53,8 +53,6 @@ def sort_relationships(diffsync: DiffSync): return if not isinstance(diffsync, DiffSync): raise TypeError("Parameter for `sort_relationships()` must be of type DiffSync.") - if not diffsync.sorted_relationships: - return for entry in diffsync.sorted_relationships: From c564d068feab5adf93c425f2ab8781ebdaa74dfd Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Fri, 27 Dec 2024 13:37:55 -0600 Subject: [PATCH 13/29] initial commit --- nautobot_ssot/contrib/sorting.py | 108 +++++++++++++ nautobot_ssot/tests/contrib/__init__.py | 0 nautobot_ssot/tests/contrib/test_sorting.py | 169 ++++++++++++++++++++ 3 files changed, 277 insertions(+) create mode 100644 nautobot_ssot/contrib/sorting.py create mode 100644 nautobot_ssot/tests/contrib/__init__.py create mode 100644 nautobot_ssot/tests/contrib/test_sorting.py diff --git a/nautobot_ssot/contrib/sorting.py b/nautobot_ssot/contrib/sorting.py new file mode 100644 index 000000000..0c9a77619 --- /dev/null +++ b/nautobot_ssot/contrib/sorting.py @@ -0,0 +1,108 @@ +"""""" + +from typing import Annotated +#from typing_extensions import get_type_hints +#from dataclasses import dataclass +from pprint import pprint +from typing import List, Optional, TypedDict +import typing +from enum import Enum +from typing_extensions import get_type_hints +from nautobot_ssot.contrib.types import FieldType +from diffsync import Adapter, DiffSyncModel + + +def _is_sortable_field(attribute_type_hints): + """Checks type hints to verify if field labled as sortable or not.""" + if attribute_type_hints.__name__ != "Annotated" or \ + not hasattr(attribute_type_hints, "__metadata__"): + return False + for metadata in attribute_type_hints.__metadata__: + if metadata == FieldType.SORTED_FIELD: + return True + return False + + +def _get_sortable_obj_type(attribute_type_hints): + """""" + attr_type = attribute_type_hints.__args__[0] + attr_type_args = getattr(attr_type, "__args__") + if attr_type_args: + return attr_type_args[0] + return None + + +def _get_sortable_obj_sort_key(sortable_obj_type): + """""" + content_obj_type_hints = get_type_hints(sortable_obj_type, include_extras=True) + for key, value in content_obj_type_hints.items(): + if not value.__name__ == "Annotated": + continue + for metadata in getattr(value, "__metadata__", ()): + if metadata == FieldType.SORT_BY: + return key + return None + + +def _get_sortable_fields_from_model(model): + """""" + sortable_fields = [] + model_type_hints = get_type_hints(model, include_extras=True) + + for attribute_name in model._attributes: + attribute_type_hints = model_type_hints.get(attribute_name) + if not _is_sortable_field(attribute_type_hints): + continue + + sortable_obj_type = _get_sortable_obj_type(attribute_type_hints) + sort_key = _get_sortable_obj_sort_key(sortable_obj_type) + + sortable_fields.append({ + "attribute": attribute_name, + "sort_key": sort_key, + }) + return sortable_fields + + +def sort_relationships(source: Adapter, target: Adapter): + """Sort relationships based on the metadata defined in the DiffSync model.""" + if not isinstance(source, Adapter) or not isinstance(target, Adapter): + raise TypeError("Parameters for `sort_relationships()` must be of type DiffSync.") + + # Loop through Top Level entries + for level in target.top_level: + # Get the DiffSync Model + model = getattr(target, level) + if not model: + continue + + # Get sortable fields from model + sortable_fields = _get_sortable_fields_from_model(target) + if not sortable_fields: + continue + + for sortable in sortable_fields: + attribute = sortable["attribute"] + key = sortable["sort_key"] + + #source_items = source.get_all(attribute) + #target_items = target.get_all(attribute) + + for adapter in (source, target): + for obj in adapter.get_all(attribute): + if key: + sorted_data = sorted( + getattr(obj, attribute), + key=lambda x: x[key], + ) + else: + sorted_data = sorted( + getattr(obj, attribute) + ) + setattr(obj, attribute, sorted_data) + adapter.update(obj) + # end for + + + + \ No newline at end of file diff --git a/nautobot_ssot/tests/contrib/__init__.py b/nautobot_ssot/tests/contrib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py new file mode 100644 index 000000000..b446fd285 --- /dev/null +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -0,0 +1,169 @@ + +from typing import List, Optional +from unittest import skip +from unittest.mock import MagicMock + +from nautobot_ssot.contrib.types import FieldType +import nautobot.circuits.models as circuits_models +import nautobot.dcim.models as dcim_models +import nautobot.extras.models as extras_models +import nautobot.ipam.models as ipam_models +import nautobot.tenancy.models as tenancy_models +from diffsync.exceptions import ObjectNotCreated, ObjectNotDeleted, ObjectNotUpdated +from django.contrib.contenttypes.models import ContentType +#from nautobot.core.testing import TestCase +from django.test import TestCase as TestCase +from nautobot.dcim.choices import InterfaceTypeChoices +from typing_extensions import Annotated, TypedDict +from typing_extensions import get_type_hints + +from nautobot_ssot.contrib.sorting import ( + _find_list_sort_key, + _get_sortable_list_type_from_annotations, + _sort_top_level_model_attributes, + _get_sortable_fields_from_model, + _is_sortable_field, + _get_sortable_obj_type, + _get_sortable_obj_sort_key, + sort_relationships, +) + +from nautobot_ssot.contrib import ( + CustomFieldAnnotation, + CustomRelationshipAnnotation, + NautobotAdapter, + NautobotModel, + RelationshipSideEnum, +) + +class BasicTagDict(TypedDict): + """Many-to-many relationship typed dict explaining which fields are interesting.""" + + id: int + name: str + + +class TagDict(TypedDict): + """Many-to-many relationship typed dict explaining which fields are interesting.""" + + id: int + name: Annotated[str, FieldType.SORT_BY] + description: Optional[str] + + +class BasicNautobotTenant(NautobotModel): + """A tenant model for testing the `NautobotModel` base class.""" + + _model = tenancy_models.Tenant + _modelname = "tenant" + _identifiers = ("name",) + _attributes = ("description", "tenant_group__name", "tags") + + name: str + description: Optional[str] = None + tenant_group__name: Optional[str] = None + tags: List[BasicTagDict] = [] + + +class NautobotTenant(NautobotModel): + """A tenant model for testing the `NautobotModel` base class.""" + + _model = tenancy_models.Tenant + _modelname = "tenant" + _identifiers = ("name",) + _attributes = ("description", "tenant_group__name", "tags") + + name: str + description: Optional[str] = None + tenant_group__name: Optional[str] = None + tags: Annotated[List[TagDict], FieldType.SORTED_FIELD] = [] + + +class TestCaseIsSortableField(TestCase): + """""" + + @classmethod + def setUpTestData(cls): + cls.type_hints = get_type_hints(NautobotTenant, include_extras=True) + + def test_non_sortable_field(self): + test = _is_sortable_field(self.type_hints["name"]) + self.assertFalse(test) + + def test_sortable_field(self): + test = _is_sortable_field(self.type_hints["tags"]) + self.assertTrue(test) + + +class TestGetSortKey(TestCase): + """""" + + def test_get_sort_key(self): + test = _get_sortable_obj_sort_key(TagDict) + self.assertEqual(test, "name") + + def test_no_sort_key(self): + """""" + class TestClass(TypedDict): + id: str + name:str + + test = _get_sortable_obj_sort_key(TestClass) + self.assertIsNone(test) + + + + +class TestCaseGetSortableFieldsFromModel(TestCase): + """""" + + + + + + + + + + + + + + + +''' + + +class TestCaseGetSortableEntriesFunction(TestCase): + """""" + + def test_nonsortable_entry(self): + """""" + test = _get_sortable_list_type_from_annotations(BasicNautobotTenant, "description") + self.assertIsNone(test) + + def test_sortable_entry_without_annotation(self): + """""" + test = _get_sortable_list_type_from_annotations(BasicNautobotTenant, "tags") + self.assertEqual(test, BasicTagDict) + + def test_sortable_entry_with_annotation(self): + """""" + test = _get_sortable_list_type_from_annotations(NautobotTenant, "tags") + self.assertEqual(test, TagDict) + + +class TestCaseGetSortKeyFunction(TestCase): + """""" + + def test_without_sort_key_annotation(self): + """""" + test = _find_list_sort_key(BasicTagDict) + self.assertIsNone(test) + + def test_with_sort_key_annotation(self): + """""" + test = _find_list_sort_key(TagDict) + self.assertEqual(test, "name") + +''' From b26b5c3587cdb3e4a8d70e461b30ff670e38b681 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 11:30:57 -0600 Subject: [PATCH 14/29] updates --- nautobot_ssot/contrib/sorting.py | 56 ++++----- nautobot_ssot/tests/contrib/test_sorting.py | 128 ++++++++++---------- 2 files changed, 90 insertions(+), 94 deletions(-) diff --git a/nautobot_ssot/contrib/sorting.py b/nautobot_ssot/contrib/sorting.py index 0c9a77619..a72a032d7 100644 --- a/nautobot_ssot/contrib/sorting.py +++ b/nautobot_ssot/contrib/sorting.py @@ -4,9 +4,6 @@ #from typing_extensions import get_type_hints #from dataclasses import dataclass from pprint import pprint -from typing import List, Optional, TypedDict -import typing -from enum import Enum from typing_extensions import get_type_hints from nautobot_ssot.contrib.types import FieldType from diffsync import Adapter, DiffSyncModel @@ -24,8 +21,14 @@ def _is_sortable_field(attribute_type_hints): def _get_sortable_obj_type(attribute_type_hints): - """""" + """Get the object type of a sortable list based on the type hints.""" + if not hasattr(attribute_type_hints, "__args__"): + return None + if not attribute_type_hints.__args__: + return None attr_type = attribute_type_hints.__args__[0] + if not hasattr(attr_type, "__args__"): + return None attr_type_args = getattr(attr_type, "__args__") if attr_type_args: return attr_type_args[0] @@ -33,7 +36,7 @@ def _get_sortable_obj_type(attribute_type_hints): def _get_sortable_obj_sort_key(sortable_obj_type): - """""" + """Get the sort key from a TypedDict type if set in the metadata.""" content_obj_type_hints = get_type_hints(sortable_obj_type, include_extras=True) for key, value in content_obj_type_hints.items(): if not value.__name__ == "Annotated": @@ -44,8 +47,8 @@ def _get_sortable_obj_sort_key(sortable_obj_type): return None -def _get_sortable_fields_from_model(model): - """""" +def _get_sortable_fields_from_model(model: DiffSyncModel): + """Get a list of sortable fields and their sort key from a DiffSync model.""" sortable_fields = [] model_type_hints = get_type_hints(model, include_extras=True) @@ -62,7 +65,24 @@ def _get_sortable_fields_from_model(model): "sort_key": sort_key, }) return sortable_fields - + + +def _sort_diffsync_object(obj, attribute, key): + """Update the sortable attribute in a DiffSync object.""" + sorted_data = None + if key: + sorted_data = sorted( + getattr(obj, attribute), + key=lambda x: x[key], + ) + else: + sorted_data = sorted( + getattr(obj, attribute) + ) + if sorted_data: + setattr(obj, attribute, sorted_data) + return obj + def sort_relationships(source: Adapter, target: Adapter): """Sort relationships based on the metadata defined in the DiffSync model.""" @@ -85,24 +105,6 @@ def sort_relationships(source: Adapter, target: Adapter): attribute = sortable["attribute"] key = sortable["sort_key"] - #source_items = source.get_all(attribute) - #target_items = target.get_all(attribute) - for adapter in (source, target): for obj in adapter.get_all(attribute): - if key: - sorted_data = sorted( - getattr(obj, attribute), - key=lambda x: x[key], - ) - else: - sorted_data = sorted( - getattr(obj, attribute) - ) - setattr(obj, attribute, sorted_data) - adapter.update(obj) - # end for - - - - \ No newline at end of file + adapter.update(_sort_diffsync_object(obj, attribute, key)) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index b446fd285..1e80219c9 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -1,40 +1,23 @@ from typing import List, Optional -from unittest import skip -from unittest.mock import MagicMock from nautobot_ssot.contrib.types import FieldType -import nautobot.circuits.models as circuits_models -import nautobot.dcim.models as dcim_models -import nautobot.extras.models as extras_models -import nautobot.ipam.models as ipam_models import nautobot.tenancy.models as tenancy_models -from diffsync.exceptions import ObjectNotCreated, ObjectNotDeleted, ObjectNotUpdated -from django.contrib.contenttypes.models import ContentType -#from nautobot.core.testing import TestCase from django.test import TestCase as TestCase -from nautobot.dcim.choices import InterfaceTypeChoices from typing_extensions import Annotated, TypedDict from typing_extensions import get_type_hints from nautobot_ssot.contrib.sorting import ( - _find_list_sort_key, - _get_sortable_list_type_from_annotations, - _sort_top_level_model_attributes, _get_sortable_fields_from_model, _is_sortable_field, _get_sortable_obj_type, _get_sortable_obj_sort_key, sort_relationships, + _sort_diffsync_object, ) -from nautobot_ssot.contrib import ( - CustomFieldAnnotation, - CustomRelationshipAnnotation, - NautobotAdapter, - NautobotModel, - RelationshipSideEnum, -) +from nautobot_ssot.contrib import NautobotModel + class BasicTagDict(TypedDict): """Many-to-many relationship typed dict explaining which fields are interesting.""" @@ -48,7 +31,7 @@ class TagDict(TypedDict): id: int name: Annotated[str, FieldType.SORT_BY] - description: Optional[str] + description: Optional[str] = "" class BasicNautobotTenant(NautobotModel): @@ -79,7 +62,7 @@ class NautobotTenant(NautobotModel): tags: Annotated[List[TagDict], FieldType.SORTED_FIELD] = [] -class TestCaseIsSortableField(TestCase): +class TestCaseIsSortableFieldFunction(TestCase): """""" @classmethod @@ -95,7 +78,7 @@ def test_sortable_field(self): self.assertTrue(test) -class TestGetSortKey(TestCase): +class TestGetSortKeyFunction(TestCase): """""" def test_get_sort_key(self): @@ -112,58 +95,69 @@ class TestClass(TypedDict): self.assertIsNone(test) +class TestCaseGetSortableFieldsFromModelFunction(TestCase): + """Tests for `_get_sortable_fields_from_model` function.""" + def test_with_sortable_fields(self): + """Test get sortable fields with one sortable field identified.""" + test = _get_sortable_fields_from_model(NautobotTenant) + self.assertEqual(len(test), 1) -class TestCaseGetSortableFieldsFromModel(TestCase): - """""" - - - - - - - - - - - - - + def test_without_sortable_fields(self): + """Test get sortable fields with no sortable fields identified.""" + test = _get_sortable_fields_from_model(BasicNautobotTenant) + self.assertEqual(len(test), 0) -''' - - -class TestCaseGetSortableEntriesFunction(TestCase): +class TestSortDiffSyncObjectFunction(TestCase): """""" - def test_nonsortable_entry(self): + @classmethod + def setUpTestData(cls): """""" - test = _get_sortable_list_type_from_annotations(BasicNautobotTenant, "description") + cls.obj_1 = NautobotTenant( + name="", + description="DiffSync object with a sortable field.", + tags=[ + TagDict( + id=1, + name="b", + description="", + ), + TagDict( + id=2, + name="a", + description="", + ), + ] + ) + + def test_with_sortable_field(self): + """Test to make sure `_sort_diffsync_object` sorts attribute.""" + self.assertEqual( + self.obj_1.tags[0]["name"], + "b", + msg="List of `TagDict` entries must start with `name='b'` to verify proper sorting." + ) + test = _sort_diffsync_object(self.obj_1, "tags", "name") + self.assertEqual(test.tags[0]["name"], "a") + + +class TestGetSortableObjectTypeFunction(TestCase): + """Tests for `_get_sortable_object_type` function.""" + + def test_get_sortable_object_type(self): + """Test to validate `_get_sortable_obj_type` function returns correct object type.""" + type_hints = get_type_hints(NautobotTenant, include_extras=True) + test = _get_sortable_obj_type(type_hints.get("tags")) + self.assertTrue(test == TagDict) + + def test_get_sortable_object_type(self): + """Test to validate `_get_sortable_obj_type` function returns None.""" + type_hints = get_type_hints(BasicNautobotTenant, include_extras=True) + test = _get_sortable_obj_type(type_hints["tags"]) self.assertIsNone(test) - def test_sortable_entry_without_annotation(self): - """""" - test = _get_sortable_list_type_from_annotations(BasicNautobotTenant, "tags") - self.assertEqual(test, BasicTagDict) - - def test_sortable_entry_with_annotation(self): - """""" - test = _get_sortable_list_type_from_annotations(NautobotTenant, "tags") - self.assertEqual(test, TagDict) - -class TestCaseGetSortKeyFunction(TestCase): +class TestSortRelationships(TestCase): """""" - - def test_without_sort_key_annotation(self): - """""" - test = _find_list_sort_key(BasicTagDict) - self.assertIsNone(test) - - def test_with_sort_key_annotation(self): - """""" - test = _find_list_sort_key(TagDict) - self.assertEqual(test, "name") - -''' From a080d15808fe22a821058b84719beac6b884b22a Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 12:27:01 -0600 Subject: [PATCH 15/29] Updates --- nautobot_ssot/contrib/adapter.py | 40 +-------------------- nautobot_ssot/contrib/sorting.py | 28 +++++++-------- nautobot_ssot/contrib/types.py | 10 ++++++ nautobot_ssot/tests/contrib/test_sorting.py | 29 ++++++++------- nautobot_ssot/tests/test_contrib_adapter.py | 3 +- 5 files changed, 38 insertions(+), 72 deletions(-) diff --git a/nautobot_ssot/contrib/adapter.py b/nautobot_ssot/contrib/adapter.py index 8a069df46..54757ffce 100644 --- a/nautobot_ssot/contrib/adapter.py +++ b/nautobot_ssot/contrib/adapter.py @@ -34,44 +34,6 @@ ParameterSet = FrozenSet[Tuple[str, Hashable]] -def sort_relationships(diffsync: DiffSync): - """Helper function for SSoT adapters for sorting relationships entries to avoid false updates actions. - - This function checks the `sorted_relationships` attribute in the DiffSync object/adpater. If present, it will - loop through all entries of the attribute and sort the objects accordingly. - - The `sorted_relationships` should be a list or tuple of lists/tuples. Each entry must have three strings: - - Name of the DiffSync model with attribute to be sorted - - Name of the attribute to be sorted - - Name of the key within the attribute to be sorted by - - NOTE: The listed attribute MUST be a list of dictionaries indicating many-to-many relationships on either side - or a one-to-many relationship from the many side. - """ - if not hasattr(diffsync, "sorted_relationships"): - # Nothing to do - return - if not isinstance(diffsync, DiffSync): - raise TypeError("Parameter for `sort_relationships()` must be of type DiffSync.") - - for entry in diffsync.sorted_relationships: - - if not isinstance(entry, tuple) and not isinstance(entry, list): - raise TypeError(f"Invalid type: {type(entry)}. Valid types include tuples or lists.") - - obj_name = entry[0] - attr_name = entry[1] - sort_by_key = entry[2] - - for obj in diffsync.get_all(obj_name): - sorted_data = sorted( - getattr(obj, attr_name), - key=lambda x: x[sort_by_key], # pylint: disable=cell-var-from-loop - ) - setattr(obj, attr_name, sorted_data) - diffsync.update(obj) - - class NautobotAdapter(DiffSync): """ Adapter for loading data from Nautobot through the ORM. @@ -213,7 +175,7 @@ def load(self): # for this specific model class as well as its children without returning anything. self._load_objects(diffsync_model) - sort_relationships(self) + # sort_relationships(self) def _get_diffsync_class(self, model_name): """Given a model name, return the diffsync class.""" diff --git a/nautobot_ssot/contrib/sorting.py b/nautobot_ssot/contrib/sorting.py index a72a032d7..88721f2b3 100644 --- a/nautobot_ssot/contrib/sorting.py +++ b/nautobot_ssot/contrib/sorting.py @@ -1,18 +1,14 @@ -"""""" +"""Functions for sorting DiffSync model lists ensuring they are sorted to prevent false actions.""" -from typing import Annotated -#from typing_extensions import get_type_hints -#from dataclasses import dataclass -from pprint import pprint +from diffsync import Adapter, DiffSyncModel from typing_extensions import get_type_hints + from nautobot_ssot.contrib.types import FieldType -from diffsync import Adapter, DiffSyncModel def _is_sortable_field(attribute_type_hints): """Checks type hints to verify if field labled as sortable or not.""" - if attribute_type_hints.__name__ != "Annotated" or \ - not hasattr(attribute_type_hints, "__metadata__"): + if attribute_type_hints.__name__ != "Annotated" or not hasattr(attribute_type_hints, "__metadata__"): return False for metadata in attribute_type_hints.__metadata__: if metadata == FieldType.SORTED_FIELD: @@ -56,14 +52,16 @@ def _get_sortable_fields_from_model(model: DiffSyncModel): attribute_type_hints = model_type_hints.get(attribute_name) if not _is_sortable_field(attribute_type_hints): continue - + sortable_obj_type = _get_sortable_obj_type(attribute_type_hints) sort_key = _get_sortable_obj_sort_key(sortable_obj_type) - sortable_fields.append({ - "attribute": attribute_name, - "sort_key": sort_key, - }) + sortable_fields.append( + { + "attribute": attribute_name, + "sort_key": sort_key, + } + ) return sortable_fields @@ -76,9 +74,7 @@ def _sort_diffsync_object(obj, attribute, key): key=lambda x: x[key], ) else: - sorted_data = sorted( - getattr(obj, attribute) - ) + sorted_data = sorted(getattr(obj, attribute)) if sorted_data: setattr(obj, attribute, sorted_data) return obj diff --git a/nautobot_ssot/contrib/types.py b/nautobot_ssot/contrib/types.py index 36d2cbc8e..7c03bb1bc 100644 --- a/nautobot_ssot/contrib/types.py +++ b/nautobot_ssot/contrib/types.py @@ -8,6 +8,16 @@ from typing import Optional +class FieldType(Enum): + """Enum to specify field types for DiffSynModels and TypedDicts.""" + + # Indicates if a model field is to be sorted + SORTED_FIELD = 1 + + # Indicates what field to sort by for lists of dictionaries + SORT_BY = 2 + + class RelationshipSideEnum(Enum): """This details which side of a custom relationship the model it's defined on is on.""" diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 1e80219c9..eeb88a79f 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -1,22 +1,20 @@ +"""Unit tests for contrib sorting.""" from typing import List, Optional -from nautobot_ssot.contrib.types import FieldType import nautobot.tenancy.models as tenancy_models from django.test import TestCase as TestCase -from typing_extensions import Annotated, TypedDict -from typing_extensions import get_type_hints +from typing_extensions import Annotated, TypedDict, get_type_hints +from nautobot_ssot.contrib import NautobotModel from nautobot_ssot.contrib.sorting import ( _get_sortable_fields_from_model, - _is_sortable_field, - _get_sortable_obj_type, _get_sortable_obj_sort_key, - sort_relationships, + _get_sortable_obj_type, + _is_sortable_field, _sort_diffsync_object, ) - -from nautobot_ssot.contrib import NautobotModel +from nautobot_ssot.contrib.types import FieldType class BasicTagDict(TypedDict): @@ -63,7 +61,7 @@ class NautobotTenant(NautobotModel): class TestCaseIsSortableFieldFunction(TestCase): - """""" + """Tests for `_is_sortable_field` function.""" @classmethod def setUpTestData(cls): @@ -79,7 +77,7 @@ def test_sortable_field(self): class TestGetSortKeyFunction(TestCase): - """""" + """Tests for `_get_sortable_obj_key` function.""" def test_get_sort_key(self): test = _get_sortable_obj_sort_key(TagDict) @@ -87,9 +85,10 @@ def test_get_sort_key(self): def test_no_sort_key(self): """""" + class TestClass(TypedDict): id: str - name:str + name: str test = _get_sortable_obj_sort_key(TestClass) self.assertIsNone(test) @@ -110,7 +109,7 @@ def test_without_sortable_fields(self): class TestSortDiffSyncObjectFunction(TestCase): - """""" + """Tests for `_sort_diffsync_object` function.""" @classmethod def setUpTestData(cls): @@ -129,7 +128,7 @@ def setUpTestData(cls): name="a", description="", ), - ] + ], ) def test_with_sortable_field(self): @@ -137,7 +136,7 @@ def test_with_sortable_field(self): self.assertEqual( self.obj_1.tags[0]["name"], "b", - msg="List of `TagDict` entries must start with `name='b'` to verify proper sorting." + msg="List of `TagDict` entries must start with `name='b'` to verify proper sorting.", ) test = _sort_diffsync_object(self.obj_1, "tags", "name") self.assertEqual(test.tags[0]["name"], "a") @@ -152,7 +151,7 @@ def test_get_sortable_object_type(self): test = _get_sortable_obj_type(type_hints.get("tags")) self.assertTrue(test == TagDict) - def test_get_sortable_object_type(self): + def test_get_nonsortable_object_type(self): """Test to validate `_get_sortable_obj_type` function returns None.""" type_hints = get_type_hints(BasicNautobotTenant, include_extras=True) test = _get_sortable_obj_type(type_hints["tags"]) diff --git a/nautobot_ssot/tests/test_contrib_adapter.py b/nautobot_ssot/tests/test_contrib_adapter.py index 9d787fbf6..7ce5856d9 100644 --- a/nautobot_ssot/tests/test_contrib_adapter.py +++ b/nautobot_ssot/tests/test_contrib_adapter.py @@ -18,8 +18,7 @@ from nautobot.tenancy import models as tenancy_models from typing_extensions import Annotated, TypedDict -from nautobot_ssot.contrib import NautobotAdapter, NautobotModel, CustomFieldAnnotation, sort_relationships -from nautobot_ssot.contrib import CustomFieldAnnotation, NautobotAdapter, NautobotModel +from nautobot_ssot.contrib import CustomFieldAnnotation, NautobotAdapter, NautobotModel, sort_relationships from nautobot_ssot.tests.contrib_base_classes import ( NautobotCable, NautobotDevice, From d242f4dfd369bb8ee0e196a1313c60f4435e5ae8 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 12:35:35 -0600 Subject: [PATCH 16/29] remove missing import --- nautobot_ssot/contrib/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nautobot_ssot/contrib/__init__.py b/nautobot_ssot/contrib/__init__.py index c322a3a77..6022ee267 100644 --- a/nautobot_ssot/contrib/__init__.py +++ b/nautobot_ssot/contrib/__init__.py @@ -1,6 +1,6 @@ """SSoT Contrib.""" -from nautobot_ssot.contrib.adapter import NautobotAdapter, sort_relationships +from nautobot_ssot.contrib.adapter import NautobotAdapter from nautobot_ssot.contrib.model import NautobotModel from nautobot_ssot.contrib.types import ( CustomFieldAnnotation, From 58361855852db8d3b3fa39c03f61f166f1ad7b72 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 13:30:31 -0600 Subject: [PATCH 17/29] updates --- nautobot_ssot/contrib/__init__.py | 1 - nautobot_ssot/contrib/sorting.py | 22 +++++------ nautobot_ssot/tests/contrib/test_sorting.py | 28 +++++++------- nautobot_ssot/tests/test_contrib_adapter.py | 41 +-------------------- 4 files changed, 26 insertions(+), 66 deletions(-) diff --git a/nautobot_ssot/contrib/__init__.py b/nautobot_ssot/contrib/__init__.py index 6022ee267..1ace601c8 100644 --- a/nautobot_ssot/contrib/__init__.py +++ b/nautobot_ssot/contrib/__init__.py @@ -14,5 +14,4 @@ "NautobotAdapter", "NautobotModel", "RelationshipSideEnum", - "sort_relationships", ) diff --git a/nautobot_ssot/contrib/sorting.py b/nautobot_ssot/contrib/sorting.py index 88721f2b3..794ba5a81 100644 --- a/nautobot_ssot/contrib/sorting.py +++ b/nautobot_ssot/contrib/sorting.py @@ -6,7 +6,7 @@ from nautobot_ssot.contrib.types import FieldType -def _is_sortable_field(attribute_type_hints): +def is_sortable_field(attribute_type_hints): """Checks type hints to verify if field labled as sortable or not.""" if attribute_type_hints.__name__ != "Annotated" or not hasattr(attribute_type_hints, "__metadata__"): return False @@ -16,7 +16,7 @@ def _is_sortable_field(attribute_type_hints): return False -def _get_sortable_obj_type(attribute_type_hints): +def get_sortable_obj_type(attribute_type_hints): """Get the object type of a sortable list based on the type hints.""" if not hasattr(attribute_type_hints, "__args__"): return None @@ -31,7 +31,7 @@ def _get_sortable_obj_type(attribute_type_hints): return None -def _get_sortable_obj_sort_key(sortable_obj_type): +def get_sortable_obj_sort_key(sortable_obj_type): """Get the sort key from a TypedDict type if set in the metadata.""" content_obj_type_hints = get_type_hints(sortable_obj_type, include_extras=True) for key, value in content_obj_type_hints.items(): @@ -43,18 +43,18 @@ def _get_sortable_obj_sort_key(sortable_obj_type): return None -def _get_sortable_fields_from_model(model: DiffSyncModel): +def get_sortable_fields_from_model(model: DiffSyncModel): """Get a list of sortable fields and their sort key from a DiffSync model.""" sortable_fields = [] model_type_hints = get_type_hints(model, include_extras=True) - for attribute_name in model._attributes: + for attribute_name in model._attributes: # pylint: disable=protected-access attribute_type_hints = model_type_hints.get(attribute_name) - if not _is_sortable_field(attribute_type_hints): + if not is_sortable_field(attribute_type_hints): continue - sortable_obj_type = _get_sortable_obj_type(attribute_type_hints) - sort_key = _get_sortable_obj_sort_key(sortable_obj_type) + sortable_obj_type = get_sortable_obj_type(attribute_type_hints) + sort_key = get_sortable_obj_sort_key(sortable_obj_type) sortable_fields.append( { @@ -65,7 +65,7 @@ def _get_sortable_fields_from_model(model: DiffSyncModel): return sortable_fields -def _sort_diffsync_object(obj, attribute, key): +def sort_diffsync_object(obj, attribute, key): """Update the sortable attribute in a DiffSync object.""" sorted_data = None if key: @@ -93,7 +93,7 @@ def sort_relationships(source: Adapter, target: Adapter): continue # Get sortable fields from model - sortable_fields = _get_sortable_fields_from_model(target) + sortable_fields = get_sortable_fields_from_model(target) if not sortable_fields: continue @@ -103,4 +103,4 @@ def sort_relationships(source: Adapter, target: Adapter): for adapter in (source, target): for obj in adapter.get_all(attribute): - adapter.update(_sort_diffsync_object(obj, attribute, key)) + adapter.update(sort_diffsync_object(obj, attribute, key)) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index eeb88a79f..beadcf5b5 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -8,11 +8,11 @@ from nautobot_ssot.contrib import NautobotModel from nautobot_ssot.contrib.sorting import ( - _get_sortable_fields_from_model, - _get_sortable_obj_sort_key, - _get_sortable_obj_type, - _is_sortable_field, - _sort_diffsync_object, + get_sortable_fields_from_model, + get_sortable_obj_sort_key, + get_sortable_obj_type, + is_sortable_field, + sort_diffsync_object, ) from nautobot_ssot.contrib.types import FieldType @@ -68,11 +68,11 @@ def setUpTestData(cls): cls.type_hints = get_type_hints(NautobotTenant, include_extras=True) def test_non_sortable_field(self): - test = _is_sortable_field(self.type_hints["name"]) + test = is_sortable_field(self.type_hints["name"]) self.assertFalse(test) def test_sortable_field(self): - test = _is_sortable_field(self.type_hints["tags"]) + test = is_sortable_field(self.type_hints["tags"]) self.assertTrue(test) @@ -80,7 +80,7 @@ class TestGetSortKeyFunction(TestCase): """Tests for `_get_sortable_obj_key` function.""" def test_get_sort_key(self): - test = _get_sortable_obj_sort_key(TagDict) + test = get_sortable_obj_sort_key(TagDict) self.assertEqual(test, "name") def test_no_sort_key(self): @@ -90,7 +90,7 @@ class TestClass(TypedDict): id: str name: str - test = _get_sortable_obj_sort_key(TestClass) + test = get_sortable_obj_sort_key(TestClass) self.assertIsNone(test) @@ -99,12 +99,12 @@ class TestCaseGetSortableFieldsFromModelFunction(TestCase): def test_with_sortable_fields(self): """Test get sortable fields with one sortable field identified.""" - test = _get_sortable_fields_from_model(NautobotTenant) + test = get_sortable_fields_from_model(NautobotTenant) self.assertEqual(len(test), 1) def test_without_sortable_fields(self): """Test get sortable fields with no sortable fields identified.""" - test = _get_sortable_fields_from_model(BasicNautobotTenant) + test = get_sortable_fields_from_model(BasicNautobotTenant) self.assertEqual(len(test), 0) @@ -138,7 +138,7 @@ def test_with_sortable_field(self): "b", msg="List of `TagDict` entries must start with `name='b'` to verify proper sorting.", ) - test = _sort_diffsync_object(self.obj_1, "tags", "name") + test = sort_diffsync_object(self.obj_1, "tags", "name") self.assertEqual(test.tags[0]["name"], "a") @@ -148,13 +148,13 @@ class TestGetSortableObjectTypeFunction(TestCase): def test_get_sortable_object_type(self): """Test to validate `_get_sortable_obj_type` function returns correct object type.""" type_hints = get_type_hints(NautobotTenant, include_extras=True) - test = _get_sortable_obj_type(type_hints.get("tags")) + test = get_sortable_obj_type(type_hints.get("tags")) self.assertTrue(test == TagDict) def test_get_nonsortable_object_type(self): """Test to validate `_get_sortable_obj_type` function returns None.""" type_hints = get_type_hints(BasicNautobotTenant, include_extras=True) - test = _get_sortable_obj_type(type_hints["tags"]) + test = get_sortable_obj_type(type_hints["tags"]) self.assertIsNone(test) diff --git a/nautobot_ssot/tests/test_contrib_adapter.py b/nautobot_ssot/tests/test_contrib_adapter.py index 7ce5856d9..321cd9315 100644 --- a/nautobot_ssot/tests/test_contrib_adapter.py +++ b/nautobot_ssot/tests/test_contrib_adapter.py @@ -18,7 +18,7 @@ from nautobot.tenancy import models as tenancy_models from typing_extensions import Annotated, TypedDict -from nautobot_ssot.contrib import CustomFieldAnnotation, NautobotAdapter, NautobotModel, sort_relationships +from nautobot_ssot.contrib import CustomFieldAnnotation, NautobotAdapter, NautobotModel from nautobot_ssot.tests.contrib_base_classes import ( NautobotCable, NautobotDevice, @@ -368,42 +368,3 @@ class AdapterCustomRelationshipSortingTest(NautobotAdapter): "name", ), ) - - -class TestSortedRelationships(TestCase): - """Tests for `sort_relationships` function.""" - - def setUp(self): - self.adapter = AdapterCustomRelationshipSortingTest(job=MagicMock()) - - self.adapter.add( - self.adapter.tenant( - name="Tenant 1", - tenants=[ - {"name": "C"}, - {"name": "B"}, - {"name": "A"}, - ], - ) - ) - - def test_valid_sorting(self): - """Test to ensure the function properly sorts basic information.""" - # Validate before settings - before = self.adapter.get("tenant", identifier="Tenant 1") - self.assertEqual(before.tenants[0]["name"], "C") - self.assertEqual(before.tenants[1]["name"], "B") - self.assertEqual(before.tenants[2]["name"], "A") - - sort_relationships(self.adapter) - - after = self.adapter.get("tenant", identifier="Tenant 1") - self.assertEqual(after.tenants[0]["name"], "A") - self.assertEqual(after.tenants[1]["name"], "B") - self.assertEqual(after.tenants[2]["name"], "C") - - def test_invalid_type(self): - """Test passing invalid type to function.""" - self.adapter.sorted_relationships = {"Entry 1": "Value 1"} - with self.assertRaises(TypeError): - sort_relationships(self.adapter) From bb62c507851c49321509f47956e1de8bb9a0560b Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 13:39:21 -0600 Subject: [PATCH 18/29] udpate --- nautobot_ssot/tests/contrib/test_sorting.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index beadcf5b5..24bbe64b5 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -3,7 +3,7 @@ from typing import List, Optional import nautobot.tenancy.models as tenancy_models -from django.test import TestCase as TestCase +from django.test import TestCase from typing_extensions import Annotated, TypedDict, get_type_hints from nautobot_ssot.contrib import NautobotModel @@ -84,8 +84,7 @@ def test_get_sort_key(self): self.assertEqual(test, "name") def test_no_sort_key(self): - """""" - + """Test function with no wort key.""" class TestClass(TypedDict): id: str name: str @@ -113,7 +112,6 @@ class TestSortDiffSyncObjectFunction(TestCase): @classmethod def setUpTestData(cls): - """""" cls.obj_1 = NautobotTenant( name="", description="DiffSync object with a sortable field.", @@ -159,4 +157,4 @@ def test_get_nonsortable_object_type(self): class TestSortRelationships(TestCase): - """""" + """Tests for `sort_relationships` function.""" From f326d07089bb1fab005cdef7cb95f06727a2e6db Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 13:42:41 -0600 Subject: [PATCH 19/29] update spacing --- nautobot_ssot/tests/contrib/test_sorting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 24bbe64b5..892e206a0 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -85,6 +85,7 @@ def test_get_sort_key(self): def test_no_sort_key(self): """Test function with no wort key.""" + class TestClass(TypedDict): id: str name: str From ccd7f5f59760935a8c68f445e273589167b624f5 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 14:00:11 -0600 Subject: [PATCH 20/29] udpates --- nautobot_ssot/tests/contrib/__init__.py | 1 + nautobot_ssot/tests/contrib/test_sorting.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nautobot_ssot/tests/contrib/__init__.py b/nautobot_ssot/tests/contrib/__init__.py index e69de29bb..fd7c7fde7 100644 --- a/nautobot_ssot/tests/contrib/__init__.py +++ b/nautobot_ssot/tests/contrib/__init__.py @@ -0,0 +1 @@ +"""Unit tests for contrib.""" diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 892e206a0..372fb7f6b 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -86,7 +86,7 @@ def test_get_sort_key(self): def test_no_sort_key(self): """Test function with no wort key.""" - class TestClass(TypedDict): + class TestClass(TypedDict): # pylint: disable=missing-class-docstring id: str name: str From 44c620ebe5f868c80fb262ab274fc57242618554 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 14:46:46 -0600 Subject: [PATCH 21/29] update --- nautobot_ssot/tests/contrib/test_sorting.py | 33 +++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 372fb7f6b..b0b70b9dc 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -16,15 +16,16 @@ ) from nautobot_ssot.contrib.types import FieldType +from nautobot_ssot.tests.contrib_base_classes import NautobotTenant as BasicNautobotTenant, TagDict as BasicTagDict -class BasicTagDict(TypedDict): - """Many-to-many relationship typed dict explaining which fields are interesting.""" +# class BasicTagDict(TypedDict): +# """Many-to-many relationship typed dict explaining which fields are interesting.""" - id: int - name: str +# id: int +# name: str -class TagDict(TypedDict): +class TagDict(BasicTagDict): """Many-to-many relationship typed dict explaining which fields are interesting.""" id: int @@ -32,21 +33,21 @@ class TagDict(TypedDict): description: Optional[str] = "" -class BasicNautobotTenant(NautobotModel): - """A tenant model for testing the `NautobotModel` base class.""" +# class BasicNautobotTenant(NautobotModel): +# """A tenant model for testing the `NautobotModel` base class.""" - _model = tenancy_models.Tenant - _modelname = "tenant" - _identifiers = ("name",) - _attributes = ("description", "tenant_group__name", "tags") +# _model = tenancy_models.Tenant +# _modelname = "tenant" +# _identifiers = ("name",) +# _attributes = ("description", "tenant_group__name", "tags") - name: str - description: Optional[str] = None - tenant_group__name: Optional[str] = None - tags: List[BasicTagDict] = [] +# name: str +# description: Optional[str] = None +# tenant_group__name: Optional[str] = None +# tags: List[BasicTagDict] = [] -class NautobotTenant(NautobotModel): +class NautobotTenant(BasicNautobotTenant): """A tenant model for testing the `NautobotModel` base class.""" _model = tenancy_models.Tenant From 3638315b23dc1fb940309308b748fd62e12728df Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 14:47:10 -0600 Subject: [PATCH 22/29] remove unused code --- nautobot_ssot/tests/contrib/test_sorting.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index b0b70b9dc..1bd96a0a8 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -6,7 +6,6 @@ from django.test import TestCase from typing_extensions import Annotated, TypedDict, get_type_hints -from nautobot_ssot.contrib import NautobotModel from nautobot_ssot.contrib.sorting import ( get_sortable_fields_from_model, get_sortable_obj_sort_key, @@ -18,12 +17,6 @@ from nautobot_ssot.tests.contrib_base_classes import NautobotTenant as BasicNautobotTenant, TagDict as BasicTagDict -# class BasicTagDict(TypedDict): -# """Many-to-many relationship typed dict explaining which fields are interesting.""" - -# id: int -# name: str - class TagDict(BasicTagDict): """Many-to-many relationship typed dict explaining which fields are interesting.""" @@ -33,20 +26,6 @@ class TagDict(BasicTagDict): description: Optional[str] = "" -# class BasicNautobotTenant(NautobotModel): -# """A tenant model for testing the `NautobotModel` base class.""" - -# _model = tenancy_models.Tenant -# _modelname = "tenant" -# _identifiers = ("name",) -# _attributes = ("description", "tenant_group__name", "tags") - -# name: str -# description: Optional[str] = None -# tenant_group__name: Optional[str] = None -# tags: List[BasicTagDict] = [] - - class NautobotTenant(BasicNautobotTenant): """A tenant model for testing the `NautobotModel` base class.""" From fb18a13de4e689d7bda361d35fb56635f0b15104 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 14:52:15 -0600 Subject: [PATCH 23/29] fix error --- nautobot_ssot/tests/contrib/test_sorting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 1bd96a0a8..45e13dbd6 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -14,8 +14,8 @@ sort_diffsync_object, ) from nautobot_ssot.contrib.types import FieldType - -from nautobot_ssot.tests.contrib_base_classes import NautobotTenant as BasicNautobotTenant, TagDict as BasicTagDict +from nautobot_ssot.tests.contrib_base_classes import NautobotTenant as BasicNautobotTenant +from nautobot_ssot.tests.contrib_base_classes import TagDict as BasicTagDict class TagDict(BasicTagDict): From 6fa35808e071ec9bc0a126ad951dcae892dfffd9 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 15:17:12 -0600 Subject: [PATCH 24/29] fix pylint --- nautobot_ssot/tests/contrib/test_sorting.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 45e13dbd6..26278818a 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -27,16 +27,8 @@ class TagDict(BasicTagDict): class NautobotTenant(BasicNautobotTenant): - """A tenant model for testing the `NautobotModel` base class.""" + """A updated tenant model for testing the `NautobotModel` base class.""" - _model = tenancy_models.Tenant - _modelname = "tenant" - _identifiers = ("name",) - _attributes = ("description", "tenant_group__name", "tags") - - name: str - description: Optional[str] = None - tenant_group__name: Optional[str] = None tags: Annotated[List[TagDict], FieldType.SORTED_FIELD] = [] From 81d1b192e17a4c5d61615b009c9f6a3c41d8af49 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Mon, 30 Dec 2024 15:18:23 -0600 Subject: [PATCH 25/29] ruff fix --- nautobot_ssot/tests/contrib/test_sorting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nautobot_ssot/tests/contrib/test_sorting.py b/nautobot_ssot/tests/contrib/test_sorting.py index 26278818a..e2f147eef 100644 --- a/nautobot_ssot/tests/contrib/test_sorting.py +++ b/nautobot_ssot/tests/contrib/test_sorting.py @@ -2,7 +2,6 @@ from typing import List, Optional -import nautobot.tenancy.models as tenancy_models from django.test import TestCase from typing_extensions import Annotated, TypedDict, get_type_hints From f1811b4bf43624ead01edd35b4fb859828229f9d Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Tue, 31 Dec 2024 09:13:24 -0600 Subject: [PATCH 26/29] Update TypedDicts --- nautobot_ssot/contrib/typeddicts.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/nautobot_ssot/contrib/typeddicts.py b/nautobot_ssot/contrib/typeddicts.py index 832696788..a6b081029 100644 --- a/nautobot_ssot/contrib/typeddicts.py +++ b/nautobot_ssot/contrib/typeddicts.py @@ -1,32 +1,34 @@ """Common TypedDict definitions used in Many-to-Many relationships.""" -from typing import TypedDict +from typing import Annotated, TypedDict + +from nautobot_ssot.contrib.types import FieldType class ContentTypeDict(TypedDict): """TypedDict for Django Content Types.""" app_label: str - model: str + model: Annotated[str, FieldType.SORT_BY] class TagDict(TypedDict): """TypedDict for Nautobot Tags.""" - name: str + name: Annotated[str, FieldType.SORT_BY] class LocationDict(TypedDict): """TypedDict for DCIM Locations.""" - name: str + name: Annotated[str, FieldType.SORT_BY] parent__name: str class DeviceDict(TypedDict): """TypedDict for DCIM Devices.""" - name: str + name: Annotated[str, FieldType.SORT_BY] location__name: str tenant__name: str rack__name: str @@ -40,14 +42,14 @@ class DeviceDict(TypedDict): class InterfaceDict(TypedDict): """TypedDict for DCIM INterfaces.""" - name: str + name: Annotated[str, FieldType.SORT_BY] device__name: str class PrefixDict(TypedDict): """TypedDict for IPAM Prefixes.""" - network: str + network: Annotated[str, FieldType.SORT_BY] prefix_length: int namespace__name: str @@ -55,20 +57,20 @@ class PrefixDict(TypedDict): class VLANDict(TypedDict): """TypedDict for IPAM VLANs.""" - vid: int + vid: Annotated[int, FieldType.SORT_BY] vlan_group__name: str class IPAddressDict(TypedDict): """TypedDict for IPAM IP Address.""" - host: str + host: Annotated[str, FieldType.SORT_BY] prefix_length: int class VirtualMachineDict(TypedDict): """TypedDict for IPAM .""" - name: str + name: Annotated[str, FieldType.SORT_BY] cluster__name: str tenant__name: str From 720f8825fbb3461938a86781042586a683ded8c7 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Tue, 31 Dec 2024 12:57:10 -0600 Subject: [PATCH 27/29] fix error in tests --- nautobot_ssot/contrib/sorting.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nautobot_ssot/contrib/sorting.py b/nautobot_ssot/contrib/sorting.py index 794ba5a81..3d3a8be0e 100644 --- a/nautobot_ssot/contrib/sorting.py +++ b/nautobot_ssot/contrib/sorting.py @@ -82,11 +82,10 @@ def sort_diffsync_object(obj, attribute, key): def sort_relationships(source: Adapter, target: Adapter): """Sort relationships based on the metadata defined in the DiffSync model.""" - if not isinstance(source, Adapter) or not isinstance(target, Adapter): - raise TypeError("Parameters for `sort_relationships()` must be of type DiffSync.") - + if not source or not target: + return # Loop through Top Level entries - for level in target.top_level: + for level in getattr(target, "top_level", []): # Get the DiffSync Model model = getattr(target, level) if not model: From d951ecd8d64774a28026e85d680307afbb743cd1 Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Tue, 31 Dec 2024 14:28:00 -0600 Subject: [PATCH 28/29] Update Docs --- docs/user/modeling.md | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/user/modeling.md b/docs/user/modeling.md index d106ad312..cb6ee6bc5 100644 --- a/docs/user/modeling.md +++ b/docs/user/modeling.md @@ -167,6 +167,54 @@ Through us defining the model, Nautobot will now be able to dynamically load IP !!! note Although `Interface.ip_addresses` is a generic relation, there is only one content type (i.e. `ipam.ipaddress`) that may be related through this relation, therefore we don't have to specific this in any way. +## Sorting Relationships + +One-to-many (from the one side) and many-to-many relationships are typically stored as lists in the DiffSync object. A problem with false updates during the diff arises when the data source and target load these lists in a different order from each other. + +### The DiffSync Model + +The first step is to add the appropriate metadata to the attributes in the DiffSync Model using annotations. The `FieldType` enum is used to identify what fields in the model are to be sorted. + +```python +from typing import Annotated, List + +from nautobot.dcim.models import Location +from nautobot_ssot.contrib import NautobotModel +from nautobot_ssot.contrib.types import FieldType + + +class TagDict(TypedDict): + name: str + color: str + + +class LocationModel(NautobotModel): + _model = Location + _modelname = "location" + _identifiers = ("name",) + _attributes("tags",) + + name: str + tags: Annotated[List[TagDict], FieldType.SORTED_FIELD] +``` + +A model attribute will not be sorted without the `FieldType.SORTED_FIELD` in the annotation. For lists of simple objects (such as strings, integers, etc.), no additional configuration is required. The sorting process is done automatically. + +### Sorting Dictionaries + +Sorting lists of more complex objects requires an additional configuration to properly sort. In the case of dictionaries, you must specify which key the list of dictionaries should be sorted by. + +The existing DiffSync process uses lists of `TypedDict` objects to represent some one-to-many and many-to-many relationships. Looking at the example from above, we can see that the `TagDict` class doesn't have any specification as to which key entry the list of `TagDict` objects should be sorted by. We'll update the class by adding the `FieldType.SORT_BY` marker as follows: + +```python +... + +class TagDict(TypedDict): + name: Annotated[str, FieldType.SORT_BY] + color: str + +... +``` ## Filtering Objects Loaded From Nautobot From 443ddd6f0d0eef3166785143d69d5de41a48ef2c Mon Sep 17 00:00:00 2001 From: Andrew Turner Date: Tue, 31 Dec 2024 14:28:07 -0600 Subject: [PATCH 29/29] add sorting to base job --- nautobot_ssot/jobs/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nautobot_ssot/jobs/base.py b/nautobot_ssot/jobs/base.py index a2d775cf4..d7a77415f 100644 --- a/nautobot_ssot/jobs/base.py +++ b/nautobot_ssot/jobs/base.py @@ -17,6 +17,7 @@ from nautobot.extras.jobs import BooleanVar, DryRunVar, Job from nautobot_ssot.choices import SyncLogEntryActionChoices +from nautobot_ssot.contrib.sorting import sort_relationships from nautobot_ssot.models import BaseModel, Sync, SyncLogEntry DataMapping = namedtuple("DataMapping", ["source_name", "source_url", "target_name", "target_url"]) @@ -180,6 +181,9 @@ def record_memory_trace(step: str): if memory_profiling: record_memory_trace("target_load") + # Sorting relationships must be done before calculating diffs. + sort_relationships(self.source_adapter, self.target_adapter) + self.logger.info("Calculating diffs...") self.calculate_diff() calculate_diff_time = datetime.now()