diff --git a/diffsync/__init__.py b/diffsync/__init__.py index acaf732..e2484c5 100644 --- a/diffsync/__init__.py +++ b/diffsync/__init__.py @@ -65,7 +65,7 @@ class DiffSyncModel(BaseModel): This class has several underscore-prefixed class variables that subclasses should set as desired; see below. - NOTE: The groupings _identifiers, _attributes, and _children are mutually exclusive; any given field name can + NOTE: The groupings _identifiers and _attributes are mutually exclusive; any given field name can be included in **at most** one of these three tuples. """ @@ -91,22 +91,13 @@ class DiffSyncModel(BaseModel): _attributes: ClassVar[Tuple[str, ...]] = () """Optional: list of additional model fields (beyond those in `_identifiers`) that are relevant to this model. - Only the fields in `_attributes` (as well as any `_children` fields, see below) will be considered - for the purposes of Diff calculation. + Only the fields in `_attributes` will be considered for the purposes of Diff calculation. A model may define additional fields (not included in `_attributes`) for its internal use; a common example would be a locally significant database primary key or id value. Note: inclusion in `_attributes` is mutually exclusive from inclusion in `_identifiers`; a field cannot be in both! """ - _children: ClassVar[Dict[str, str]] = {} - """Optional: dict of `{_modelname: field_name}` entries describing how to store "child" models in this model. - - When calculating a Diff or performing a sync, DiffSync will automatically recurse into these child models. - - Note: inclusion in `_children` is mutually exclusive from inclusion in `_identifiers` or `_attributes`. - """ - model_flags: DiffSyncModelFlags = DiffSyncModelFlags.NONE """Optional: any non-default behavioral flags for this DiffSyncModel. @@ -141,20 +132,11 @@ def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: for attr in cls._attributes: if attr not in cls.model_fields: raise AttributeError(f"_attributes {cls._attributes} references missing or un-annotated attr {attr}") - for attr in cls._children.values(): - if attr not in cls.model_fields: - raise AttributeError(f"_children {cls._children} references missing or un-annotated attr {attr}") - # Any given field can only be in one of (_identifiers, _attributes, _children) + # Any given field can only be in one of (_identifiers, _attributes) id_attr_overlap = set(cls._identifiers).intersection(cls._attributes) if id_attr_overlap: raise AttributeError(f"Fields {id_attr_overlap} are included in both _identifiers and _attributes.") - id_child_overlap = set(cls._identifiers).intersection(cls._children.values()) - if id_child_overlap: - raise AttributeError(f"Fields {id_child_overlap} are included in both _identifiers and _children.") - attr_child_overlap = set(cls._attributes).intersection(cls._children.values()) - if attr_child_overlap: - raise AttributeError(f"Fields {attr_child_overlap} are included in both _attributes and _children.") def __repr__(self) -> str: return f'{self.get_type()} "{self.get_unique_id()}"' @@ -176,24 +158,10 @@ def json(self, **kwargs: Any) -> StrType: kwargs["exclude_defaults"] = True return super().model_dump_json(**kwargs) - def str(self, include_children: bool = True, indent: int = 0) -> StrType: - """Build a detailed string representation of this DiffSyncModel and optionally its children.""" + def str(self, indent: int = 0) -> StrType: + """Build a detailed string representation of this DiffSyncModel.""" margin = " " * indent output = f"{margin}{self.get_type()}: {self.get_unique_id()}: {self.get_attrs()}" - for modelname, fieldname in self._children.items(): - output += f"\n{margin} {fieldname}" - child_ids = getattr(self, fieldname) - if not child_ids: - output += ": []" - elif not self.adapter or not include_children: - output += f": {child_ids}" - else: - for child_id in child_ids: - try: - child = self.adapter.get(modelname, child_id) - output += "\n" + child.str(include_children=include_children, indent=indent + 4) - except ObjectNotFound: - output += f"\n{margin} {child_id} (ERROR: details unavailable)" return output def set_status(self, status: DiffSyncStatus, message: StrType = "") -> None: @@ -320,11 +288,6 @@ def create_unique_id(cls, **identifiers: Dict[StrType, Any]) -> StrType: """ return "__".join(str(identifiers[key]) for key in cls._identifiers) - @classmethod - def get_children_mapping(cls) -> Dict[StrType, StrType]: - """Get the mapping of types to fieldnames for child models of this model.""" - return cls._children - def get_identifiers(self) -> Dict: """Get a dict of all identifiers (primary keys) and their values for this object. @@ -373,56 +336,6 @@ def get_status(self) -> Tuple[DiffSyncStatus, StrType]: """Get the status of the last create/update/delete operation on this object, and any associated message.""" return self._status, self._status_message - def add_child(self, child: "DiffSyncModel") -> None: - """Add a child reference to an object. - - The child object isn't stored, only its unique id. - The name of the target attribute is defined in `_children` per object type - - Raises: - ObjectStoreWrongType: if the type is not part of `_children` - ObjectAlreadyExists: if the unique id is already stored - """ - child_type = child.get_type() - - if child_type not in self._children: - raise ObjectStoreWrongType( - f"Unable to store {child_type} as a child of {self.get_type()}; " - f"valid types are {sorted(self._children.keys())}" - ) - - attr_name = self._children[child_type] - childs = getattr(self, attr_name) - if child.get_unique_id() in childs: - raise ObjectAlreadyExists( - f"Already storing a {child_type} with unique_id {child.get_unique_id()}", - child, - ) - childs.append(child.get_unique_id()) - - def remove_child(self, child: "DiffSyncModel") -> None: - """Remove a child reference from an object. - - The name of the storage attribute is defined in `_children` per object type. - - Raises: - ObjectStoreWrongType: if the child model type is not part of `_children` - ObjectNotFound: if the child wasn't previously present. - """ - child_type = child.get_type() - - if child_type not in self._children: - raise ObjectStoreWrongType( - f"Unable to find and delete {child_type} as a child of {self.get_type()}; " - f"valid types are {sorted(self._children.keys())}" - ) - - attr_name = self._children[child_type] - childs = getattr(self, attr_name) - if child.get_unique_id() not in childs: - raise ObjectNotFound(f"{child} was not found as a child in {attr_name}") - childs.remove(child.get_unique_id()) - class Adapter: # pylint: disable=too-many-public-methods """Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance.""" @@ -788,12 +701,6 @@ def get_tree_traversal(cls, as_dict: bool = False) -> Union[StrType, Dict]: model_obj = getattr(cls, key) if not get_path(output_dict, key): set_key(output_dict, [key]) - if hasattr(model_obj, "_children"): - children = getattr(model_obj, "_children") - for child_key in list(children.keys()): - path = get_path(output_dict, key) or [key] - path.append(child_key) - set_key(output_dict, path) if as_dict: return output_dict return tree_string(output_dict, cls.__name__) @@ -820,17 +727,16 @@ def update(self, obj: DiffSyncModel) -> None: """ return self.store.update(obj=obj) - def remove(self, obj: DiffSyncModel, remove_children: bool = False) -> None: + def remove(self, obj: DiffSyncModel) -> None: """Remove a DiffSyncModel object from the store. Args: obj: object to remove - remove_children: If True, also recursively remove any children of this object Raises: ObjectNotFound: if the object is not present """ - return self.store.remove(obj=obj, remove_children=remove_children) + return self.store.remove(obj=obj) def get_or_instantiate( self, model: Type[DiffSyncModel], ids: Dict, attrs: Optional[Dict] = None diff --git a/diffsync/diff.py b/diffsync/diff.py index c85117a..be43927 100644 --- a/diffsync/diff.py +++ b/diffsync/diff.py @@ -77,7 +77,7 @@ def has_diffs(self) -> bool: """ for group in self.groups(): for child in self.children[group].values(): - if child.has_diffs(include_children=True): + if child.has_diffs(): return True return False @@ -138,7 +138,7 @@ def str(self, indent: int = 0) -> StrType: for group in self.groups(): group_heading_added = False for child in self.children[group].values(): - if child.has_diffs(include_children=True): + if child.has_diffs(): if not group_heading_added: output.append(f"{margin}{group}") group_heading_added = True @@ -152,7 +152,7 @@ def dict(self) -> Dict[StrType, Dict[StrType, Dict]]: """Build a dictionary representation of this Diff.""" result = OrderedDefaultDict[str, Dict](dict) for child in self.get_children(): - if child.has_diffs(include_children=True): + if child.has_diffs(): result[child.type][child.name] = child.dict() return dict(result) @@ -305,23 +305,12 @@ def get_attrs_diffs(self) -> Dict[StrType, Dict[StrType, Any]]: return {"+": {key: self.source_attrs[key] for key in self.get_attrs_keys()}} return {} - def add_child(self, element: "DiffElement") -> None: - """Attach a child object of type DiffElement. - - Childs are saved in a Diff object and are organized by type and name. - """ - self.child_diff.add(element) - def get_children(self) -> Iterator["DiffElement"]: """Iterate over all child DiffElements of this one.""" yield from self.child_diff.get_children() - def has_diffs(self, include_children: bool = True) -> bool: - """Check whether this element (or optionally any of its children) has some diffs. - - Args: - include_children: If True, recursively check children for diffs as well. - """ + def has_diffs(self) -> bool: + """Check whether this element (or optionally any of its children) has some diffs.""" if (self.source_attrs is not None and self.dest_attrs is None) or ( self.source_attrs is None and self.dest_attrs is not None ): @@ -331,10 +320,6 @@ def has_diffs(self, include_children: bool = True) -> bool: if self.source_attrs.get(attr_key) != self.dest_attrs.get(attr_key): return True - if include_children: - if self.child_diff.has_diffs(): - return True - return False def summary(self) -> Dict[StrType, int]: diff --git a/diffsync/helpers.py b/diffsync/helpers.py index 60c61f2..ffc0024 100644 --- a/diffsync/helpers.py +++ b/diffsync/helpers.py @@ -75,8 +75,6 @@ def calculate_diffs(self) -> Diff: self.diff = self.diff_class() skipped_types = symmetric_difference(self.dst_diffsync.top_level, self.src_diffsync.top_level) - # This won't count everything, since these top-level types may have child types which are - # implicitly also skipped as well, but we don't want to waste too much time on this calculation. for skipped_type in skipped_types: if skipped_type in self.dst_diffsync.top_level: self.incr_models_processed(len(self.dst_diffsync.get_all(skipped_type))) @@ -226,59 +224,8 @@ def diff_object_pair( # pylint: disable=too-many-return-statements self.incr_models_processed(delta) - # Recursively diff the children of src_obj and dst_obj and attach the resulting diffs to the diff_element - self.diff_child_objects(diff_element, src_obj, dst_obj) - return diff_element - def diff_child_objects( - self, - diff_element: DiffElement, - src_obj: Optional["DiffSyncModel"], - dst_obj: Optional["DiffSyncModel"], - ) -> DiffElement: - """For all children of the given DiffSyncModel pair, diff recursively, adding diffs to the given diff_element. - - Helper method to `calculate_diffs`, usually doesn't need to be called directly. - - These helper methods work in a recursive cycle: - diff_object_list -> diff_object_pair -> diff_child_objects -> diff_object_list -> etc. - """ - children_mapping: Dict[str, str] - if src_obj and dst_obj: - # Get the subset of child types common to both src_obj and dst_obj - src_mapping = src_obj.get_children_mapping() - dst_mapping = dst_obj.get_children_mapping() - children_mapping = {} - for child_type, child_fieldname in src_mapping.items(): - if child_type in dst_mapping: - children_mapping[child_type] = child_fieldname - else: - self.incr_models_processed(len(getattr(src_obj, child_fieldname))) - for child_type, child_fieldname in dst_mapping.items(): - if child_type not in src_mapping: - self.incr_models_processed(len(getattr(dst_obj, child_fieldname))) - elif src_obj: - children_mapping = src_obj.get_children_mapping() - elif dst_obj: - children_mapping = dst_obj.get_children_mapping() - else: - raise RuntimeError("Called with neither src_obj nor dest_obj??") - - for child_type, child_fieldname in children_mapping.items(): - # for example, child_type == "device" and child_fieldname == "devices" - - # for example, getattr(src_obj, "devices") --> list of device uids - # --> src_diffsync.get_by_uids(, "device") --> list of device instances - src_objs = self.src_diffsync.get_by_uids(getattr(src_obj, child_fieldname), child_type) if src_obj else [] - dst_objs = self.dst_diffsync.get_by_uids(getattr(dst_obj, child_fieldname), child_type) if dst_obj else [] - - for child_diff_element in self.diff_object_list(src=src_objs, dst=dst_objs): - diff_element.add_child(child_diff_element) - - return diff_element - - class DiffSyncSyncer: # pylint: disable=too-many-instance-attributes """Helper class implementing data synchronization logic for DiffSync. diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index bfe096b..38467aa 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -87,7 +87,6 @@ class Site(DiffSyncModel): _modelname = "site" _identifiers = ("name",) - _children = {"device": "devices"} name: str devices: List = [] @@ -112,7 +111,6 @@ class Device(DiffSyncModel): _modelname = "device" _identifiers = ("name",) _attributes: ClassVar[Tuple[str, ...]] = ("role",) - _children = {"interface": "interfaces"} name: str site_name: Optional[str] = None # note this is not included in _attributes @@ -136,7 +134,7 @@ class Interface(DiffSyncModel): _modelname = "interface" _identifiers = ("device_name", "name") - _shortname = ("name",) + _shortname = ("device_name", "name",) _attributes = ("interface_type", "description") device_name: str @@ -180,7 +178,7 @@ class GenericBackend(Adapter): interface = Interface unused = UnusedModel - top_level = ["site", "unused"] + top_level = ["site", "device", "interface", "unused"] DATA: dict = {} @@ -193,12 +191,10 @@ def load(self): for device_name, device_data in site_data.items(): device = self.device(name=device_name, role=device_data["role"], site_name=site_name) self.add(device) - site.add_child(device) for intf_name, desc in device_data["interfaces"].items(): intf = self.interface(name=intf_name, device_name=device_name, description=desc) self.add(intf) - device.add_child(intf) class SiteA(Site): @@ -253,7 +249,6 @@ def load(self): super().load() person = self.person(name="Glenn Matthews") self.add(person) - self.get("site", "rdu").add_child(person) @pytest.fixture @@ -397,7 +392,6 @@ def load(self): super().load() place = self.place(name="Statue of Liberty") self.add(place) - self.get("site", "nyc").add_child(place) @pytest.fixture @@ -433,16 +427,16 @@ def diff_with_children(): person_element_2.add_attrs(dest={}) diff.add(person_element_2) - # device_element has no diffs of its own, but has a child intf_element + # device_element has no diffs of its own device_element = DiffElement("device", "device1", {"name": "device1"}) diff.add(device_element) - # intf_element exists in both source and dest as a child of device_element, and has differing attrs + # intf_element exists in both source and dest and has differing attrs intf_element = DiffElement("interface", "eth0", {"device_name": "device1", "name": "eth0"}) source_attrs = {"interface_type": "ethernet", "description": "my interface"} dest_attrs = {"description": "your interface"} intf_element.add_attrs(source=source_attrs, dest=dest_attrs) - device_element.add_child(intf_element) + diff.add(intf_element) # address_element exists in both source and dest but has no diffs address_element = DiffElement("address", "RTP", {"name": "RTP"}) diff --git a/tests/unit/test_diff.py b/tests/unit/test_diff.py index 63c6310..e6082d0 100644 --- a/tests/unit/test_diff.py +++ b/tests/unit/test_diff.py @@ -102,11 +102,9 @@ def test_diff_str_with_diffs(diff_with_children): person person: Jimbo MISSING in dest person: Sully MISSING in source -device - device: device1 - interface - interface: eth0 - description source(my interface) dest(your interface)\ +interface + interface: eth0 + description source(my interface) dest(your interface)\ """ ) @@ -114,11 +112,7 @@ def test_diff_str_with_diffs(diff_with_children): def test_diff_dict_with_diffs(diff_with_children): # Since the address element has no diffs, we don't have any "address" entry in the diff dict: assert diff_with_children.dict() == { - "device": { - "device1": { - "interface": {"eth0": {"+": {"description": "my interface"}, "-": {"description": "your interface"}}} - } - }, + "interface": {"eth0": {"+": {"description": "my interface"}, "-": {"description": "your interface"}}}, "person": {"Jimbo": {"+": {}}, "Sully": {"-": {}}}, } @@ -144,7 +138,7 @@ def order_children_default(cls, children): # Validating default order method diff_a_b = backend_a.diff_from(backend_b, diff_class=MyDiff) children = diff_a_b.get_children() - children_names = [child.name for child in children] + children_names = [child.name for child in children if child.type == "site"] assert children_names == ["atl", "nyc", "rdu", "sfo"] @@ -163,5 +157,5 @@ def order_children_site(cls, children): diff_a_b = backend_a.diff_from(backend_b, diff_class=MyDiff) children = diff_a_b.get_children() - children_names = [child.name for child in children] + children_names = [child.name for child in children if child.type == "site"] assert children_names == ["sfo", "rdu", "nyc", "atl"] diff --git a/tests/unit/test_diff_element.py b/tests/unit/test_diff_element.py index 9d44290..4d69426 100644 --- a/tests/unit/test_diff_element.py +++ b/tests/unit/test_diff_element.py @@ -31,8 +31,6 @@ def test_diff_element_empty(): assert element.dest_attrs is None assert not list(element.get_children()) assert not element.has_diffs() - assert not element.has_diffs(include_children=True) - assert not element.has_diffs(include_children=False) assert element.get_attrs_keys() == [] element2 = DiffElement( @@ -71,8 +69,6 @@ def test_diff_element_attrs(): assert element.source_attrs == source_attrs assert element.has_diffs() - assert element.has_diffs(include_children=True) - assert element.has_diffs(include_children=False) assert element.get_attrs_keys() == source_attrs.keys() dest_attrs = {"description": "your interface"} @@ -81,8 +77,6 @@ def test_diff_element_attrs(): assert element.dest_attrs == dest_attrs assert element.has_diffs() - assert element.has_diffs(include_children=True) - assert element.has_diffs(include_children=False) assert element.get_attrs_keys() == ["description"] # intersection of source_attrs.keys() and dest_attrs.keys() @@ -130,62 +124,3 @@ def test_diff_element_dict_with_diffs_no_attrs(): element.add_attrs(dest={}) assert element.dict() == {"+": {}, "-": {}} - -def test_diff_element_children(): - """Test the basic functionality of the DiffElement class when storing and retrieving child elements.""" - child_element = DiffElement("interface", "eth0", {"device_name": "device1", "name": "eth0"}) - parent_element = DiffElement("device", "device1", {"name": "device1"}) - - parent_element.add_child(child_element) - assert list(parent_element.get_children()) == [child_element] - assert not parent_element.has_diffs() - assert not parent_element.has_diffs(include_children=True) - assert not parent_element.has_diffs(include_children=False) - - source_attrs = {"interface_type": "ethernet", "description": "my interface"} - dest_attrs = {"description": "your interface"} - child_element.add_attrs(source=source_attrs, dest=dest_attrs) - - assert parent_element.has_diffs() - assert parent_element.has_diffs(include_children=True) - assert not parent_element.has_diffs(include_children=False) - - -def test_diff_element_summary_with_child_diffs(diff_element_with_children): - # create interface "lo0" - # delete interface "lo1" - # update device "device1" and interface "eth0" - # no change to interface "lo100" - assert diff_element_with_children.summary() == {"create": 1, "update": 2, "delete": 1, "no-change": 1} - - -def test_diff_element_str_with_child_diffs(diff_element_with_children): - assert ( - diff_element_with_children.str() - == """\ -device: device1 - role source(switch) dest(router) - interface - interface: eth0 - description source(my interface) dest(your interface) - interface: lo0 MISSING in dest - interface: lo1 MISSING in source\ -""" - ) - - -def test_diff_element_dict_with_child_diffs(diff_element_with_children): - assert diff_element_with_children.dict() == { - "-": {"role": "router"}, - "+": {"role": "switch"}, - "interface": { - "eth0": {"-": {"description": "your interface"}, "+": {"description": "my interface"}}, - "lo0": {"+": {}}, - "lo1": {"-": {"description": "Loopback 1"}}, - }, - } - - -def test_diff_element_len_with_child_diffs(diff_element_with_children): - assert len(diff_element_with_children) == 5 - assert len(diff_element_with_children) == sum(count for count in diff_element_with_children.summary().values())