Update docker, add methods to load from dictionary, get tree traversa… (#174)

* Update docker, add methods to load from dictionary, get tree traversal, and update docs

* Apply suggestions from code review

Co-authored-by: Glenn Matthews <glenn.matthews@networktocode.com>

* Additional recommended updates for loop efficiency and using classmethod

* update docs

* mypy feedback

* lock poetry

Co-authored-by: Glenn Matthews <glenn.matthews@networktocode.com>

Author: 2 people

Dockerfile

@@ -2,6 +2,9 @@ ARG PYTHON_VER
 
 FROM python:${PYTHON_VER}-slim
 
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends build-essential
+
 RUN pip install --upgrade pip \
   && pip install poetry
 

diffsync/__init__.py

@@ -20,12 +20,13 @@
 from pydantic import BaseModel, PrivateAttr
 import structlog  # type: ignore
 
-from .diff import Diff
-from .enum import DiffSyncModelFlags, DiffSyncFlags, DiffSyncStatus
-from .exceptions import DiffClassMismatch, ObjectAlreadyExists, ObjectStoreWrongType, ObjectNotFound
-from .helpers import DiffSyncDiffer, DiffSyncSyncer
-from .store import BaseStore
-from .store.local import LocalStore
+from diffsync.diff import Diff
+from diffsync.enum import DiffSyncModelFlags, DiffSyncFlags, DiffSyncStatus
+from diffsync.exceptions import DiffClassMismatch, ObjectAlreadyExists, ObjectStoreWrongType, ObjectNotFound
+from diffsync.helpers import DiffSyncDiffer, DiffSyncSyncer
+from diffsync.store import BaseStore
+from diffsync.store.local import LocalStore
+from diffsync.utils import get_path, set_key, tree_string
 
 
 class DiffSyncModel(BaseModel):
@@ -396,7 +397,7 @@ def remove_child(self, child: "DiffSyncModel"):
         childs.remove(child.get_unique_id())
 
 
-class DiffSync:
+class DiffSync:  # pylint: disable=too-many-public-methods
     """Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance."""
 
     # In any subclass, you would add mapping of names to specific model classes here:
@@ -460,6 +461,26 @@ def __len__(self):
         """Total number of elements stored."""
         return self.store.count()
 
+    @classmethod
+    def _get_initial_value_order(cls) -> List[str]:
+        """Get the initial value order of diffsync object.
+
+        Returns:
+            List of model-referencing attribute names in the order they are initially processed.
+        """
+        if hasattr(cls, "top_level") and isinstance(getattr(cls, "top_level"), list):
+            value_order = cls.top_level.copy()
+        else:
+            value_order = []
+
+        for item in dir(cls):
+            _method = getattr(cls, item)
+            if item in value_order:
+                continue
+            if isclass(_method) and issubclass(_method, DiffSyncModel):
+                value_order.append(item)
+        return value_order
+
     def load(self):
         """Load all desired data from whatever backend data source into this instance."""
         # No-op in this generic class
@@ -489,6 +510,18 @@ def str(self, indent: int = 0) -> str:
                     output += "\n" + model.str(indent=indent + 2)
         return output
 
+    def load_from_dict(self, data: Dict):
+        """The reverse of `dict` method, taking a dictionary and loading into the inventory.
+
+        Args:
+            data: Dictionary in the format that `dict` would export as
+        """
+        value_order = self._get_initial_value_order()
+        for field_name in value_order:
+            model_class = getattr(self, field_name)
+            for values in data.get(field_name, {}).values():
+                self.add(model_class(**values))
+
     # ------------------------------------------------------------------------------
     # Synchronization between DiffSync instances
     # ------------------------------------------------------------------------------
@@ -625,7 +658,6 @@ def get_all_model_names(self):
     def get(
         self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[Text, Mapping]
     ) -> DiffSyncModel:
-
         """Get one object from the data store based on its unique id.
 
         Args:
@@ -638,6 +670,26 @@ def get(
         """
         return self.store.get(model=obj, identifier=identifier)
 
+    def get_or_none(
+        self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]], identifier: Union[Text, Mapping]
+    ) -> Optional[DiffSyncModel]:
+        """Get one object from the data store based on its unique id or get a None
+
+        Args:
+            obj: DiffSyncModel class or instance, or modelname string, that defines the type of the object to retrieve
+            identifier: Unique ID of the object to retrieve, or dict of unique identifier keys/values
+
+        Raises:
+            ValueError: if obj is a str and identifier is a dict (can't convert dict into a uid str without a model class)
+
+        Returns:
+            DiffSyncModel matching provided criteria
+        """
+        try:
+            return self.get(obj, identifier)
+        except ObjectNotFound:
+            return None
+
     def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]) -> List[DiffSyncModel]:
         """Get all objects of a given type.
 
@@ -663,6 +715,32 @@ def get_by_uids(
         """
         return self.store.get_by_uids(uids=uids, model=obj)
 
+    @classmethod
+    def get_tree_traversal(cls, as_dict: bool = False) -> Union[Text, Mapping]:
+        """Get a string describing the tree traversal for the diffsync object.
+
+        Args:
+            as_dict: Whether or not to return as a dictionary
+
+        Returns:
+            A string or dictionary representation of tree
+        """
+        value_order = cls._get_initial_value_order()
+        output_dict: Dict = {}
+        for key in value_order:
+            model_obj = getattr(cls, key)
+            if not get_path(output_dict, key):
+                set_key(output_dict, [key])
+            if hasattr(model_obj, "_children"):
+                children = getattr(model_obj, "_children")
+                for child_key in list(children.keys()):
+                    path = get_path(output_dict, key) or [key]
+                    path.append(child_key)
+                    set_key(output_dict, path)
+        if as_dict:
+            return output_dict
+        return tree_string(output_dict, cls.__name__)
+
     def add(self, obj: DiffSyncModel):
         """Add a DiffSyncModel object to the store.
 

diffsync/utils.py

@@ -16,7 +16,12 @@
 """
 
 from collections import OrderedDict
-from typing import List
+from typing import Iterator, List, Dict, Optional
+
+SPACE = "    "
+BRANCH = "│   "
+TEE = "├── "
+LAST = "└── "
 
 
 def intersection(lst1, lst2) -> List:
@@ -42,3 +47,44 @@ def __missing__(self, key):
         """When trying to access a nonexistent key, initialize the key value based on the internal factory."""
         self[key] = value = self.factory()
         return value
+
+
+# from: https://stackoverflow.com/questions/72618673/list-directory-tree-structure-in-python-from-a-list-of-path-file
+def _tree(data: Dict, prefix: str = "") -> Iterator[str]:
+    """Given a dictionary will yield a visual tree structure.
+
+    A recursive generator, given a dictionary will yield a visual tree structure line by line
+    with each line prefixed by the same characters.
+    """
+    # contents each get pointers that are ├── with a final └── :
+    pointers = [TEE] * (len(data) - 1) + [LAST]
+    for pointer, path in zip(pointers, data):
+        yield prefix + pointer + path
+        if isinstance(data[path], dict):  # extend the prefix and recurse:
+            extension = BRANCH if pointer == TEE else SPACE
+            # i.e. SPACE because LAST, └── , above so no more |
+            yield from _tree(data[path], prefix=prefix + extension)
+
+
+def tree_string(data: Dict, root: str) -> str:
+    """String wrapper around `_tree` function to add header and provide tree view of a dictionary."""
+    return "\n".join([root, *_tree(data)])
+
+
+def set_key(data: Dict, keys: List):
+    """Set a nested dictionary key given a list of keys."""
+    current_level = data
+    for key in keys:
+        current_level = current_level.setdefault(key, {})
+
+
+def get_path(nested_dict: Dict, search_value: str) -> Optional[List]:
+    """Find the path of keys in a dictionary, given a single unique value."""
+    for key in nested_dict.keys():
+        if key == search_value:
+            return [key]
+        if isinstance(nested_dict[key], dict):
+            path = get_path(nested_dict[key], search_value)
+            if path is not None:
+                return [key] + path
+    return None

docs/source/getting_started/01-getting-started.md

@@ -127,6 +127,24 @@ This can be visualized here in the included diagram.
 
 ![Preorder Tree Traversal](../../images/preorder-tree-traversal.drawio.png "Preorder Tree Traversal")
 
+### Mapping Tree Traversal with `get_tree_traversal` method
+
+For your convenience, there is a helper method that will provide a mapping of the order. The `DiffSync.get_tree_traversal()` class method will return a tree-like string, or optionally a dictionary when passing the `as_dict=True` parameter.
+
+```python
+>>> from nautobot_device_onboarding.network_importer.adapters.network_device.adapter import NetworkImporterAdapter
+>>> print(NetworkImporterAdapter.get_tree_traversal())
+NetworkImporterAdapter
+├── status
+├── site
+│   ├── vlan
+│   └── prefix
+└── device
+    └── interface
+        └── ip_address
+>>> 
+```
+
 # Store data in a `DiffSync` object
 
 To add a site to the local cache/store, you need to pass a valid `DiffSyncModel` object to the `add()` function.