Replace sub region with population, add diff and flags

examples/example3/README.md

@@ -4,7 +4,7 @@
 This is a simple example to show how DiffSync can be used to compare and synchronize data with a remote system like via a REST API like Nautobot.
 
 For this example, we have a shared model for Region and Country defined in `models.py`.
-A Country must be associated with a Region and can be part of a Subregion too.
+A country must be part of a region and has an attribute to capture its population.
 
 The comparison and synchronization of dataset is done between a local JSON file and the [public instance of Nautobot](https://demo.nautobot.com).
 
@@ -16,17 +16,26 @@ to use this example you must have some dependencies installed, please make sure
 pip install -r requirements.txt
 ```
 
+## Setup the environment
+
+By default this example will interact with the public sandbox of Nautobot at https://demo.nautobot.com but you can use your own version of Nautobot by providing a new URL and a new API token using the environment variables `NAUTOBOT_URL` & `NAUTOBOT_TOKEN`
+
+```
+export NAUTOBOT_URL = "https://demo.nautobot.com"
+export NAUTOBOT_TOKEN = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+```
+
 ## Try the example
 
 The first time a lot of changes should be reported between Nautobot and the local data because by default the demo instance doesn't have the subregion define.
 After the first sync, the diff should show no difference. 
-At this point, Diffsync will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode.
+At this point, `Diffsync` will be able to identify and fix all changes in Nautobot. You can try to add/update or delete any country in Nautobot and DiffSync will automatically catch it and it will fix it with running in sync mode.
 
 ```
 ### DIFF Compare the data between Nautobot and the local JSON file.
-main.py --diff
+python main.py --diff
 
 ### SYNC Update the list of country in Nautobot.
-main.py --sync
+python main.py --sync
 ```
 

examples/example3/diff.py

@@ -0,0 +1,15 @@
+from diffsync.diff import Diff
+
+
+class AlphabeticalOrderDiff(Diff):
+    """Simple diff to return all children country in alphabetical order."""
+
+    @classmethod
+    def order_children_default(cls, children):
+        """Simple diff to return all children in alphabetical order."""
+        for child_name, child in sorted(children.items()):
+
+            # it's possible to access additional information about the object
+            #  like  child.action can be "update", "create" or "delete"
+
+            yield children[child_name]

examples/example3/local_adapter.py

@@ -45,9 +45,11 @@ def load(self, filename=COUNTRIES_FILE):
             region = self.get(obj=self.region, identifier=slugify(country.get("region")))
 
             name = country.get("country")
-            item = self.country(
-                slug=slugify(name), name=name, subregion=country.get("subregion", None), region=region.slug
-            )
+
+            # The population is store in thousands in the local file so we need to convert it
+            population = int(float(country.get("pop2021")) * 1000)
+
+            item = self.country(slug=slugify(name), name=name, population=population, region=region.slug)
             self.add(item)
 
             region.add_child(item)

examples/example3/main.py

@@ -5,10 +5,12 @@
 import pprint
 
 from diffsync import Diff
+from diffsync.enum import DiffSyncFlags
 from diffsync.logging import enable_console_logging
 
 from local_adapter import LocalAdapter
 from nautobot_adapter import NautobotAdapter
+from diff import AlphabeticalOrderDiff
 
 
 def main():
@@ -26,21 +28,21 @@ def main():
     print("Initializing and loading Local Data ...")
     local = LocalAdapter()
     local.load()
-    # print(local.str())
 
     print("Initializing and loading Nautobot Data ...")
     nautobot = NautobotAdapter()
     nautobot.load()
-    # print(nautobot.str())
+
+    flags = DiffSyncFlags.SKIP_UNMATCHED_DST
 
     if args.diff:
         print("Calculating the Diff between the local adapter and Nautobot ...")
-        diff = nautobot.diff_from(local)
+        diff = nautobot.diff_from(local, flags=flags, diff_class=AlphabeticalOrderDiff)
         print(diff.str())
 
     elif args.sync:
         print("Updating the list of countries in Nautobot ...")
-        nautobot.sync_from(local)
+        nautobot.sync_from(local, flags=flags, diff_class=AlphabeticalOrderDiff)
 
 
 if __name__ == "__main__":

examples/example3/models.py

@@ -21,14 +21,14 @@ class Region(DiffSyncModel):
 class Country(DiffSyncModel):
     """Example model of a Country.
 
-    A must be part of a region and can be also associated with a subregion.
+    A country must be part of a region and has an attribute to capture its population.
     """
 
     _modelname = "country"
     _identifiers = ("slug",)
-    _attributes = ("name", "region", "subregion")
+    _attributes = ("name", "region", "population")
 
     slug: str
     name: str
     region: str
-    subregion: Optional[str]
+    population: Optional[int]

examples/example3/nautobot_adapter.py

@@ -2,110 +2,21 @@
 import pynautobot
 
 from diffsync import DiffSync
-from models import Region, Country
+
+from nautobot_models import NautobotCountry, NautobotRegion
 
 NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com")
 NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
 
-
-class NautobotRegion(Region):
-    """Extend the Region object to store Nautobot specific information.
-
-    Region are represented in Nautobot as a dcim.region object without parent.
-    """
-
-    remote_id: str
-    """Store the nautobot uuid in the object to allow update and delete of existing object."""
-
-
-class NautobotCountry(Country):
-    """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE.
-    
-    Country are represented in Nautobot as a dcim.region object as well but a country must have a parent.
-    Subregion information will be store in the description of the object in Nautobot
-    """
-
-    remote_id: str
-    """Store the nautobot uuid in the object to allow update and delete of existing object."""
-
-    @classmethod
-    def create(cls, diffsync: "DiffSync", ids: dict, attrs: dict):
-        """Create a country object in Nautobot.
-
-        Args:
-            diffsync: The master data store for other DiffSyncModel instances that we might need to reference
-            ids: Dictionary of unique-identifiers needed to create the new object
-            attrs: Dictionary of additional attributes to set on the new object
-
-        Returns:
-            NautobotCountry: DiffSync object newly created
-        """
-
-        # Retrieve the parent region in internal cache to access its UUID
-        #  because the UUID is required to associate the object to its parent region in Nautobot
-        region = diffsync.get(diffsync.region, attrs.get("region"))
-
-        # Create the new country in Nautobot and attach it to its parent
-        try:
-            country = diffsync.nautobot.dcim.regions.create(
-                slug=ids.get("slug"),
-                name=attrs.get("name"),
-                description=attrs.get("subregion", None),
-                parent=region.remote_id,
-            )
-            print(f"Created country : {ids} | {attrs} | {country.id}")
-
-        except pynautobot.core.query.RequestError as exc:
-            print(f"Unable to create country {ids} | {attrs} | {exc}")
-            return None
-
-        # Add the newly created remote_id and create the internal object for this resource.
-        attrs["remote_id"] = country.id
-        item = super().create(ids=ids, diffsync=diffsync, attrs=attrs)
-        return item
-
-    def update(self, attrs: dict):
-        """Update a country object in Nautobot.
-
-        Args:
-            attrs: Dictionary of attributes to update on the object
-
-        Returns:
-            DiffSyncModel: this instance, if all data was successfully updated.
-            None: if data updates failed in such a way that child objects of this model should not be modified.
-
-        Raises:
-            ObjectNotUpdated: if an error occurred.
-        """
-
-        # Retrive the pynautobot object from Nautobot since we only have the UUID internally
-        remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
-
-        # Convert the internal attrs to Nautobot format
-        nautobot_attrs = {}
-        if "subregion" in attrs:
-            nautobot_attrs["description"] = attrs.get("subregion")
-        if "name" in attrs:
-            nautobot_attrs["name"] = attrs.get("name")
-
-        if nautobot_attrs:
-            remote.update(data=nautobot_attrs)
-            print(f"Updated Country {self.slug} | {attrs}")
-
-        return super().update(attrs)
-
-    def delete(self):
-        """Delete a country object in Nautobot.
-
-        Returns:
-            NautobotCountry: DiffSync object
-        """
-        # Retrieve the pynautobot object and delete the object in Nautobot
-        remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
-        remote.delete()
-
-        super().delete()
-        return self
+CUSTOM_FIELDS = [
+    {
+        "name": "country_population",
+        "display": "Population (nbr people)",
+        "content_types": ["dcim.region"],
+        "type": "integer",
+        "description": "Number of inhabitant per country",
+    }
+]
 
 
 class NautobotAdapter(DiffSync):
@@ -132,7 +43,7 @@ def load(self):
 
         # Initialize pynautobot to interact with Nautobot and store it within the adapter
         # to reuse it later
-        self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN,)
+        self.nautobot = pynautobot.api(url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN)
 
         # Pull all regions from Nautobot, which includes all regions and all countries
         regions = self.nautobot.dcim.regions.all()
@@ -142,10 +53,6 @@ def load(self):
             if region.parent:
                 continue
 
-            # We are excluding the networktocode because it's not present in the local file
-            if region.slug == "networktocode":
-                continue
-
             item = self.region(slug=region.slug, name=region.name, remote_id=region.id)
             self.add(item)
 
@@ -160,8 +67,19 @@ def load(self):
                 slug=country.slug,
                 name=country.name,
                 region=parent.slug,
-                subregion=country.description,
+                population=country.custom_fields.get("country_population", None),
                 remote_id=country.id,
             )
             self.add(item)
             parent.add_child(item)
+
+    def sync_from(self, *args, **kwargs):
+        """Sync the data with Nautobot but first ensure that all the required Custom fields are present in Nautobot."""
+
+        # Check if all required custom fields exist, create them if they don't
+        for custom_field in CUSTOM_FIELDS:
+            nb_cfs = self.cfs = self.nautobot.extras.custom_fields.filter(name=custom_field.get("name"))
+            if not nb_cfs:
+                self.nautobot.extras.custom_fields.create(**custom_field)
+
+        super().sync_from(*args, **kwargs)

examples/example3/nautobot_models.py

@@ -0,0 +1,104 @@
+import os
+import pynautobot
+
+
+from diffsync import DiffSync
+from models import Region, Country
+
+
+class NautobotRegion(Region):
+    """Extend the Region object to store Nautobot specific information.
+
+    Region are represented in Nautobot as a dcim.region object without parent.
+    """
+
+    remote_id: str
+    """Store the nautobot uuid in the object to allow update and delete of existing object."""
+
+
+class NautobotCountry(Country):
+    """Extend the Country to manage Country in Nautobot. CREATE/UPDATE/DELETE.
+    
+    Country are represented in Nautobot as a dcim.region object as well but a country must have a parent.
+    Subregion information will be store in the description of the object in Nautobot
+    """
+
+    remote_id: str
+    """Store the nautobot uuid in the object to allow update and delete of existing object."""
+
+    @classmethod
+    def create(cls, diffsync: DiffSync, ids: dict, attrs: dict):
+        """Create a country object in Nautobot.
+
+        Args:
+            diffsync: The master data store for other DiffSyncModel instances that we might need to reference
+            ids: Dictionary of unique-identifiers needed to create the new object
+            attrs: Dictionary of additional attributes to set on the new object
+
+        Returns:
+            NautobotCountry: DiffSync object newly created
+        """
+
+        # Retrieve the parent region in internal cache to access its UUID
+        #  because the UUID is required to associate the object to its parent region in Nautobot
+        region = diffsync.get(diffsync.region, attrs.get("region"))
+
+        # Create the new country in Nautobot and attach it to its parent
+        try:
+            country = diffsync.nautobot.dcim.regions.create(
+                slug=ids.get("slug"),
+                name=attrs.get("name"),
+                custom_fields=dict(population=attrs.get("population")),
+                parent=region.remote_id,
+            )
+            print(f"Created country : {ids} | {attrs} | {country.id}")
+
+        except pynautobot.core.query.RequestError as exc:
+            print(f"Unable to create country {ids} | {attrs} | {exc}")
+            return None
+
+        # Add the newly created remote_id and create the internal object for this resource.
+        attrs["remote_id"] = country.id
+        item = super().create(ids=ids, diffsync=diffsync, attrs=attrs)
+        return item
+
+    def update(self, attrs: dict):
+        """Update a country object in Nautobot.
+
+        Args:
+            attrs: Dictionary of attributes to update on the object
+
+        Returns:
+            DiffSyncModel: this instance, if all data was successfully updated.
+            None: if data updates failed in such a way that child objects of this model should not be modified.
+
+        Raises:
+            ObjectNotUpdated: if an error occurred.
+        """
+
+        # Retrive the pynautobot object from Nautobot since we only have the UUID internally
+        remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
+
+        # Convert the internal attrs to Nautobot format
+        if "population" in attrs:
+            remote.custom_fields["country_population"] = attrs.get("population")
+        if "name" in attrs:
+            remote.name = attrs.get("name")
+
+        remote.save()
+        print(f"Updated Country {self.slug} | {attrs}")
+
+        return super().update(attrs)
+
+    def delete(self):
+        """Delete a country object in Nautobot.
+
+        Returns:
+            NautobotCountry: DiffSync object
+        """
+        # Retrieve the pynautobot object and delete the object in Nautobot
+        remote = self.diffsync.nautobot.dcim.regions.get(self.remote_id)
+        remote.delete()
+
+        super().delete()
+        return self