Develop -> Main

.github/workflows/ci.yml

@@ -112,7 +112,7 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.7", "3.8", "3.9"]
-        nautobot-version: ["1.1.6", "1.2.4"]
+        nautobot-version: ["1.2.4"]
     runs-on: "ubuntu-20.04"
     env:
       INVOKE_NAUTOBOT_SSOT_IPFABRIC_PYTHON_VER: "${{ matrix.python-version }}"

GETTING_STARTED.md

@@ -285,7 +285,7 @@ namespace.configure(
     {
         "nautobot_ssot_ipfabric": {
             ...
-            "nautobot_ver": "1.0.2",
+            "nautobot_ver": "1.2.4",
 	    ...
         }
     }

invoke.example.yml

@@ -1,7 +1,7 @@
 ---
 nautobot_ssot_ipfabric:
   project_name: "nautobot-ssot-ipfabric"
-  nautobot_ver: "1.1.4"
+  nautobot_ver: "1.2.4"
   local: false
   python_ver: "3.7"
   compose_dir: "development"

nautobot_ssot_ipfabric/diffsync/adapter_ipfabric.py

@@ -4,8 +4,8 @@
 
 from diffsync import ObjectAlreadyExists
 from django.conf import settings
-from netutils.mac import mac_to_format
 from nautobot.ipam.models import VLAN
+from netutils.mac import mac_to_format
 
 from nautobot_ssot_ipfabric.diffsync import DiffSyncModelAdapters
 

nautobot_ssot_ipfabric/diffsync/adapter_nautobot.py

@@ -7,11 +7,12 @@
 from diffsync import DiffSync
 from diffsync.exceptions import ObjectAlreadyExists
 from django.conf import settings
-from django.db import transaction
+from django.db import IntegrityError, transaction
 from django.db.models import ProtectedError, Q
 from nautobot.dcim.models import Device, Site
 from nautobot.extras.models import Tag
-from nautobot.ipam.models import VLAN
+from nautobot.ipam.models import VLAN, Interface
+from nautobot.utilities.choices import ColorChoices
 from netutils.mac import mac_to_format
 
 from nautobot_ssot_ipfabric.diffsync import DiffSyncModelAdapters
@@ -31,12 +32,12 @@ class NautobotDiffSync(DiffSyncModelAdapters):
     _vlan: ClassVar[Any] = VLAN
     _device: ClassVar[Any] = Device
     _site: ClassVar[Any] = Site
+    _interface: ClassVar[Any] = Interface
 
     def __init__(
         self,
         job,
         sync,
-        safe_delete_mode: bool,
         sync_ipfabric_tagged_only: bool,
         site_filter: Site,
         *args,
@@ -46,7 +47,6 @@ def __init__(
         super().__init__(*args, **kwargs)
         self.job = job
         self.sync = sync
-        self.safe_delete_mode = safe_delete_mode
         self.sync_ipfabric_tagged_only = sync_ipfabric_tagged_only
         self.site_filter = site_filter
 
@@ -61,16 +61,22 @@ def sync_complete(self, source: DiffSync, *args, **kwargs):
         """
         for grouping in (
             "_vlan",
+            "_interface",
             "_device",
             "_site",
         ):
             for nautobot_object in self.objects_to_delete[grouping]:
-                if self.safe_delete_mode:
+                if NautobotDiffSync.safe_delete_mode:
                     continue
                 try:
                     nautobot_object.delete()
                 except ProtectedError:
                     self.job.log_failure(obj=nautobot_object, message="Deletion failed protected object")
+                except IntegrityError:
+                    self.job.log_failure(
+                        obj=nautobot_object, message=f"Deletion failed due to IntegrityError with {nautobot_object}"
+                    )
+
             self.objects_to_delete[grouping] = []
         return super().sync_complete(source, *args, **kwargs)
 
@@ -101,14 +107,13 @@ def load_interfaces(self, device_record: Device, diffsync_device):
                 if interface_record.ip_addresses.first()
                 else None,
             )
-            if not self.safe_delete_mode:
-                self.interface.safe_delete_mode = self.safe_delete_mode
             self.add(interface)
             diffsync_device.add_child(interface)
 
     def load_device(self, filtered_devices: List, location):
         """Load Devices from Nautobot."""
         for device_record in filtered_devices:
+            self.job.log_debug(message=f"Loading Nautobot Device: {device_record.name}")
             device = self.device(
                 diffsync=self,
                 name=device_record.name,
@@ -119,8 +124,6 @@ def load_device(self, filtered_devices: List, location):
                 status=device_record.status.name,
                 serial_number=device_record.serial if device_record.serial else "",
             )
-            if not self.safe_delete_mode:
-                self.device.safe_delete_mode = self.safe_delete_mode
             try:
                 self.add(device)
             except ObjectAlreadyExists:
@@ -144,8 +147,6 @@ def load_vlans(self, filtered_vlans: List, location):
                 vlan_pk=vlan_record.pk,
                 description=vlan_record.description,
             )
-            if not self.safe_delete_mode:
-                self.vlan.safe_delete_mode = self.safe_delete_mode
             try:
                 self.add(vlan)
             except ObjectAlreadyExists:
@@ -178,19 +179,32 @@ def get_initial_site(self, ssot_tag: Tag):
     @transaction.atomic
     def load_data(self):
         """Add Nautobot Site objects as DiffSync Location models."""
-        ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric")
+        ssot_tag, _ = Tag.objects.get_or_create(
+            slug="ssot-synced-from-ipfabric",
+            name="SSoT Synced from IPFabric",
+            defaults={
+                "description": "Object synced at some point from IPFabric to Nautobot",
+                "color": ColorChoices.COLOR_LIGHT_GREEN,
+            },
+        )
         site_objects = self.get_initial_site(ssot_tag)
         # The parent object that stores all children, is the Site.
+        self.job.log_debug(message=f"Found {site_objects.count()} Nautobot Site objects to start sync from")
+
         if site_objects:
             for site_record in site_objects:
-                location = self.location(
-                    diffsync=self,
-                    name=site_record.name,
-                    site_id=site_record.custom_field_data.get("ipfabric-site-id"),
-                    status=site_record.status.name,
-                )
-                if not self.safe_delete_mode:
-                    self.location.safe_delete_mode = self.safe_delete_mode
+                try:
+                    location = self.location(
+                        diffsync=self,
+                        name=site_record.name,
+                        site_id=site_record.custom_field_data.get("ipfabric-site-id"),
+                        status=site_record.status.name,
+                    )
+                except AttributeError:
+                    self.job.log_debug(
+                        message=f"Error loading {site_record}, invalid or missing attributes on object. Skipping..."
+                    )
+                    continue
                 self.add(location)
                 try:
                     # Load Site's Children - Devices with Interfaces, if any.

nautobot_ssot_ipfabric/diffsync/adapters_shared.py

@@ -1,5 +1,7 @@
 """Diff sync shared adapter class attritbutes to synchronize applications."""
 
+from typing import ClassVar
+
 from diffsync import DiffSync
 
 from nautobot_ssot_ipfabric.diffsync import diffsync_models
@@ -8,6 +10,8 @@
 class DiffSyncModelAdapters(DiffSync):
     """Nautobot adapter for DiffSync."""
 
+    safe_delete_mode: ClassVar[bool] = True
+
     location = diffsync_models.Location
     device = diffsync_models.Device
     interface = diffsync_models.Interface

nautobot_ssot_ipfabric/diffsync/diffsync_models.py

@@ -1,9 +1,8 @@
 # Ignore return statements for updates and deletes, #  pylint:disable=R1710
 # Ignore too many args #  pylint:disable=too-many-locals
 """DiffSyncModel subclasses for Nautobot-to-IPFabric data sync."""
-from typing import Any, ClassVar, List, Optional
+from typing import Any, List, Optional, ClassVar
 from uuid import UUID
-
 from diffsync import DiffSyncModel
 from django.conf import settings
 from django.core.exceptions import ValidationError
@@ -13,6 +12,7 @@
 from nautobot.extras.models import Tag
 from nautobot.extras.models.statuses import Status
 from nautobot.ipam.models import VLAN
+from nautobot.utilities.choices import ColorChoices
 
 import nautobot_ssot_ipfabric.utilities.nbutils as tonb_nbutils
 
@@ -33,16 +33,15 @@ class DiffSyncExtras(DiffSyncModel):
 
     safe_delete_mode: ClassVar[bool] = True
 
-    def safe_delete(self, nautobot_object: Any, safe_mode: bool = True, safe_delete_status: Optional[str] = None):
+    def safe_delete(self, nautobot_object: Any, safe_delete_status: Optional[str] = None):
         """Safe delete an object, by adding tags or changing it's default status.
 
         Args:
             nautobot_object (Any): Any type of Nautobot object
-            safe_mode (bool): Safe mode or not
             safe_delete_status (Optional[str], optional): Status name, optional as some objects don't have status field. Defaults to None.
         """
         update = False
-        if not safe_mode:
+        if not self.safe_delete_mode:  # This could just check self, refactor.
             self.diffsync.job.log_warning(
                 message=f"{nautobot_object} will be deleted as safe delete mode is not enabled."
             )
@@ -66,11 +65,19 @@ def safe_delete(self, nautobot_object: Any, safe_mode: bool = True, safe_delete_
                     # Not everything has a status. This may come in handy once more models are synced.
                     self.diffsync.job.log_warning(message=f"{nautobot_object} has no Status attribute.")
             if hasattr(nautobot_object, "tags"):
-                ssot_safe_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete")
+                ssot_safe_tag, _ = Tag.objects.get_or_create(
+                    slug="ssot-safe-delete",
+                    name="SSoT Safe Delete",
+                    defaults={
+                        "description": "Safe Delete Mode tag to flag an object, but not delete from Nautobot.",
+                        "color": ColorChoices.COLOR_RED,
+                    },
+                )
                 object_tags = nautobot_object.tags.all()
                 # No exception raised for empty iterator, safe to do this any
                 if not any(obj_tag for obj_tag in object_tags if obj_tag.name == ssot_safe_tag.name):
                     nautobot_object.tags.add(ssot_safe_tag)
+                    self.diffsync.job.log_warning(message=f"Tagging {nautobot_object} with `ssot-safe-delete`.")
                     update = True
             if update:
                 tonb_nbutils.tag_object(nautobot_object=nautobot_object, custom_field="ssot-synced-from-ipfabric")
@@ -105,9 +112,9 @@ def create(cls, diffsync, ids, attrs):
     def delete(self) -> Optional["DiffSyncModel"]:
         """Delete Site in Nautobot."""
         site_object = Site.objects.get(name=self.name)
+
         self.safe_delete(
             site_object,
-            self.safe_delete_mode,
             SAFE_DELETE_SITE_STATUS,
         )
         return self
@@ -117,6 +124,7 @@ def update(self, attrs):
         site = Site.objects.get(name=self.name)
         if attrs.get("site_id"):
             site.custom_field_data["ipfabric-site-id"] = attrs.get("site_id")
+            site.validated_save()
         if attrs.get("status") == "Active":
             safe_delete_tag, _ = Tag.objects.get_or_create(name="SSoT Safe Delete")
             if not site.status == "Active":
@@ -204,7 +212,6 @@ def delete(self) -> Optional["DiffSyncModel"]:
             device_object = NautobotDevice.objects.get(name=self.name)
             self.safe_delete(
                 device_object,
-                self.safe_delete_mode,
                 SAFE_DELETE_DEVICE_STATUS,
             )
             return self
@@ -294,6 +301,8 @@ def create(cls, diffsync, ids, attrs):
         )
         ip_address = attrs["ip_address"]
         if ip_address:
+            if interface_obj.ip_addresses.all().exists():
+                interface_obj.ip_addresses.all().delete()
             ip_address_obj = tonb_nbutils.create_ip(
                 ip_address=attrs["ip_address"],
                 subnet_mask=attrs["subnet_mask"],
@@ -314,18 +323,16 @@ def delete(self) -> Optional["DiffSyncModel"]:
         try:
             ssot_tag, _ = Tag.objects.get_or_create(name="SSoT Synced from IPFabric")
             device = NautobotDevice.objects.filter(Q(name=self.device_name) & Q(tags__slug=ssot_tag.slug)).first()
-            # TODO: Implement ordering of delete for diffsync. Interfaces would be first, followed by devices, vlan, site.
             if not device:
                 return
             interface = device.interfaces.get(name=self.name)
             # Access the addr within an interface, change the status if necessary
             if interface.ip_addresses.first():
-                self.safe_delete(interface.ip_addresses.first(), self.safe_delete_mode, SAFE_DELETE_IPADDRESS_STATUS)
+                self.safe_delete(interface.ip_addresses.first(), SAFE_DELETE_IPADDRESS_STATUS)
             # Then do the parent interface
             # Attached interfaces do not have a status to update.
             self.safe_delete(
                 interface,
-                self.safe_delete_mode,
             )
             return self
         except NautobotDevice.DoesNotExist:
@@ -354,7 +361,9 @@ def update(self, attrs):
             if attrs.get("mgmt_only"):
                 interface.mgmt_only = attrs["mgmt_only"]
             if attrs.get("ip_address"):
-                interface.ip_addresses.all().delete()
+                if interface.ip_addresses.all().exists():
+                    self.diffsync.job.log_debug(message=f"Replacing IP from interface {interface} on {device.name}")
+                    interface.ip_addresses.all().delete()
                 ip_address_obj = tonb_nbutils.create_ip(
                     ip_address=attrs.get("ip_address"),
                     subnet_mask=attrs.get("subnet_mask") if attrs.get("subnet_mask") else "255.255.255.255",
@@ -364,6 +373,7 @@ def update(self, attrs):
                 interface.ip_addresses.add(ip_address_obj)
             tonb_nbutils.tag_object(nautobot_object=interface, custom_field="ssot-synced-from-ipfabric")
             return super().update(attrs)
+
         except NautobotDevice.DoesNotExist:
             self.diffsync.job.log_warning(f"Unable to match device by name, {self.name}")
 
@@ -390,7 +400,7 @@ def create(cls, diffsync, ids, attrs):
         site = Site.objects.get(name=ids["site"])
         name = ids["name"] if ids["name"] else f"VLAN{attrs['vid']}"
         description = attrs["description"] if attrs["description"] else None
-        diffsync.job.log_debug(message=f"Creating VLAN: {name}: {len(name)}, {description}: {len(description)}")
+        diffsync.job.log_debug(message=f"Creating VLAN: {name} description: {description}")
         tonb_nbutils.create_vlan(
             vlan_name=name,
             vlan_id=attrs["vid"],
@@ -405,7 +415,6 @@ def delete(self) -> Optional["DiffSyncModel"]:
         vlan = VLAN.objects.get(name=self.name, pk=self.vlan_pk)
         self.safe_delete(
             vlan,
-            self.safe_delete_mode,
             SAFE_DELETE_VLAN_STATUS,
         )
         return self
@@ -421,7 +430,6 @@ def update(self, attrs):
             device_tags = vlan.tags.filter(pk=safe_delete_tag.pk)
             if device_tags.exists():
                 vlan.tags.remove(safe_delete_tag)
-            tonb_nbutils.tag_object(nautobot_object=vlan, custom_field="ssot-synced-from-ipfabric")
         if attrs.get("description"):
             vlan.description = vlan.description