Merge pull request #5212 from netbox-community/4900-cable-paths

#4900: New model for cable paths

docs/release-notes/version-2.10.md

@@ -34,6 +34,12 @@ http://netbox/api/dcim/sites/ \
 --data '[{"id": 10, "description": "Foo"}, {"id": 11, "description": "Bar"}]'
 ```
 
+#### Improved Cable Trace Performance ([#4900](https://github.com/netbox-community/netbox/issues/4900))
+
+All end-to-end cable paths are now cached using the new CablePath model. This allows NetBox to now immediately return the complete path originating from any endpoint directly from the database, rather than having to trace each cable recursively. It also resolves some systemic validation issues with the original implementation.
+
+**Note:** As part of this change, cable traces will no longer traverse circuits: A circuit termination will be considered the origin or destination of an end-to-end path.
+
 ### Enhancements
 
 * [#1503](https://github.com/netbox-community/netbox/issues/1503) - Allow assigment of secrets to virtual machines
@@ -54,11 +60,44 @@ http://netbox/api/dcim/sites/ \
 
 ### REST API Changes
 
-* Added support for `PUT`, `PATCH`, and `DELETE` operations on list endpoints
+* Added support for `PUT`, `PATCH`, and `DELETE` operations on list endpoints (bulk update and delete)
+* circuits.CircuitTermination:
+  * Added the `/trace/` endpoint
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
 * dcim.Cable: Added `custom_fields`
+* dcim.ConsolePort:
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
+  * Removed `connection_status` from nested serializer
+* dcim.ConsoleServerPort:
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
+  * Removed `connection_status` from nested serializer
+* dcim.FrontPort:
+  * Replaced the `/trace/` endpoint with `/paths/`, which returns a list of cable paths
+  * Added `cable_peer` and `cable_peer_type`
+* dcim.Interface:
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
+  * Removed `connection_status` from nested serializer
 * dcim.InventoryItem: The `_depth` field has been added to reflect MPTT positioning
+* dcim.PowerFeed:
+  * Added the `/trace/` endpoint
+  * Added fields `connected_endpoint`, `connected_endpoint_type`, `connected_endpoint_reachable`, `cable_peer`, and `cable_peer_type`
+* dcim.PowerOutlet:
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
+  * Removed `connection_status` from nested serializer
 * dcim.PowerPanel: Added `custom_fields`
+* dcim.PowerPort
+  * Replaced `connection_status` with `connected_endpoint_reachable` (boolean)
+  * Added `cable_peer` and `cable_peer_type`
+  * Removed `connection_status` from nested serializer
 * dcim.RackReservation: Added `custom_fields`
+* dcim.RearPort:
+  * Replaced the `/trace/` endpoint with `/paths/`, which returns a list of cable paths
+  * Added `cable_peer` and `cable_peer_type`
 * dcim.VirtualChassis: Added `custom_fields`
 * extras.ExportTemplate: The `template_language` field has been removed
 * extras.Graph: This API endpoint has been removed (see #4349)

netbox/circuits/api/serializers.py

@@ -3,7 +3,7 @@
 from circuits.choices import CircuitStatusChoices
 from circuits.models import Provider, Circuit, CircuitTermination, CircuitType
 from dcim.api.nested_serializers import NestedCableSerializer, NestedInterfaceSerializer, NestedSiteSerializer
-from dcim.api.serializers import ConnectedEndpointSerializer
+from dcim.api.serializers import CableTerminationSerializer, ConnectedEndpointSerializer
 from extras.api.customfields import CustomFieldModelSerializer
 from extras.api.serializers import TaggedObjectSerializer
 from tenancy.api.nested_serializers import NestedTenantSerializer
@@ -67,7 +67,7 @@ class Meta:
         ]
 
 
-class CircuitTerminationSerializer(ConnectedEndpointSerializer):
+class CircuitTerminationSerializer(CableTerminationSerializer, ConnectedEndpointSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='circuits-api:circuittermination-detail')
     circuit = NestedCircuitSerializer()
     site = NestedSiteSerializer()
@@ -77,5 +77,6 @@ class Meta:
         model = CircuitTermination
         fields = [
             'id', 'url', 'circuit', 'term_side', 'site', 'port_speed', 'upstream_speed', 'xconnect_id', 'pp_info',
-            'description', 'connected_endpoint_type', 'connected_endpoint', 'connection_status', 'cable',
+            'description', 'cable', 'cable_peer', 'cable_peer_type', 'connected_endpoint', 'connected_endpoint_type',
+            'connected_endpoint_reachable'
         ]

netbox/circuits/api/views.py

@@ -3,6 +3,7 @@
 
 from circuits import filters
 from circuits.models import Provider, CircuitTermination, CircuitType, Circuit
+from dcim.api.views import PathEndpointMixin
 from extras.api.views import CustomFieldModelViewSet
 from utilities.api import ModelViewSet
 from . import serializers
@@ -46,9 +47,7 @@ class CircuitTypeViewSet(ModelViewSet):
 
 class CircuitViewSet(CustomFieldModelViewSet):
     queryset = Circuit.objects.prefetch_related(
-        Prefetch('terminations', queryset=CircuitTermination.objects.prefetch_related(
-            'site', 'connected_endpoint__device'
-        )),
+        Prefetch('terminations', queryset=CircuitTermination.objects.prefetch_related('site')),
         'type', 'tenant', 'provider',
     ).prefetch_related('tags')
     serializer_class = serializers.CircuitSerializer
@@ -59,9 +58,9 @@ class CircuitViewSet(CustomFieldModelViewSet):
 # Circuit Terminations
 #
 
-class CircuitTerminationViewSet(ModelViewSet):
+class CircuitTerminationViewSet(PathEndpointMixin, ModelViewSet):
     queryset = CircuitTermination.objects.prefetch_related(
-        'circuit', 'site', 'connected_endpoint__device', 'cable'
+        'circuit', 'site', '_path__destination', 'cable'
     )
     serializer_class = serializers.CircuitTerminationSerializer
     filterset_class = filters.CircuitTerminationFilterSet

netbox/circuits/filters.py

@@ -1,6 +1,7 @@
 import django_filters
 from django.db.models import Q
 
+from dcim.filters import CableTerminationFilterSet, PathEndpointFilterSet
 from dcim.models import Region, Site
 from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet
 from tenancy.filters import TenancyFilterSet
@@ -144,7 +145,7 @@ def search(self, queryset, name, value):
         ).distinct()
 
 
-class CircuitTerminationFilterSet(BaseFilterSet):
+class CircuitTerminationFilterSet(BaseFilterSet, CableTerminationFilterSet, PathEndpointFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',

netbox/circuits/migrations/0021_cache_cable_peer.py

@@ -0,0 +1,49 @@
+import sys
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def cache_cable_peers(apps, schema_editor):
+    ContentType = apps.get_model('contenttypes', 'ContentType')
+    Cable = apps.get_model('dcim', 'Cable')
+    CircuitTermination = apps.get_model('circuits', 'CircuitTermination')
+
+    if 'test' not in sys.argv:
+        print(f"\n    Updating circuit termination cable peers...", flush=True)
+    ct = ContentType.objects.get_for_model(CircuitTermination)
+    for cable in Cable.objects.filter(termination_a_type=ct):
+        CircuitTermination.objects.filter(pk=cable.termination_a_id).update(
+            _cable_peer_type_id=cable.termination_b_type_id,
+            _cable_peer_id=cable.termination_b_id
+        )
+    for cable in Cable.objects.filter(termination_b_type=ct):
+        CircuitTermination.objects.filter(pk=cable.termination_b_id).update(
+            _cable_peer_type_id=cable.termination_a_type_id,
+            _cable_peer_id=cable.termination_a_id
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('contenttypes', '0002_remove_content_type_name'),
+        ('circuits', '0020_custom_field_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='circuittermination',
+            name='_cable_peer_id',
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='circuittermination',
+            name='_cable_peer_type',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='contenttypes.contenttype'),
+        ),
+        migrations.RunPython(
+            code=cache_cable_peers,
+            reverse_code=migrations.RunPython.noop
+        ),
+    ]

netbox/circuits/migrations/0022_cablepath.py

@@ -0,0 +1,26 @@
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dcim', '0121_cablepath'),
+        ('circuits', '0021_cache_cable_peer'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='circuittermination',
+            name='_path',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='dcim.cablepath'),
+        ),
+        migrations.RemoveField(
+            model_name='circuittermination',
+            name='connected_endpoint',
+        ),
+        migrations.RemoveField(
+            model_name='circuittermination',
+            name='connection_status',
+        ),
+    ]

netbox/circuits/models.py

@@ -2,9 +2,8 @@
 from django.urls import reverse
 from taggit.managers import TaggableManager
 
-from dcim.constants import CONNECTION_STATUS_CHOICES
 from dcim.fields import ASNField
-from dcim.models import CableTermination
+from dcim.models import CableTermination, PathEndpoint
 from extras.models import ChangeLoggedModel, CustomFieldModel, ObjectChange, TaggedItem
 from extras.utils import extras_features
 from utilities.querysets import RestrictedQuerySet
@@ -232,7 +231,7 @@ def termination_z(self):
         return self._get_termination('Z')
 
 
-class CircuitTermination(CableTermination):
+class CircuitTermination(PathEndpoint, CableTermination):
     circuit = models.ForeignKey(
         to='circuits.Circuit',
         on_delete=models.CASCADE,
@@ -248,18 +247,6 @@ class CircuitTermination(CableTermination):
         on_delete=models.PROTECT,
         related_name='circuit_terminations'
     )
-    connected_endpoint = models.OneToOneField(
-        to='dcim.Interface',
-        on_delete=models.SET_NULL,
-        related_name='+',
-        blank=True,
-        null=True
-    )
-    connection_status = models.BooleanField(
-        choices=CONNECTION_STATUS_CHOICES,
-        blank=True,
-        null=True
-    )
     port_speed = models.PositiveIntegerField(
         verbose_name='Port speed (Kbps)'
     )

netbox/circuits/tests/test_filters.py

@@ -3,7 +3,7 @@
 from circuits.choices import *
 from circuits.filters import *
 from circuits.models import Circuit, CircuitTermination, CircuitType, Provider
-from dcim.models import Region, Site
+from dcim.models import Cable, Region, Site
 from tenancy.models import Tenant, TenantGroup
 
 
@@ -286,6 +286,8 @@ def setUpTestData(cls):
         ))
         CircuitTermination.objects.bulk_create(circuit_terminations)
 
+        Cable(termination_a=circuit_terminations[0], termination_b=circuit_terminations[1]).save()
+
     def test_term_side(self):
         params = {'term_side': 'A'}
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 3)
@@ -313,3 +315,13 @@ def test_site(self):
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4)
         params = {'site': [sites[0].slug, sites[1].slug]}
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4)
+
+    def test_cabled(self):
+        params = {'cabled': True}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
+
+    def test_connected(self):
+        params = {'connected': True}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
+        params = {'connected': False}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 4)

netbox/circuits/urls.py

@@ -1,6 +1,6 @@
 from django.urls import path
 
-from dcim.views import CableCreateView, CableTraceView
+from dcim.views import CableCreateView, PathTraceView
 from extras.views import ObjectChangeLogView
 from . import views
 from .models import Circuit, CircuitTermination, CircuitType, Provider
@@ -45,6 +45,6 @@
     path('circuit-terminations/<int:pk>/edit/', views.CircuitTerminationEditView.as_view(), name='circuittermination_edit'),
     path('circuit-terminations/<int:pk>/delete/', views.CircuitTerminationDeleteView.as_view(), name='circuittermination_delete'),
     path('circuit-terminations/<int:termination_a_id>/connect/<str:termination_b_type>/', CableCreateView.as_view(), name='circuittermination_connect', kwargs={'termination_a_type': CircuitTermination}),
-    path('circuit-terminations/<int:pk>/trace/', CableTraceView.as_view(), name='circuittermination_trace', kwargs={'model': CircuitTermination}),
+    path('circuit-terminations/<int:pk>/trace/', PathTraceView.as_view(), name='circuittermination_trace', kwargs={'model': CircuitTermination}),
 
 ]

netbox/circuits/views.py

@@ -131,15 +131,15 @@ def get(self, request, pk):
         circuit = get_object_or_404(self.queryset, pk=pk)
 
         termination_a = CircuitTermination.objects.restrict(request.user, 'view').prefetch_related(
-            'site__region', 'connected_endpoint__device'
+            'site__region'
         ).filter(
             circuit=circuit, term_side=CircuitTerminationSideChoices.SIDE_A
         ).first()
         if termination_a and termination_a.connected_endpoint:
             termination_a.ip_addresses = termination_a.connected_endpoint.ip_addresses.restrict(request.user, 'view')
 
         termination_z = CircuitTermination.objects.restrict(request.user, 'view').prefetch_related(
-            'site__region', 'connected_endpoint__device'
+            'site__region'
         ).filter(
             circuit=circuit, term_side=CircuitTerminationSideChoices.SIDE_Z
         ).first()

netbox/dcim/api/nested_serializers.py

@@ -1,8 +1,7 @@
 from rest_framework import serializers
 
-from dcim.constants import CONNECTION_STATUS_CHOICES
 from dcim import models
-from utilities.api import ChoiceField, WritableNestedSerializer
+from utilities.api import WritableNestedSerializer
 
 __all__ = [
     'NestedCableSerializer',
@@ -228,51 +227,46 @@ class Meta:
 class NestedConsoleServerPortSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='dcim-api:consoleserverport-detail')
     device = NestedDeviceSerializer(read_only=True)
-    connection_status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, read_only=True)
 
     class Meta:
         model = models.ConsoleServerPort
-        fields = ['id', 'url', 'device', 'name', 'cable', 'connection_status']
+        fields = ['id', 'url', 'device', 'name', 'cable']
 
 
 class NestedConsolePortSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='dcim-api:consoleport-detail')
     device = NestedDeviceSerializer(read_only=True)
-    connection_status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, read_only=True)
 
     class Meta:
         model = models.ConsolePort
-        fields = ['id', 'url', 'device', 'name', 'cable', 'connection_status']
+        fields = ['id', 'url', 'device', 'name', 'cable']
 
 
 class NestedPowerOutletSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='dcim-api:poweroutlet-detail')
     device = NestedDeviceSerializer(read_only=True)
-    connection_status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, read_only=True)
 
     class Meta:
         model = models.PowerOutlet
-        fields = ['id', 'url', 'device', 'name', 'cable', 'connection_status']
+        fields = ['id', 'url', 'device', 'name', 'cable']
 
 
 class NestedPowerPortSerializer(WritableNestedSerializer):
     url = serializers.HyperlinkedIdentityField(view_name='dcim-api:powerport-detail')
     device = NestedDeviceSerializer(read_only=True)
-    connection_status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, read_only=True)
 
     class Meta:
         model = models.PowerPort
-        fields = ['id', 'url', 'device', 'name', 'cable', 'connection_status']
+        fields = ['id', 'url', 'device', 'name', 'cable']
 
 
 class NestedInterfaceSerializer(WritableNestedSerializer):
     device = NestedDeviceSerializer(read_only=True)
     url = serializers.HyperlinkedIdentityField(view_name='dcim-api:interface-detail')
-    connection_status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, read_only=True)
 
     class Meta:
         model = models.Interface
-        fields = ['id', 'url', 'device', 'name', 'cable', 'connection_status']
+        fields = ['id', 'url', 'device', 'name', 'cable']
 
 
 class NestedRearPortSerializer(WritableNestedSerializer):