Closes #11890: Sync/upload reports & scripts (#12059)

* Initial work on #11890

* Consolidate get_scripts() and get_reports() functions

* Introduce proxy models for script & report modules

* Add add/delete views for reports & scripts

* Add deletion links for modules

* Enable resolving scripts/reports from module class

* Remove get_modules() utility function

* Show results in report/script lists

* Misc cleanup

* Fix file uploads

* Support automatic migration for submodules

* Fix module child ordering

* Template cleanup

* Remove ManagedFile views

* Move is_script(), is_report() into extras.utils

* Fix URLs for nested reports & scripts

* Misc cleanup

netbox/core/choices.py

@@ -20,7 +20,6 @@ class DataSourceTypeChoices(ChoiceSet):
 
 
 class DataSourceStatusChoices(ChoiceSet):
-
     NEW = 'new'
     QUEUED = 'queued'
     SYNCING = 'syncing'
@@ -34,3 +33,17 @@ class DataSourceStatusChoices(ChoiceSet):
         (COMPLETED, _('Completed'), 'green'),
         (FAILED, _('Failed'), 'red'),
     )
+
+
+#
+# Managed files
+#
+
+class ManagedFileRootPathChoices(ChoiceSet):
+    SCRIPTS = 'scripts'  # settings.SCRIPTS_ROOT
+    REPORTS = 'reports'  # settings.REPORTS_ROOT
+
+    CHOICES = (
+        (SCRIPTS, _('Scripts')),
+        (REPORTS, _('Reports')),
+    )

netbox/core/forms/model_forms.py

@@ -3,12 +3,14 @@
 from django import forms
 
 from core.models import *
+from extras.forms.mixins import SyncedDataMixin
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
 from utilities.forms import CommentField, get_field_value
 
 __all__ = (
     'DataSourceForm',
+    'ManagedFileForm',
 )
 
 
@@ -73,3 +75,37 @@ def save(self, *args, **kwargs):
         self.instance.parameters = parameters
 
         return super().save(*args, **kwargs)
+
+
+class ManagedFileForm(SyncedDataMixin, NetBoxModelForm):
+    upload_file = forms.FileField(
+        required=False
+    )
+
+    fieldsets = (
+        ('File Upload', ('upload_file',)),
+        ('Data Source', ('data_source', 'data_file')),
+    )
+
+    class Meta:
+        model = ManagedFile
+        fields = ('data_source', 'data_file')
+
+    def clean(self):
+        super().clean()
+
+        if self.cleaned_data.get('upload_file') and self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Cannot upload a file and sync from an existing file")
+        if not self.cleaned_data.get('upload_file') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must upload a file or select a data file to sync")
+
+        return self.cleaned_data
+
+    def save(self, *args, **kwargs):
+        # If a file was uploaded, save it to disk
+        if self.cleaned_data['upload_file']:
+            self.instance.file_path = self.cleaned_data['upload_file'].name
+            with open(self.instance.full_path, 'wb+') as new_file:
+                new_file.write(self.cleaned_data['upload_file'].read())
+
+        return super().save(*args, **kwargs)

netbox/core/migrations/0002_managedfile.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.1.7 on 2023-03-23 17:35
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ManagedFile',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+                ('data_path', models.CharField(blank=True, editable=False, max_length=1000)),
+                ('data_synced', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('last_updated', models.DateTimeField(blank=True, editable=False, null=True)),
+                ('file_root', models.CharField(max_length=1000)),
+                ('file_path', models.FilePathField(editable=False)),
+                ('data_file', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile')),
+                ('data_source', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource')),
+            ],
+            options={
+                'ordering': ('file_root', 'file_path'),
+            },
+        ),
+        migrations.AddIndex(
+            model_name='managedfile',
+            index=models.Index(fields=['file_root', 'file_path'], name='core_managedfile_root_path'),
+        ),
+        migrations.AddConstraint(
+            model_name='managedfile',
+            constraint=models.UniqueConstraint(fields=('file_root', 'file_path'), name='core_managedfile_unique_root_path'),
+        ),
+    ]

netbox/core/models/__init__.py

@@ -1 +1,2 @@
 from .data import *
+from .files import *

netbox/core/models/data.py

@@ -14,7 +14,6 @@
 from django.utils.module_loading import import_string
 from django.utils.translation import gettext as _
 
-from extras.models import JobResult
 from netbox.models import PrimaryModel
 from netbox.registry import registry
 from utilities.files import sha256_hash
@@ -113,6 +112,8 @@ def enqueue_sync_job(self, request):
         """
         Enqueue a background job to synchronize the DataSource by calling sync().
         """
+        from extras.models import JobResult
+
         # Set the status to "syncing"
         self.status = DataSourceStatusChoices.QUEUED
         DataSource.objects.filter(pk=self.pk).update(status=self.status)
@@ -314,3 +315,14 @@ def refresh_from_disk(self, source_root):
                 self.data = f.read()
 
         return is_modified
+
+    def write_to_disk(self, path, overwrite=False):
+        """
+        Write the object's data to disk at the specified path
+        """
+        # Check whether file already exists
+        if os.path.isfile(path) and not overwrite:
+            raise FileExistsError()
+
+        with open(path, 'wb+') as new_file:
+            new_file.write(self.data)

netbox/core/models/files.py

@@ -0,0 +1,88 @@
+import logging
+import os
+
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from django.utils.translation import gettext as _
+
+from ..choices import ManagedFileRootPathChoices
+from netbox.models.features import SyncedDataMixin
+from utilities.querysets import RestrictedQuerySet
+
+__all__ = (
+    'ManagedFile',
+)
+
+logger = logging.getLogger('netbox.core.files')
+
+
+class ManagedFile(SyncedDataMixin, models.Model):
+    """
+    Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule)
+    to provide additional functionality.
+    """
+    created = models.DateTimeField(
+        auto_now_add=True
+    )
+    last_updated = models.DateTimeField(
+        editable=False,
+        blank=True,
+        null=True
+    )
+    file_root = models.CharField(
+        max_length=1000,
+        choices=ManagedFileRootPathChoices
+    )
+    file_path = models.FilePathField(
+        editable=False,
+        help_text=_("File path relative to the designated root path")
+    )
+
+    objects = RestrictedQuerySet.as_manager()
+
+    class Meta:
+        ordering = ('file_root', 'file_path')
+        constraints = (
+            models.UniqueConstraint(
+                fields=('file_root', 'file_path'),
+                name='%(app_label)s_%(class)s_unique_root_path'
+            ),
+        )
+        indexes = [
+            models.Index(fields=('file_root', 'file_path'), name='core_managedfile_root_path'),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse('core:managedfile', args=[self.pk])
+
+    @property
+    def name(self):
+        return self.file_path
+
+    @property
+    def full_path(self):
+        return os.path.join(self._resolve_root_path(), self.file_path)
+
+    def _resolve_root_path(self):
+        return {
+            'scripts': settings.SCRIPTS_ROOT,
+            'reports': settings.REPORTS_ROOT,
+        }[self.file_root]
+
+    def sync_data(self):
+        if self.data_file:
+            self.file_path = os.path.basename(self.data_path)
+            self.data_file.write_to_disk(self.full_path, overwrite=True)
+
+    def delete(self, *args, **kwargs):
+        # Delete file from disk
+        try:
+            os.remove(self.full_path)
+        except FileNotFoundError:
+            pass
+
+        return super().delete(*args, **kwargs)

netbox/extras/api/views.py

@@ -16,8 +16,8 @@
 from extras.choices import JobResultStatusChoices
 from extras.models import *
 from extras.models import CustomField
-from extras.reports import get_report, get_reports, run_report
-from extras.scripts import get_script, get_scripts, run_script
+from extras.reports import get_report, run_report
+from extras.scripts import get_script, run_script
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
 from netbox.api.features import SyncedDataMixin
 from netbox.api.metadata import ContentTypeMetadata
@@ -27,7 +27,6 @@
 from utilities.utils import copy_safe_request, count_related
 from . import serializers
 from .mixins import ConfigTemplateRenderMixin
-from .nested_serializers import NestedConfigTemplateSerializer
 
 
 class ExtrasRootView(APIRootView):
@@ -189,7 +188,6 @@ def list(self, request):
         """
         Compile all reports and their related results (if any). Result data is deferred in the list view.
         """
-        report_list = []
         report_content_type = ContentType.objects.get(app_label='extras', model='report')
         results = {
             r.name: r
@@ -199,13 +197,13 @@ def list(self, request):
             ).order_by('name', '-created').distinct('name').defer('data')
         }
 
-        # Iterate through all available Reports.
-        for module_name, reports in get_reports().items():
-            for report in reports.values():
+        report_list = []
+        for report_module in ReportModule.objects.restrict(request.user):
+            report_list.extend([report() for report in report_module.reports.values()])
 
-                # Attach the relevant JobResult (if any) to each Report.
-                report.result = results.get(report.full_name, None)
-                report_list.append(report)
+        # Attach JobResult objects to each report (if any)
+        for report in report_list:
+            report.result = results.get(report.full_name, None)
 
         serializer = serializers.ReportSerializer(report_list, many=True, context={
             'request': request,
@@ -296,15 +294,15 @@ def list(self, request):
             ).order_by('name', '-created').distinct('name').defer('data')
         }
 
-        flat_list = []
-        for script_list in get_scripts().values():
-            flat_list.extend(script_list.values())
+        script_list = []
+        for script_module in ScriptModule.objects.restrict(request.user):
+            script_list.extend(script_module.scripts.values())
 
         # Attach JobResult objects to each script (if any)
-        for script in flat_list:
+        for script in script_list:
             script.result = results.get(script.full_name, None)
 
-        serializer = serializers.ScriptSerializer(flat_list, many=True, context={'request': request})
+        serializer = serializers.ScriptSerializer(script_list, many=True, context={'request': request})
 
         return Response(serializer.data)
 

netbox/extras/management/commands/runreport.py

@@ -5,8 +5,8 @@
 from django.utils import timezone
 
 from extras.choices import JobResultStatusChoices
-from extras.models import JobResult
-from extras.reports import get_reports, run_report
+from extras.models import JobResult, ReportModule
+from extras.reports import run_report
 
 
 class Command(BaseCommand):
@@ -17,13 +17,9 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
 
-        # Gather all available reports
-        reports = get_reports()
-
-        # Run reports
-        for module_name, report_list in reports.items():
-            for report in report_list.values():
-                if module_name in options['reports'] or report.full_name in options['reports']:
+        for module in ReportModule.objects.all():
+            for report in module.reports.values():
+                if module.name in options['reports'] or report.full_name in options['reports']:
 
                     # Run the report and create a new JobResult
                     self.stdout.write(