Closes #10945: Enable recurring execution of scheduled reports & scri…

…pts (#11096)

* Add interval to JobResult

* Accept a recurrence interval when executing scripts & reports

* Cleaned up jobs list display

* Schedule next job only if a reference start time can be determined

* Improve validation for scheduled jobs

netbox/dcim/tables/sites.py

@@ -99,9 +99,9 @@ class SiteTable(TenancyColumnsMixin, ContactsColumnMixin, NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = Site
         fields = (
-            'pk', 'id', 'name', 'slug', 'status', 'facility', 'region', 'group', 'tenant', 'tenant_group', 'asns', 'asn_count',
-            'time_zone', 'description', 'physical_address', 'shipping_address', 'latitude', 'longitude', 'comments',
-            'contacts', 'tags', 'created', 'last_updated', 'actions',
+            'pk', 'id', 'name', 'slug', 'status', 'facility', 'region', 'group', 'tenant', 'tenant_group', 'asns',
+            'asn_count', 'time_zone', 'description', 'physical_address', 'shipping_address', 'latitude', 'longitude',
+            'comments', 'contacts', 'tags', 'created', 'last_updated', 'actions',
         )
         default_columns = ('pk', 'name', 'status', 'facility', 'region', 'group', 'tenant', 'description')
 

netbox/extras/api/serializers.py

@@ -385,8 +385,8 @@ class JobResultSerializer(BaseModelSerializer):
     class Meta:
         model = JobResult
         fields = [
-            'id', 'url', 'display', 'status', 'created', 'scheduled', 'started', 'completed', 'name', 'obj_type',
-            'user', 'data', 'job_id',
+            'id', 'url', 'display', 'status', 'created', 'scheduled', 'interval', 'started', 'completed', 'name',
+            'obj_type', 'user', 'data', 'job_id',
         ]
 
 
@@ -414,6 +414,7 @@ class ReportDetailSerializer(ReportSerializer):
 
 class ReportInputSerializer(serializers.Serializer):
     schedule_at = serializers.DateTimeField(required=False, allow_null=True)
+    interval = serializers.IntegerField(required=False, allow_null=True)
 
 
 #
@@ -448,6 +449,7 @@ class ScriptInputSerializer(serializers.Serializer):
     data = serializers.JSONField()
     commit = serializers.BooleanField()
     schedule_at = serializers.DateTimeField(required=False, allow_null=True)
+    interval = serializers.IntegerField(required=False, allow_null=True)
 
 
 class ScriptLogMessageSerializer(serializers.Serializer):

netbox/extras/api/views.py

@@ -1,5 +1,4 @@
 from django.contrib.contenttypes.models import ContentType
-from django.db.models import Q
 from django.http import Http404
 from django_rq.queues import get_connection
 from rest_framework import status
@@ -246,16 +245,14 @@ def run(self, request, pk):
         input_serializer = serializers.ReportInputSerializer(data=request.data)
 
         if input_serializer.is_valid():
-            schedule_at = input_serializer.validated_data.get('schedule_at')
-
-            report_content_type = ContentType.objects.get(app_label='extras', model='report')
             job_result = JobResult.enqueue_job(
                 run_report,
-                report.full_name,
-                report_content_type,
-                request.user,
+                name=report.full_name,
+                obj_type=ContentType.objects.get_for_model(Report),
+                user=request.user,
                 job_timeout=report.job_timeout,
-                schedule_at=schedule_at,
+                schedule_at=input_serializer.validated_data.get('schedule_at'),
+                interval=input_serializer.validated_data.get('interval')
             )
             report.result = job_result
 
@@ -329,21 +326,17 @@ def post(self, request, pk):
             raise RQWorkerNotRunningException()
 
         if input_serializer.is_valid():
-            data = input_serializer.data['data']
-            commit = input_serializer.data['commit']
-            schedule_at = input_serializer.validated_data.get('schedule_at')
-
-            script_content_type = ContentType.objects.get(app_label='extras', model='script')
             job_result = JobResult.enqueue_job(
                 run_script,
-                script.full_name,
-                script_content_type,
-                request.user,
-                data=data,
+                name=script.full_name,
+                obj_type=ContentType.objects.get_for_model(Script),
+                user=request.user,
+                data=input_serializer.data['data'],
                 request=copy_safe_request(request),
-                commit=commit,
+                commit=input_serializer.data['commit'],
                 job_timeout=script.job_timeout,
-                schedule_at=schedule_at,
+                schedule_at=input_serializer.validated_data.get('schedule_at'),
+                interval=input_serializer.validated_data.get('interval')
             )
             script.result = job_result
             serializer = serializers.ScriptDetailSerializer(script, context={'request': request})

netbox/extras/choices.py

@@ -148,12 +148,12 @@ class JobResultStatusChoices(ChoiceSet):
     STATUS_FAILED = 'failed'
 
     CHOICES = (
-        (STATUS_PENDING, 'Pending'),
-        (STATUS_SCHEDULED, 'Scheduled'),
-        (STATUS_RUNNING, 'Running'),
-        (STATUS_COMPLETED, 'Completed'),
-        (STATUS_ERRORED, 'Errored'),
-        (STATUS_FAILED, 'Failed'),
+        (STATUS_PENDING, 'Pending', 'cyan'),
+        (STATUS_SCHEDULED, 'Scheduled', 'gray'),
+        (STATUS_RUNNING, 'Running', 'blue'),
+        (STATUS_COMPLETED, 'Completed', 'green'),
+        (STATUS_ERRORED, 'Errored', 'red'),
+        (STATUS_FAILED, 'Failed', 'red'),
     )
 
     TERMINAL_STATE_CHOICES = (

netbox/extras/filtersets.py

@@ -17,10 +17,10 @@
     'ConfigContextFilterSet',
     'ContentTypeFilterSet',
     'CustomFieldFilterSet',
-    'JobResultFilterSet',
     'CustomLinkFilterSet',
     'ExportTemplateFilterSet',
     'ImageAttachmentFilterSet',
+    'JobResultFilterSet',
     'JournalEntryFilterSet',
     'LocalConfigContextFilterSet',
     'ObjectChangeFilterSet',
@@ -537,7 +537,7 @@ class JobResultFilterSet(BaseFilterSet):
 
     class Meta:
         model = JobResult
-        fields = ('id', 'status', 'user', 'obj_type', 'name')
+        fields = ('id', 'interval', 'status', 'user', 'obj_type', 'name')
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/extras/forms/reports.py

@@ -1,4 +1,5 @@
 from django import forms
+from django.utils import timezone
 from django.utils.translation import gettext as _
 
 from utilities.forms import BootstrapMixin, DateTimePicker
@@ -15,3 +16,16 @@ class ReportForm(BootstrapMixin, forms.Form):
         label=_("Schedule at"),
         help_text=_("Schedule execution of report to a set time"),
     )
+    interval = forms.IntegerField(
+        required=False,
+        min_value=1,
+        label=_("Recurs every"),
+        help_text=_("Interval at which this report is re-run (in minutes)")
+    )
+
+    def clean_schedule_at(self):
+        scheduled_time = self.cleaned_data['schedule_at']
+        if scheduled_time and scheduled_time < timezone.now():
+            raise forms.ValidationError(_('Scheduled time must be in the future.'))
+
+        return scheduled_time

netbox/extras/forms/scripts.py

@@ -1,4 +1,5 @@
 from django import forms
+from django.utils import timezone
 from django.utils.translation import gettext as _
 
 from utilities.forms import BootstrapMixin, DateTimePicker
@@ -21,19 +22,36 @@ class ScriptForm(BootstrapMixin, forms.Form):
         label=_("Schedule at"),
         help_text=_("Schedule execution of script to a set time"),
     )
+    _interval = forms.IntegerField(
+        required=False,
+        min_value=1,
+        label=_("Recurs every"),
+        help_text=_("Interval at which this script is re-run (in minutes)")
+    )
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
         # Move _commit and _schedule_at to the end of the form
         schedule_at = self.fields.pop('_schedule_at')
+        interval = self.fields.pop('_interval')
         commit = self.fields.pop('_commit')
         self.fields['_schedule_at'] = schedule_at
+        self.fields['_interval'] = interval
         self.fields['_commit'] = commit
 
+    def clean__schedule_at(self):
+        scheduled_time = self.cleaned_data['_schedule_at']
+        if scheduled_time and scheduled_time < timezone.now():
+            raise forms.ValidationError({
+                '_schedule_at': _('Scheduled time must be in the future.')
+            })
+
+        return scheduled_time
+
     @property
     def requires_input(self):
         """
-        A boolean indicating whether the form requires user input (ignore the _commit and _schedule_at fields).
+        A boolean indicating whether the form requires user input (ignore the built-in fields).
         """
-        return bool(len(self.fields) > 2)
+        return bool(len(self.fields) > 3)

netbox/extras/migrations/0079_scheduled_jobs.py

@@ -1,3 +1,4 @@
+import django.core.validators
 from django.db import migrations, models
 
 
@@ -13,6 +14,11 @@ class Migration(migrations.Migration):
             name='scheduled',
             field=models.DateTimeField(blank=True, null=True),
         ),
+        migrations.AddField(
+            model_name='jobresult',
+            name='interval',
+            field=models.PositiveIntegerField(blank=True, null=True, validators=[django.core.validators.MinValueValidator(1)]),
+        ),
         migrations.AddField(
             model_name='jobresult',
             name='started',

netbox/extras/models/models.py

@@ -7,7 +7,7 @@
 from django.contrib.contenttypes.fields import GenericForeignKey
 from django.contrib.contenttypes.models import ContentType
 from django.core.cache import cache
-from django.core.validators import ValidationError
+from django.core.validators import MinValueValidator, ValidationError
 from django.db import models
 from django.http import HttpResponse, QueryDict
 from django.urls import reverse
@@ -587,6 +587,14 @@ class JobResult(models.Model):
         null=True,
         blank=True
     )
+    interval = models.PositiveIntegerField(
+        blank=True,
+        null=True,
+        validators=(
+            MinValueValidator(1),
+        ),
+        help_text=_("Recurrence interval (in minutes)")
+    )
     started = models.DateTimeField(
         null=True,
         blank=True
@@ -635,6 +643,9 @@ def delete(self, *args, **kwargs):
     def get_absolute_url(self):
         return reverse(f'extras:{self.obj_type.name}_result', args=[self.pk])
 
+    def get_status_color(self):
+        return JobResultStatusChoices.colors.get(self.status)
+
     @property
     def duration(self):
         if not self.completed:
@@ -664,33 +675,32 @@ def set_status(self, status):
             self.completed = timezone.now()
 
     @classmethod
-    def enqueue_job(cls, func, name, obj_type, user, schedule_at=None, *args, **kwargs):
+    def enqueue_job(cls, func, name, obj_type, user, schedule_at=None, interval=None, *args, **kwargs):
         """
         Create a JobResult instance and enqueue a job using the given callable
 
-        func: The callable object to be enqueued for execution
-        name: Name for the JobResult instance
-        obj_type: ContentType to link to the JobResult instance obj_type
-        user: User object to link to the JobResult instance
-        schedule_at: Schedule the job to be executed at the passed date and time
-        args: additional args passed to the callable
-        kwargs: additional kargs passed to the callable
+        Args:
+            func: The callable object to be enqueued for execution
+            name: Name for the JobResult instance
+            obj_type: ContentType to link to the JobResult instance obj_type
+            user: User object to link to the JobResult instance
+            schedule_at: Schedule the job to be executed at the passed date and time
+            interval: Recurrence interval (in minutes)
         """
-        job_result: JobResult = cls.objects.create(
+        rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.name, RQ_QUEUE_DEFAULT)
+        queue = django_rq.get_queue(rq_queue_name)
+        status = JobResultStatusChoices.STATUS_SCHEDULED if schedule_at else JobResultStatusChoices.STATUS_PENDING
+        job_result: JobResult = JobResult.objects.create(
             name=name,
+            status=status,
             obj_type=obj_type,
+            scheduled=schedule_at,
+            interval=interval,
             user=user,
             job_id=uuid.uuid4()
         )
 
-        rq_queue_name = get_config().QUEUE_MAPPINGS.get(obj_type.name, RQ_QUEUE_DEFAULT)
-        queue = django_rq.get_queue(rq_queue_name)
-
         if schedule_at:
-            job_result.status = JobResultStatusChoices.STATUS_SCHEDULED
-            job_result.scheduled = schedule_at
-            job_result.save()
-
             queue.enqueue_at(schedule_at, func, job_id=str(job_result.job_id), job_result=job_result, **kwargs)
         else:
             queue.enqueue(func, job_id=str(job_result.job_id), job_result=job_result, **kwargs)

netbox/extras/reports.py

@@ -1,8 +1,8 @@
-import importlib
 import inspect
 import logging
 import pkgutil
 import traceback
+from datetime import timedelta
 
 from django.conf import settings
 from django.utils import timezone
@@ -11,7 +11,6 @@
 from .choices import JobResultStatusChoices, LogLevelChoices
 from .models import JobResult
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -85,10 +84,24 @@ def run_report(job_result, *args, **kwargs):
     try:
         job_result.start()
         report.run(job_result)
-    except Exception as e:
+    except Exception:
         job_result.set_status(JobResultStatusChoices.STATUS_ERRORED)
         job_result.save()
         logging.error(f"Error during execution of report {job_result.name}")
+    finally:
+        # Schedule the next job if an interval has been set
+        start_time = job_result.scheduled or job_result.started
+        if start_time and job_result.interval:
+            new_scheduled_time = start_time + timedelta(minutes=job_result.interval)
+            JobResult.enqueue_job(
+                run_report,
+                name=job_result.name,
+                obj_type=job_result.obj_type,
+                user=job_result.user,
+                job_timeout=report.job_timeout,
+                schedule_at=new_scheduled_time,
+                interval=job_result.interval
+            )
 
 
 class Report(object):

netbox/extras/scripts.py

@@ -4,8 +4,9 @@
 import os
 import pkgutil
 import sys
-import traceback
 import threading
+import traceback
+from datetime import timedelta
 
 import yaml
 from django import forms
@@ -16,6 +17,7 @@
 
 from extras.api.serializers import ScriptOutputSerializer
 from extras.choices import JobResultStatusChoices, LogLevelChoices
+from extras.models import JobResult
 from extras.signals import clear_webhooks
 from ipam.formfields import IPAddressFormField, IPNetworkFormField
 from ipam.validators import MaxPrefixLengthValidator, MinPrefixLengthValidator, prefix_validator
@@ -491,6 +493,22 @@ def _run_script():
     else:
         _run_script()
 
+    # Schedule the next job if an interval has been set
+    if job_result.interval:
+        new_scheduled_time = job_result.scheduled + timedelta(minutes=job_result.interval)
+        JobResult.enqueue_job(
+            run_script,
+            name=job_result.name,
+            obj_type=job_result.obj_type,
+            user=job_result.user,
+            schedule_at=new_scheduled_time,
+            interval=job_result.interval,
+            job_timeout=script.job_timeout,
+            data=data,
+            request=request,
+            commit=commit
+        )
+
 
 def get_scripts(use_names=False):
     """