Merge pull request #20 from RockefellerArchiveCenter/signals

Uses Celery Signals to mark services as active or inactive

requirements.txt

@@ -6,7 +6,7 @@ certifi==2018.4.16
 chardet==3.0.4
 Django==2.0.8
 django-celery-beat==1.2.0
-django-celery-results==1.0.1
+django-celery-results==1.0.4
 django-crispy-forms==1.7.2
 django-datatables-view==1.17.0
 django-timezone-field==3.0

zodiac/gateway/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'gateway.apps.GatewayConfig'

zodiac/gateway/apps.py

@@ -3,3 +3,6 @@
 
 class GatewayConfig(AppConfig):
  name = 'gateway'
+
+ def ready(self):
+ import gateway.signals

zodiac/gateway/migrations/0015_auto_20181214_1718.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-12-14 17:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('gateway', '0014_auto_20181116_2141'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='requestlog',
+ name='async_result_id',
+ field=models.CharField(blank=True, max_length=36, null=True),
+ ),
+ ]

zodiac/gateway/migrations/0016_serviceregistry_has_active_task.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.8 on 2018-12-14 21:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('gateway', '0015_auto_20181214_1718'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='serviceregistry',
+ name='has_active_task',
+ field=models.BooleanField(default=False),
+ ),
+ ]

zodiac/gateway/migrations/0017_requestlog_task_result.py

@@ -0,0 +1,20 @@
+# Generated by Django 2.0.8 on 2018-12-20 04:08
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_celery_results', '0003_auto_20181106_1101'),
+ ('gateway', '0016_serviceregistry_has_active_task'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='requestlog',
+ name='task_result',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='django_celery_results.TaskResult'),
+ ),
+ ]

zodiac/gateway/migrations/0018_auto_20181220_0428.py

@@ -0,0 +1,19 @@
+# Generated by Django 2.0.8 on 2018-12-20 04:28
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('gateway', '0017_requestlog_task_result'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='requestlog',
+ name='task_result',
+ field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='request_log', to='django_celery_results.TaskResult'),
+ ),
+ ]

zodiac/gateway/models.py

@@ -1,9 +1,13 @@
 # -*- coding: utf-8 -*-
+import ast
+
 from django.db import models
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 from django.urls import reverse
 
+from django_celery_results.models import TaskResult
+
 
 class Consumer(models.Model):
  user = models.OneToOneField(
@@ -57,7 +61,8 @@ class ServiceRegistry(models.Model):
  consumers = models.ManyToManyField(Consumer, blank=True)
  is_active = models.BooleanField(default=True)
  is_private = models.BooleanField(default=False)
- method = models.CharField(max_length=10,choices=HTTP_REQUESTS_METHODS)
+ has_active_task = models.BooleanField(default=False)
+ method = models.CharField(max_length=10, choices=HTTP_REQUESTS_METHODS)
  callback_service = models.ForeignKey(
  'self',
  on_delete=models.CASCADE,
@@ -109,15 +114,17 @@ class RequestLog(models.Model):
  )
  status_code = models.CharField(max_length=4, blank=True, null=True)
  request_url = models.URLField(blank=True, null=True)
- async_result_id = models.CharField(max_length=30, blank=True, null=True)
+ async_result_id = models.CharField(max_length=36, blank=True, null=True)
  created_time = models.DateTimeField(auto_now_add=True)
+ task_result = models.ForeignKey(TaskResult, on_delete=models.CASCADE, blank=True, null=True, related_name='request_log')
 
  @classmethod
- def create(cls, service, status_code, request_url, async_result_id=None):
+ def create(cls, service, status_code, request_url, async_result_id=None, task_result=None):
  record = cls(
  service=service,
  status_code=status_code,
  request_url=request_url,
- async_result_id=async_result_id
+ async_result_id=async_result_id,
+ task_result=task_result
  ).save()
  return record

zodiac/gateway/service_library.py

@@ -5,11 +5,13 @@
 from .tasks import queue_request
 from .views_library import render_service_path
 
-
 def send_service_request(service, request={}):
  headers = {}
  files = {}
 
+ if service.has_active_task:
+ return False
+
  if request:
  files = request.FILES
 
@@ -46,8 +48,10 @@ def send_service_request(service, request={}):
  headers=headers,
  data=data,
  files=files,
- params={'post_service_url': render_service_path(service.post_service)}
+ params={'post_service_url': render_service_path(service.post_service)},
+ service_id=service.pk,
  )
+
  return async_result.id
 
 

zodiac/gateway/signals.py

@@ -0,0 +1,34 @@
+from celery.signals import task_prerun, task_postrun
+from django_celery_results.models import TaskResult
+from .models import ServiceRegistry, RequestLog
+
+@task_prerun.connect
+def on_task_prerun(task_id=None, task=None, *args, **kwargs):
+
+ # Mark service as active
+ if 'service_id' in kwargs['kwargs']:
+ service = ServiceRegistry.objects.get(pk=kwargs['kwargs']['service_id'])
+ service.has_active_task = True
+ service.save()
+
+@task_postrun.connect
+def on_task_postrun(task_id=None, task=None, retval=None, state=None, *args, **kwargs):
+
+ # Mark service as inactive
+ def update_service(kwargs):
+ if 'service_id' in kwargs['kwargs']:
+ service = ServiceRegistry.objects.get(pk=kwargs['kwargs']['service_id'])
+ service.has_active_task = False
+ service.save()
+ return service
+ return None
+
+ # Add result to request log
+ task_result = TaskResult.objects.get(task_id=task_id)
+ request_log = RequestLog.create(
+ service=update_service(kwargs),
+ status_code=None,
+ request_url=kwargs['args'][1],
+ async_result_id=task_id,
+ task_result=task_result
+ )

zodiac/gateway/static/css/styles.css

@@ -5,3 +5,10 @@
  font-size: 40px;
  padding-top: 3px;
 }
+
+pre {
+ white-space: pre-wrap;
+ white-space: -moz-pre-wrap;
+ white-space: -pre-wrap;
+ white-space: -o-pre-wrap;
+}

zodiac/gateway/tasks.py

@@ -6,6 +6,7 @@
 
 from celery import shared_task, current_task
 
+from zodiac import settings
 from .models import ServiceRegistry
 from .views_library import render_service_path
 
@@ -20,31 +21,26 @@
 @shared_task()
 def queue_callbacks():
  completed = {'detail': {'callbacks': []}}
- for registry in ServiceRegistry.objects.filter(callback_service__isnull=False):
+ for registry in ServiceRegistry.objects.filter(callback_service__isnull=False, has_active_task=False).order_by('callback_service__modified_time')[:settings.MAX_SERVICES]:
  if registry.service_active(): # TODO: also check to see if last service run was okay
  callback = ServiceRegistry.objects.get(pk=registry.callback_service.pk)
- url = render_service_path(callback, '')
- r = queue_request.delay(
- 'post',
- url,
- headers={'Content-Type': 'application/json'},
- data=None,
- files=None,
- params={'post_service_url': render_service_path_url(callback)}
- )
- if r:
- completed['detail']['callbacks'].append({callback.name: r.id})
+ if not callback.has_active_task:
+ url = render_service_path(callback, '')
+ r = queue_request.delay(
+ 'post',
+ url,
+ headers={'Content-Type': 'application/json'},
+ data=None,
+ files=None,
+ params={'post_service_url': render_service_path(callback.post_service)},
+ service_id=callback.id
+ )
+ if r:
+ completed['detail']['callbacks'].append({callback.name: r.id})
  return completed
 
 
 @shared_task()
-def queue_request(method, url, headers, data, files, params):
+def queue_request(method, url, headers, data, files, params, service_id):
  r = method_map[method](url, headers=headers, data=data, files=files, params=params)
-
- # VALIDATE REsponse
- # check for json
- # if request OK
-
- # print(current_task.request.id, 'id of current task')
- # print(async_result_id)
- return r.text
+ return r.json()

zodiac/gateway/templates/gateway/applications_list_table.html

@@ -1,11 +1,11 @@
 <table class="table table-striped">
+ <thead>
+ <th>Name</th>
+ <th>Active?</th>
+ <th>Available at</th>
+ <th>Services</th>
+ </thead>
  <tbody>
- <thead>
- <th>Name</th>
- <th>Active?</th>
- <th>Available at</th>
- <th>Services</th>
- </thead>
  {% for application in object_list %}
  <tr>
  <td><a href="{{ application.get_absolute_url }}">{{ application.name }}</a></td>

zodiac/gateway/templates/gateway/results_detail.html

@@ -2,32 +2,32 @@
 
 {% block h1_title %}
  Result Detail
- <small>{{object.date_done}}</small>
+ <small>{{object.task_result.date_done}}</small>
 {% endblock %}
 
 {% block content %}
 <div class="row">
  <div class="col-md-12">
  <div class="box box-primary">
  <div class="box-body">
- <label>Identifier</label>
- <p>{{object.task_id}}</p>
- <label>Status</label>
- <p>{{object.status}}</p>
- {% if object.task_args %}
- <label>Arguments</label>
- {{object.task_args}}
- {% endif %}
- {% if object.task_kwargs %}
- <label>Keyword Arguments</label>
- {{object.task_kwargs}}
- {% endif %}
- <label>Result</label>
- <p><pre><code>{{ object.result }}</code></pre></p>
- {% if object.traceback %}
- <label>Traceback</label>
- <p><pre><code>{{object.traceback }}</code></pre></p>
- {% endif %}
+ <dl>
+  <dt>Identifier</dt>
+  <dd>{{object.async_result_id}}</dd>
+  <dt>Service</dt>
+  <dd><a href="{% url 'services-detail' pk=object.service.pk %}">{{object.service.name}}</a></dd>
+ <dt>URL</dt>
+ <dd>{{object.request_url}}</dd>
+  <dt>Arguments</dt>
+  <dd><pre>{{object.task_result.task_args}}</pre></dd>
+ <dt>Keyword Arguments</dt>
+ <dd><pre>{{object.task_result.task_kwargs}}</pre></dd>
+  <dt>Result</dt>
+  <dd><pre><code>{{ object.task_result.result }}</code></pre></dd>
+  {% if object.task_result.traceback %}
+  <dt>Traceback</dt>
+  <dd><pre><code>{{object.task_result.traceback }}</code></pre></dd>
+ {% endif %}
+ </dl>
  </div>
  </div>
  </div>

zodiac/gateway/templates/gateway/results_list.html

@@ -11,17 +11,17 @@
  <div class="box-body {% if object_list %}no-padding{% endif %}">
  <table id="results-table" class="table table-striped dataTable">
  <thead>
- <tr>
- <td>Identifier</td>
- <td>Date done</td>
- <td>Status</td>
- </tr>
+ <th>Identifier</th>
+ <th>Service</th>
+ <th>Result</th>
+ <th>Date</th>
  </thead>
  <tbody>
  <tr>
  <td></td>
  <td></td>
  <td></td>
+ <td></td>
  </tr>
  </tbody>
  </table>
@@ -36,11 +36,17 @@
  $(document).ready(function() {
  // $.fn.dataTable.moment( 'MMM D, YYYY h:mm A' );
  $('#results-table').DataTable({
+ "columnDefs": [
+ {
+ width: "50%",
+ targets: [2] }
+ ],
  'stateSave' : true,
  'paging' : true,
  'lengthChange': false,
  'searching' : true,
  'ordering' : true,
+ 'order' : [[ 3, 'desc' ]],
  'info' : true,
  'autoWidth' : true,
  'pageLength' : 25,