Bug 1308712 - Add Spark job detail view

This also fixes a few bugs in the spark job startup code and in the
cluster-detail template.

atmo/jobs/models.py

@@ -1,5 +1,6 @@
-from datetime import datetime, timedelta
+from datetime import timedelta
 
+from django.core.urlresolvers import reverse
 from django.db import models
 from django.contrib.auth.models import User
 from django.utils import timezone
@@ -77,7 +78,7 @@ def is_expired(self, at_time=None):
             return False  # job isn't even running at the moment
         if at_time is None:
             at_time = timezone.now()
-        if self.last_run_date + timedelta(hours=self.job_timeout) >= at_time:
+        if self.last_run_date and self.last_run_date + timedelta(hours=self.job_timeout) >= at_time:
             return True  # current job run expired
         return False
 
@@ -134,10 +135,12 @@ def delete(self, *args, **kwargs):
     @classmethod
     def step_all(cls):
         """Run all the scheduled tasks that are supposed to run."""
-        now = datetime.now()
         for spark_join in cls.objects.all():
-            if spark_join.should_run(now):
+            if spark_join.should_run():
                 spark_join.run()
                 spark_join.save()
-            if spark_join.is_expired(now):
+            if spark_join.is_expired():
                 spark_join.delete()
+
+    def get_absolute_url(self):
+        return reverse('jobs-detail', kwargs={'id': self.id})

atmo/jobs/urls.py

@@ -5,4 +5,5 @@
     url(r'^new/', views.new_spark_job, name='jobs-new'),
     url(r'^edit/', views.edit_spark_job, name='jobs-edit'),
     url(r'^delete/', views.delete_spark_job, name='jobs-delete'),
+    url(r'^(?P<id>[0-9]+)/$', views.detail_spark_job, name='jobs-detail'),
 ]

atmo/jobs/views.py

@@ -2,10 +2,11 @@
 from django.views.decorators.http import require_POST
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponseBadRequest
-from django.shortcuts import redirect
+from django.shortcuts import redirect, get_object_or_404, render
 
 from session_csrf import anonymous_csrf
 
+from .models import SparkJob
 from . import forms
 
 
@@ -43,3 +44,9 @@ def delete_spark_job(request):
         return HttpResponseBadRequest(form.errors.as_json(escape_html=True))
     form.save()  # this will also delete the job for us
     return redirect("/")
+
+
+@login_required
+def detail_spark_job(request, id):
+    job = get_object_or_404(SparkJob, created_by=request.user, pk=id)
+    return render(request, 'atmo/detail-spark-job.html', context={'job': job})

atmo/templates/atmo/dashboard.html

@@ -25,7 +25,7 @@ <h2>Launch a Spark Cluster</h2>
         {% for cluster in active_clusters %}
         <tr>
           <td class="hidden">{{ cluster.id }}</td>
-          <td><a href="{% url "clusters-detail" id=cluster.id %}">{{ cluster.identifier }}</a></td>
+          <td><a href="{{ cluster.get_absolute_url }}">{{ cluster.identifier }}</a></td>
           <td>{{ cluster.size }}</td>
           <td>{{ cluster.start_date }}</td>
           <td>{{ cluster.most_recent_status }}</td>
@@ -60,7 +60,7 @@ <h2>Schedule a Spark Job</h2>
         {% for spark_job in user_spark_jobs %}
         <tr>
           <td class="hidden">{{ spark_job.id }}</td>
-          <td>{{ spark_job.identifier }}</td>
+          <td><a href="{{ spark_job.get_absolute_url }}">{{ spark_job.identifier }}</a></td>
           <td>{{ spark_job.result_visibility }}</td>
           <td>{{ spark_job.size }}</td>
           <td>{{ spark_job.interval_in_hours }}</td>

atmo/templates/atmo/detail-cluster.html

@@ -1,5 +1,8 @@
 {% extends "atmo/base.html" %}
 {% block content %}
+
+{% block head_title %}Cluster {{ cluster.identifier }}{% endblock %}
+
 <h1>Cluster {{ cluster.identifier }}</h1>
 <p class="lead">Summary:</p>
 <div class="row">
@@ -27,7 +30,7 @@ <h1>Cluster {{ cluster.identifier }}</h1>
 {% if cluster.is_active %}
   <div class="row">
     <div class="col-md-2"><strong>Terminates at:</strong></div>
-    <div class="col-md-10">{{cluster.end_date}}</div>
+    <div class="col-md-10">{{cluster.end_date|date:"SHORT_DATETIME_FORMAT" }}</div>
   </div>
   <p>
     <form action="{% url 'clusters-delete' %}" method="POST" enctype="multipart/form-data">

atmo/templates/atmo/detail-spark-job.html

@@ -0,0 +1,50 @@
+{% extends "atmo/base.html" %}
+
+{% block head_title %}Spark job {{ job.identifier }}{% endblock %}
+
+{% block content %}
+<h1>Spark Job {{ job.identifier }}</h1>
+<p class="lead">Summary:</p>
+<div class="row">
+  <div class="col-md-2"><strong>Notebook S3 key:</strong></div>
+  <div class="col-md-10">{{ job.notebook_s3_key }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Result visibility:</strong></div>
+  <div class="col-md-10">{{ job.result_visibility }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Size:</strong></div>
+  <div class="col-md-10">{{ job.size }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Interval in hours:</strong></div>
+  <div class="col-md-10">{{ job.interval_in_hours }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Timeout:</strong></div>
+  <div class="col-md-10">{{ job.job_timeout }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Start date:</strong></div>
+  <div class="col-md-10">{{ job.start_date|date:"SHORT_DATETIME_FORMAT" }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>End date:</strong></div>
+  <div class="col-md-10">{{ job.end_date|date:"SHORT_DATETIME_FORMAT" }}</div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Is enabled:</strong></div>
+  <div class="col-md-10">
+    {% if job.is_enabled %}
+    <span class="glyphicon glyphicon-ok text-success" aria-hidden="true"></span>
+    {% else %}
+    <span class="glyphicon glyphicon-remove text-danger" aria-hidden="true"></span>
+    {% endif %}
+  </div>
+</div>
+<div class="row">
+  <div class="col-md-2"><strong>Last run date:</strong></div>
+  <div class="col-md-10">{{ job.last_run_date|timesince }}</div>
+</div>
+{% endblock content %}

atmo/utils/scheduling.py

@@ -72,7 +72,7 @@ def spark_job_run(user_email, identifier, notebook_uri, result_is_public, size,
                 'Path': 's3://{}/bootstrap/telemetry.sh'.format(
                     settings.AWS_CONFIG['SPARK_EMR_BUCKET']
                 ),
-                'Args': ['--timeout', job_timeout]
+                'Args': ['--timeout', str(job_timeout)]
             }
         }],
         Tags=[