Skip to content

Commit

Permalink
rewrite jobs website
Browse files Browse the repository at this point in the history
  • Loading branch information
@Digenis
Digenis committed Nov 20, 2017
1 parent 5794a4b commit d78d0c9
Showing 1 changed file with 116 additions and 74 deletions.
190 changes: 116 additions & 74 deletions scrapyd/website.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from datetime import datetime
from datetime import datetime, timedelta

import socket

Expand All @@ -12,14 +12,6 @@
from six.moves.urllib.parse import urlparse


def microsec_trunc(timelike):
if hasattr(timelike, 'microsecond'):
ms = timelike.microsecond
else:
ms = timelike.microseconds
return timelike - datetime.timedelta(microseconds=ms)


class Root(resource.Resource):

def __init__(self, config, app):
Expand Down Expand Up @@ -107,6 +99,14 @@ def render_GET(self, txrequest):
return s.encode('utf-8')


def microsec_trunc(timelike):
if hasattr(timelike, 'microsecond'):
ms = timelike.microsecond
else:
ms = timelike.microseconds
return timelike - timedelta(microseconds=ms)


class Jobs(resource.Resource):

def __init__(self, root, local_items):
Expand All @@ -115,73 +115,115 @@ def __init__(self, root, local_items):
self.local_items = local_items

cancel_button = """
<form method='post' action='/cancel.json'>
<input type='hidden' name='project' value='{project}'/>"
<input type='hidden' name='job' value='{jobid}'/>"
<input type='submit' style='float: left;' value='Cancel'/>
<form method="post" action="/cancel.json">
<input type="hidden" name="project" value="{project}"/>
<input type="hidden" name="job" value="{jobid}"/>
<input type="submit" style="float: left;" value="Cancel"/>
</form>
""".format

def render(self, txrequest):
cols = 8
s = "<html><head><title>Scrapyd</title></head>"
s += "<body>"
s += "<h1>Jobs</h1>"
s += "<p><a href='..'>Go back</a></p>"
s += "<table border='1'>"
s += "<tr><th>Project</th><th>Spider</th><th>Job</th><th>PID</th><th>Start</th><th>Runtime</th><th>Finish</th><th>Log</th><th>Cancel</th>"
if self.local_items:
s += "<th>Items</th>"
cols += 1
s += "</tr>"
s += "<tr><th colspan='%s' style='background-color: #ddd'>Pending</th></tr>" % cols
for project, queue in self.root.poller.queues.items():
for m in queue.list():
s += "<tr>"
s += "<td>%s</td>" % project
s += "<td>%s</td>" % str(m['name'])
s += "<td>%s</td>" % str(m['_job'])
s += "<td/>" * (cols - 4)
s += '<td>'
if 'cancel.json' in self.root.children:
s += self.cancel_button(project=project, jobid=m['_job'])
s += "</td>"
s += "</tr>"
s += "<tr><th colspan='%s' style='background-color: #ddd'>Running</th></tr>" % cols
for p in self.root.launcher.processes.values():
s += "<tr>"
for a in ['project', 'spider', 'job', 'pid']:
s += "<td>%s</td>" % getattr(p, a)
s += "<td>%s</td>" % microsec_trunc(p.start_time)
s += "<td>%s</td>" % microsec_trunc(datetime.now() - p.start_time)
s += "<td/>" # Finish
s += "<td><a href='/logs/%s/%s/%s.log'>Log</a></td>" % (p.project, p.spider, p.job)
if self.local_items:
s += "<td><a href='/items/%s/%s/%s.jl'>Items</a></td>" % (p.project, p.spider, p.job)
s += '<td>'
if 'cancel.json' in self.root.children:
s += self.cancel_button(project=p.project, jobid=p.job)
s += '</td>'
s += "</tr>"
s += "<tr><th colspan='%s' style='background-color: #ddd'>Finished</th></tr>" % cols
for p in self.root.launcher.finished:
s += "<tr>"
for a in ['project', 'spider', 'job']:
s += "<td>%s</td>" % getattr(p, a)
s += "<td/>" # PID
s += "<td>%s</td>" % microsec_trunc(p.start_time)
s += "<td>%s</td>" % microsec_trunc(p.end_time - p.start_time)
s += "<td>%s</td>" % microsec_trunc(p.end_time)
s += "<td><a href='/logs/%s/%s/%s.log'>Log</a></td>" % (p.project, p.spider, p.job)
if self.local_items:
s += "<td><a href='/items/%s/%s/%s.jl'>Items</a></td>" % (p.project, p.spider, p.job)
s += "<td/>"
s += "</tr>"
s += "</table>"
s += "</body>"
s += "</html>"
header_cols = [
'Project', 'Spider',
'Job', 'PID',
'Start', 'Runtime', 'Finish',
'Log', 'Items',
'Cancel',
]

def gen_css(self):
css = [
'#jobs>thead td {text-align: center; font-weight: bold}',
'#jobs>tbody>tr:first-child {background-color: #eee}',
]
if not self.local_items:
col_idx = self.header_cols.index('Items') + 1
css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx)
if 'cancel.json' not in self.root.children:
col_idx = self.header_cols.index('Cancel') + 1
css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx)
return '\n'.join(css)

def prep_row(self, cells):
if not isinstance(cells, dict):
assert len(cells) == len(self.header_cols)
else:
cells = [cells.get(k) for k in self.header_cols]
cells = ['<td>%s</td>' % ('' if c is None else c) for c in cells]
return '<tr>%s</tr>' % ''.join(cells)

def prep_doc(self):
return (
'<html>'
'<head>'
'<title>Scrapyd</title>'
'<style type="text/css">' + self.gen_css() + '</style>'
'</head>'
'<body><h1>Jobs</h1>'
'<p><a href="..">Go back</a></p>'
+ self.prep_table() +
'</body>'
'</html>'
)

def prep_table(self):
return (
'<table id="jobs" border="1">'
'<thead>' + self.prep_row(self.header_cols) + '</thead>'
'<tbody>'
+ '<tr><th colspan="%d">Pending</th></tr>' % len(self.header_cols)
+ self.prep_tab_pending() +
'</tbody>'
'<tbody>'
+ '<tr><th colspan="%d">Running</th></tr>' % len(self.header_cols)
+ self.prep_tab_running() +
'</tbody>'
'<tbody>'
+ '<tr><th colspan="%d">Finished</th></tr>' % len(self.header_cols)
+ self.prep_tab_finished() +
'</tbody>'
'</table>'
)

def prep_tab_pending(self):
return '\n'.join(
self.prep_row(dict(
Project=project, Spider=m['name'], Job=m['_job'],
Cancel=self.cancel_button(project=project, jobid=m['_job'])
))
for project, queue in self.root.poller.queues.items()
for m in queue.list()
)

def prep_tab_running(self):
return '\n'.join(
self.prep_row(dict(
Project=p.project, Spider=p.spider,
Job=p.job, PID=p.pid,
Start=microsec_trunc(p.start_time),
Runtime=microsec_trunc(datetime.now() - p.start_time),
Log='<a href="/logs/%s/%s/%s.log">Log</a>' % (p.project, p.spider, p.job),
Items='<a href="/items/%s/%s/%s.jl">Items</a>' % (p.project, p.spider, p.job),
Cancel=self.cancel_button(project=p.project, jobid=p.job)
))
for p in self.root.launcher.processes.values()
)

def prep_tab_finished(self):
return '\n'.join(
self.prep_row(dict(
Project=p.project, Spider=p.spider,
Job=p.job,
Start=microsec_trunc(p.start_time),
Runtime=microsec_trunc(p.end_time - p.start_time),
Finish=microsec_trunc(p.end_time),
Log='<a href="/logs/%s/%s/%s.log">Log</a>' % (p.project, p.spider, p.job),
Items='<a href="/items/%s/%s/%s.jl">Items</a>' % (p.project, p.spider, p.job),
))
for p in self.root.launcher.finished
)

def render(self, txrequest):
doc = self.prep_doc()
txrequest.setHeader('Content-Type', 'text/html; charset=utf-8')
txrequest.setHeader('Content-Length', len(s))

return s.encode('utf-8')
txrequest.setHeader('Content-Length', len(doc))
return doc.encode('utf-8')

0 comments on commit d78d0c9

Please sign in to comment.