diff --git a/scrapyd/website.py b/scrapyd/website.py index 1bdaeb5c..da271706 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta import socket @@ -12,14 +12,6 @@ from six.moves.urllib.parse import urlparse -def microsec_trunc(timelike): - if hasattr(timelike, 'microsecond'): - ms = timelike.microsecond - else: - ms = timelike.microseconds - return timelike - datetime.timedelta(microseconds=ms) - - class Root(resource.Resource): def __init__(self, config, app): @@ -107,6 +99,14 @@ def render_GET(self, txrequest): return s.encode('utf-8') +def microsec_trunc(timelike): + if hasattr(timelike, 'microsecond'): + ms = timelike.microsecond + else: + ms = timelike.microseconds + return timelike - timedelta(microseconds=ms) + + class Jobs(resource.Resource): def __init__(self, root, local_items): @@ -115,73 +115,115 @@ def __init__(self, root, local_items): self.local_items = local_items cancel_button = """ -
- " - " - + + + +
""".format - def render(self, txrequest): - cols = 8 - s = "Scrapyd" - s += "" - s += "

Jobs

" - s += "

Go back

" - s += "" - s += "" - if self.local_items: - s += "" - cols += 1 - s += "" - s += "" % cols - for project, queue in self.root.poller.queues.items(): - for m in queue.list(): - s += "" - s += "" % project - s += "" % str(m['name']) - s += "" % str(m['_job']) - s += "" - s += "" - s += "" % cols - for p in self.root.launcher.processes.values(): - s += "" - for a in ['project', 'spider', 'job', 'pid']: - s += "" % getattr(p, a) - s += "" % microsec_trunc(p.start_time) - s += "" % microsec_trunc(datetime.now() - p.start_time) - s += "" % (p.project, p.spider, p.job) - if self.local_items: - s += "" % (p.project, p.spider, p.job) - s += '' - s += "" - s += "" % cols - for p in self.root.launcher.finished: - s += "" - for a in ['project', 'spider', 'job']: - s += "" % getattr(p, a) - s += "" % microsec_trunc(p.start_time) - s += "" % microsec_trunc(p.end_time - p.start_time) - s += "" % microsec_trunc(p.end_time) - s += "" % (p.project, p.spider, p.job) - if self.local_items: - s += "" % (p.project, p.spider, p.job) - s += "" - s += "
ProjectSpiderJobPIDStartRuntimeFinishLogCancelItems
Pending
%s%s%s" * (cols - 4) - s += '' - if 'cancel.json' in self.root.children: - s += self.cancel_button(project=project, jobid=m['_job']) - s += "
Running
%s%s%s" # Finish - s += "LogItems' - if 'cancel.json' in self.root.children: - s += self.cancel_button(project=p.project, jobid=p.job) - s += '
Finished
%s" # PID - s += "%s%s%sLogItems" - s += "
" - s += "" - s += "" + header_cols = [ + 'Project', 'Spider', + 'Job', 'PID', + 'Start', 'Runtime', 'Finish', + 'Log', 'Items', + 'Cancel', + ] + + def gen_css(self): + css = [ + '#jobs>thead td {text-align: center; font-weight: bold}', + '#jobs>tbody>tr:first-child {background-color: #eee}', + ] + if not self.local_items: + col_idx = self.header_cols.index('Items') + 1 + css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx) + if 'cancel.json' not in self.root.children: + col_idx = self.header_cols.index('Cancel') + 1 + css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx) + return '\n'.join(css) + + def prep_row(self, cells): + if not isinstance(cells, dict): + assert len(cells) == len(self.header_cols) + else: + cells = [cells.get(k) for k in self.header_cols] + cells = ['%s' % ('' if c is None else c) for c in cells] + return '%s' % ''.join(cells) + + def prep_doc(self): + return ( + '' + '' + 'Scrapyd' + '' + '' + '

Jobs

' + '

Go back

' + + self.prep_table() + + '' + '' + ) + + def prep_table(self): + return ( + '' + '' + self.prep_row(self.header_cols) + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_pending() + + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_running() + + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_finished() + + '' + '
Pending
Running
Finished
' + ) + + def prep_tab_pending(self): + return '\n'.join( + self.prep_row(dict( + Project=project, Spider=m['name'], Job=m['_job'], + Cancel=self.cancel_button(project=project, jobid=m['_job']) + )) + for project, queue in self.root.poller.queues.items() + for m in queue.list() + ) + + def prep_tab_running(self): + return '\n'.join( + self.prep_row(dict( + Project=p.project, Spider=p.spider, + Job=p.job, PID=p.pid, + Start=microsec_trunc(p.start_time), + Runtime=microsec_trunc(datetime.now() - p.start_time), + Log='Log' % (p.project, p.spider, p.job), + Items='Items' % (p.project, p.spider, p.job), + Cancel=self.cancel_button(project=p.project, jobid=p.job) + )) + for p in self.root.launcher.processes.values() + ) + + def prep_tab_finished(self): + return '\n'.join( + self.prep_row(dict( + Project=p.project, Spider=p.spider, + Job=p.job, + Start=microsec_trunc(p.start_time), + Runtime=microsec_trunc(p.end_time - p.start_time), + Finish=microsec_trunc(p.end_time), + Log='Log' % (p.project, p.spider, p.job), + Items='Items' % (p.project, p.spider, p.job), + )) + for p in self.root.launcher.finished + ) + def render(self, txrequest): + doc = self.prep_doc() txrequest.setHeader('Content-Type', 'text/html; charset=utf-8') - txrequest.setHeader('Content-Length', len(s)) - - return s.encode('utf-8') + txrequest.setHeader('Content-Length', len(doc)) + return doc.encode('utf-8')