From 6c219c2a025df7b6fa01273dac57dfddbd245f5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 14 Nov 2017 19:03:47 +0200 Subject: [PATCH 1/3] use function to truncate microseconds in job table --- scrapyd/website.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scrapyd/website.py b/scrapyd/website.py index 6f5e189c..b5a30bf8 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -11,6 +11,15 @@ from six.moves.urllib.parse import urlparse + +def microsec_trunc(timelike): + if hasattr(timelike, 'microsecond'): + ms = timelike.microsecond + else: + ms = timelike.microseconds + return timelike - datetime.timedelta(microseconds=ms) + + class Root(resource.Resource): def __init__(self, config, app): @@ -129,8 +138,8 @@ def render(self, txrequest): s += "" for a in ['project', 'spider', 'job', 'pid']: s += "%s" % getattr(p, a) - s += "%s" % p.start_time.replace(microsecond=0) - s += "%s" % (datetime.now().replace(microsecond=0) - p.start_time.replace(microsecond=0)) + s += "%s" % microsec_trunc(p.start_time) + s += "%s" % microsec_trunc(datetime.now() - p.start_time) s += "" s += "Log" % (p.project, p.spider, p.job) if self.local_items: @@ -142,9 +151,9 @@ def render(self, txrequest): for a in ['project', 'spider', 'job']: s += "%s" % getattr(p, a) s += "" - s += "%s" % p.start_time.replace(microsecond=0) - s += "%s" % (p.end_time.replace(microsecond=0) - p.start_time.replace(microsecond=0)) - s += "%s" % p.end_time.replace(microsecond=0) + s += "%s" % microsec_trunc(p.start_time) + s += "%s" % microsec_trunc(p.end_time - p.start_time) + s += "%s" % microsec_trunc(p.end_time) s += "Log" % (p.project, p.spider, p.job) if self.local_items: s += "Items" % (p.project, p.spider, p.job) From 5794a4b62a88b5bd2424852896bab0ecb63c57ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Tue, 14 Nov 2017 20:10:07 +0200 Subject: [PATCH 2/3] add cancel form button in website --- docs/news.rst | 6 ++++++ scrapyd/website.py | 27 +++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b6d006a3..993fb00f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -7,6 +7,12 @@ Release notes ----- *Unreleased* + +Added +~~~~~ + +- Jobs website shortcut to cancel a job using the cancel.json webservice. + Removed ~~~~~~~ diff --git a/scrapyd/website.py b/scrapyd/website.py index b5a30bf8..1bdaeb5c 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -106,6 +106,7 @@ def render_GET(self, txrequest): """ % vars return s.encode('utf-8') + class Jobs(resource.Resource): def __init__(self, root, local_items): @@ -113,6 +114,14 @@ def __init__(self, root, local_items): self.root = root self.local_items = local_items + cancel_button = """ +
+ " + " + +
+ """.format + def render(self, txrequest): cols = 8 s = "Scrapyd" @@ -120,10 +129,10 @@ def render(self, txrequest): s += "

Jobs

" s += "

Go back

" s += "" - s += "" + s += "" if self.local_items: s += "" - cols = 9 + cols += 1 s += "" s += "" % cols for project, queue in self.root.poller.queues.items(): @@ -132,6 +141,11 @@ def render(self, txrequest): s += "" % project s += "" % str(m['name']) s += "" % str(m['_job']) + s += "" s += "" s += "" % cols for p in self.root.launcher.processes.values(): @@ -140,23 +154,28 @@ def render(self, txrequest): s += "" % getattr(p, a) s += "" % microsec_trunc(p.start_time) s += "" % microsec_trunc(datetime.now() - p.start_time) - s += "" + s += "" % (p.project, p.spider, p.job) if self.local_items: s += "" % (p.project, p.spider, p.job) + s += '' s += "" s += "" % cols for p in self.root.launcher.finished: s += "" for a in ['project', 'spider', 'job']: s += "" % getattr(p, a) - s += "" + s += "" % microsec_trunc(p.start_time) s += "" % microsec_trunc(p.end_time - p.start_time) s += "" % microsec_trunc(p.end_time) s += "" % (p.project, p.spider, p.job) if self.local_items: s += "" % (p.project, p.spider, p.job) + s += "" s += "
ProjectSpiderJobPIDStartRuntimeFinishLog
ProjectSpiderJobPIDStartRuntimeFinishLogCancelItems
Pending
%s%s%s" * (cols - 4) + s += '' + if 'cancel.json' in self.root.children: + s += self.cancel_button(project=project, jobid=m['_job']) + s += "
Running
%s%s%s" # Finish s += "LogItems' + if 'cancel.json' in self.root.children: + s += self.cancel_button(project=p.project, jobid=p.job) + s += '
Finished
%s" # PID s += "%s%s%sLogItems" s += "
" s += "" From d78d0c94a2d7c6fbabfe728eeb46003a93b3abf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=9D=CE=B9=CE=BA=CF=8C=CE=BB=CE=B1=CE=BF=CF=82-=CE=94?= =?UTF-8?q?=CE=B9=CE=B3=CE=B5=CE=BD=CE=AE=CF=82=20=CE=9A=CE=B1=CF=81=CE=B1?= =?UTF-8?q?=CE=B3=CE=B9=CE=AC=CE=BD=CE=BD=CE=B7=CF=82?= Date: Wed, 15 Nov 2017 14:47:33 +0200 Subject: [PATCH 3/3] rewrite jobs website --- scrapyd/website.py | 190 +++++++++++++++++++++++++++------------------ 1 file changed, 116 insertions(+), 74 deletions(-) diff --git a/scrapyd/website.py b/scrapyd/website.py index 1bdaeb5c..da271706 100644 --- a/scrapyd/website.py +++ b/scrapyd/website.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta import socket @@ -12,14 +12,6 @@ from six.moves.urllib.parse import urlparse -def microsec_trunc(timelike): - if hasattr(timelike, 'microsecond'): - ms = timelike.microsecond - else: - ms = timelike.microseconds - return timelike - datetime.timedelta(microseconds=ms) - - class Root(resource.Resource): def __init__(self, config, app): @@ -107,6 +99,14 @@ def render_GET(self, txrequest): return s.encode('utf-8') +def microsec_trunc(timelike): + if hasattr(timelike, 'microsecond'): + ms = timelike.microsecond + else: + ms = timelike.microseconds + return timelike - timedelta(microseconds=ms) + + class Jobs(resource.Resource): def __init__(self, root, local_items): @@ -115,73 +115,115 @@ def __init__(self, root, local_items): self.local_items = local_items cancel_button = """ -
- " - " - + + + +
""".format - def render(self, txrequest): - cols = 8 - s = "Scrapyd" - s += "" - s += "

Jobs

" - s += "

Go back

" - s += "" - s += "" - if self.local_items: - s += "" - cols += 1 - s += "" - s += "" % cols - for project, queue in self.root.poller.queues.items(): - for m in queue.list(): - s += "" - s += "" % project - s += "" % str(m['name']) - s += "" % str(m['_job']) - s += "" - s += "" - s += "" % cols - for p in self.root.launcher.processes.values(): - s += "" - for a in ['project', 'spider', 'job', 'pid']: - s += "" % getattr(p, a) - s += "" % microsec_trunc(p.start_time) - s += "" % microsec_trunc(datetime.now() - p.start_time) - s += "" % (p.project, p.spider, p.job) - if self.local_items: - s += "" % (p.project, p.spider, p.job) - s += '' - s += "" - s += "" % cols - for p in self.root.launcher.finished: - s += "" - for a in ['project', 'spider', 'job']: - s += "" % getattr(p, a) - s += "" % microsec_trunc(p.start_time) - s += "" % microsec_trunc(p.end_time - p.start_time) - s += "" % microsec_trunc(p.end_time) - s += "" % (p.project, p.spider, p.job) - if self.local_items: - s += "" % (p.project, p.spider, p.job) - s += "" - s += "
ProjectSpiderJobPIDStartRuntimeFinishLogCancelItems
Pending
%s%s%s" * (cols - 4) - s += '' - if 'cancel.json' in self.root.children: - s += self.cancel_button(project=project, jobid=m['_job']) - s += "
Running
%s%s%s" # Finish - s += "LogItems' - if 'cancel.json' in self.root.children: - s += self.cancel_button(project=p.project, jobid=p.job) - s += '
Finished
%s" # PID - s += "%s%s%sLogItems" - s += "
" - s += "" - s += "" + header_cols = [ + 'Project', 'Spider', + 'Job', 'PID', + 'Start', 'Runtime', 'Finish', + 'Log', 'Items', + 'Cancel', + ] + + def gen_css(self): + css = [ + '#jobs>thead td {text-align: center; font-weight: bold}', + '#jobs>tbody>tr:first-child {background-color: #eee}', + ] + if not self.local_items: + col_idx = self.header_cols.index('Items') + 1 + css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx) + if 'cancel.json' not in self.root.children: + col_idx = self.header_cols.index('Cancel') + 1 + css.append('#jobs>*>tr>*:nth-child(%d) {display: none}' % col_idx) + return '\n'.join(css) + + def prep_row(self, cells): + if not isinstance(cells, dict): + assert len(cells) == len(self.header_cols) + else: + cells = [cells.get(k) for k in self.header_cols] + cells = ['%s' % ('' if c is None else c) for c in cells] + return '%s' % ''.join(cells) + + def prep_doc(self): + return ( + '' + '' + 'Scrapyd' + '' + '' + '

Jobs

' + '

Go back

' + + self.prep_table() + + '' + '' + ) + + def prep_table(self): + return ( + '' + '' + self.prep_row(self.header_cols) + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_pending() + + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_running() + + '' + '' + + '' % len(self.header_cols) + + self.prep_tab_finished() + + '' + '
Pending
Running
Finished
' + ) + + def prep_tab_pending(self): + return '\n'.join( + self.prep_row(dict( + Project=project, Spider=m['name'], Job=m['_job'], + Cancel=self.cancel_button(project=project, jobid=m['_job']) + )) + for project, queue in self.root.poller.queues.items() + for m in queue.list() + ) + + def prep_tab_running(self): + return '\n'.join( + self.prep_row(dict( + Project=p.project, Spider=p.spider, + Job=p.job, PID=p.pid, + Start=microsec_trunc(p.start_time), + Runtime=microsec_trunc(datetime.now() - p.start_time), + Log='Log' % (p.project, p.spider, p.job), + Items='Items' % (p.project, p.spider, p.job), + Cancel=self.cancel_button(project=p.project, jobid=p.job) + )) + for p in self.root.launcher.processes.values() + ) + + def prep_tab_finished(self): + return '\n'.join( + self.prep_row(dict( + Project=p.project, Spider=p.spider, + Job=p.job, + Start=microsec_trunc(p.start_time), + Runtime=microsec_trunc(p.end_time - p.start_time), + Finish=microsec_trunc(p.end_time), + Log='Log' % (p.project, p.spider, p.job), + Items='Items' % (p.project, p.spider, p.job), + )) + for p in self.root.launcher.finished + ) + def render(self, txrequest): + doc = self.prep_doc() txrequest.setHeader('Content-Type', 'text/html; charset=utf-8') - txrequest.setHeader('Content-Length', len(s)) - - return s.encode('utf-8') + txrequest.setHeader('Content-Length', len(doc)) + return doc.encode('utf-8')