-
Notifications
You must be signed in to change notification settings - Fork 3
/
crashdb_ui.py.cgi
executable file
·388 lines (347 loc) · 15.8 KB
/
crashdb_ui.py.cgi
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
#!/usr/bin/python3
import os
import sys
from pprint import pprint
import psycopg2
import cgi
import cgitb
import html
from mycrashanalyzer import add_known_crash
import mymaloo_bugreporter
cgitb.enable()
print("Content-type: text/html")
print()
dbconn = psycopg2.connect(dbname="crashinfo", user="crashinfo", password="blah", host="localhost")
def xstr(s):
if s is None:
return ''
return str(s)
basenameurl = os.path.basename(os.environ['REQUEST_URI'])
index = basenameurl.find('?')
if index != -1:
basenameurl = basenameurl[:index]
def is_delete_allowed():
return not "external" in os.environ['REQUEST_URI']
def newreport_rows_to_table(rows):
REPORTS = ""
for row in rows:
REPORTS += '<tr><td>%d</td>' % (row[0])
REPORTS += '<td><a href="' + basenameurl + '?newid=%d">%s</a></td><td>%s</td>' % (row[0], html.escape(row[1]), html.escape(xstr(row[2])))
REPORTS += '<td>%s</td>' % (row[3].replace('\n', '<br>'))
REPORTS += '<td>' + str(row[4]) + '</td>'
REPORTS += '<td>' + str(row[5]) + '</td></tr>'
return REPORTS
def print_new_crashes(dbconn, form):
template = """<html><head><title>New crash reports</title></head>
<body>
<H2>List of untriaged crash reports (top {COUNT})</H2>
<table border=1>
<tr><th>ID</th><th>Reason</th><th>Crashing Function</th><th>Backtrace</th><th>Reports Count</th><th>Last hit</th></tr>
{REPORTS}
</table>
</body>
</html>
"""
count = 20
sort = "last_seen"
try:
if form and form.getfirst("count"):
count = int(form.getfirst("count"))
if form and form.getfirst("sort"):
if form.getfirst("sort") == "count":
sort = "hitcounts"
except:
pass
REPORTS=""
try:
cur = dbconn.cursor()
cur.execute("SELECT new_crashes.id, new_crashes.reason, new_crashes.func, new_crashes.backtrace, count(triage.newcrash_id) as hitcounts, max(triage.created_at) as last_seen from new_crashes, triage where new_crashes.id = triage.newcrash_id group by new_crashes.id ORDER BY " + sort + " DESC, hitcounts desc LIMIT %s", (count,))
rows = cur.fetchall()
REPORTS = newreport_rows_to_table(rows)
cur.close()
except psycopg2.DatabaseError as e:
REPORTS = "Database Error"
print(e)
pass
all_items = {'REPORTS': REPORTS, 'BASENAMEURL':basenameurl, 'COUNT':count}
return template.format(**all_items)
def examine_one_new_crash(dbconn, newid_str):
if not newid_str.isdigit():
return "Error! newid must be a number"
newid = int(newid_str)
template = """<html><head><title>Edit new crash report</title></head>
<body>
<H2>Editing crashreport #{NEWCRASHID}</H2>
<table border=1>
<form method=\"post\" action=\"{BASENAMEURL}\">
<tr><th>Reason</th><th>Crashing Function</th><th>Where to cut Backtrace</th><th>Reports Count</th></tr>
{REPORT}
</table>
<h2>Added fields:</h2>
<table>
<tr><td>Match messages in logs<br>(every line would be required to be present in log output<br>Copy from \"<b>Messages before crash</b>\" column below):</td><td><textarea name=\"inlogs\" rows=4 cols=50></textarea></td></tr>
<tr><td>Match messages in full crash<br>(every line would be required to be present in crash log output<br>Copy from \"<b>Full Crash</b>\" column below):</td><td><textarea name=\"infullbt\" rows=4 cols=50></textarea></td></tr>
<tr><td>Limit to a test:<br>(Copy from below \"Failing text\"):</td><td><input type=\"text\" name=\"testline\" size=50/></td></tr>
<tr><td>Delete these reports as invalid (real bug in review or some such)</td><td><input type=\"checkbox\" name=\"deletereport\" value=\"yes\"></td></tr>
<tr><td>Bug or comment:</td><td><input type=\"text\" name=\"bugdescription\" size=20 maxLength=20/></td></tr>
<tr><td>Extra info:</td><td><input type=\"text\" name=\"extrainfo\" size=60/></td></tr>
</table>
<input type=\"hidden\" name=\"newid\" value=\"{NEWCRASHID}\"/>
<input type=\"submit\" name=\"newconvert_submit\" value=\"Add to Known bugs\"/>
</form>
<h2>Failures list (last 100):</h2>
<table border=1>
<tr><th>Failing Test</th><th>Full Crash</th><th>Messages before crash</th><th>Comment</th></tr>
{TRIAGE}
</table>
<a href=\"{BASENAMEURL}\">Return to new crashes list</a>
</body>
</html>
"""
REPORTS = ""
try:
cur = dbconn.cursor()
cur.execute("SELECT new_crashes.id, new_crashes.reason, new_crashes.func, new_crashes.backtrace, count(triage.newcrash_id) as hitcounts from new_crashes, triage where new_crashes.id = triage.newcrash_id and new_crashes.id = %s group by new_crashes.id order by hitcounts desc", [newid])
if cur.rowcount != 1:
return "Error! No such element!"
row = cur.fetchone()
cur.close()
except psycopg2.DatabaseError as e:
print("db error")
print(e)
pass
else:
REPORTS += '<tr><td>%s</td>' % (html.escape(xstr(row[1])))
REPORTS += '<td>%s</td>' % (html.escape(xstr(row[2])))
REPORTS += '<td>'
for idx, btline in enumerate(row[3].splitlines()):
REPORTS += '<input type="radio" name="btline" value="cutat%d"/>%s<br>' % (idx, btline)
REPORTS += '</td>'
REPORTS += '<td>' + str(row[4]) + '</td></tr>'
TRIAGE = ""
try:
cur = dbconn.cursor()
# DISTINCT ON (testline) order by testline
cur.execute("SELECT id, testline, fullcrash, testlogs, link FROM triage where newcrash_id = %s order by created_at desc LIMIT 100", [newid])
rows = cur.fetchall()
cur.close()
if not rows:
return "Error! No actual reports for this id!"
except psycopg2.DatabaseError as e:
print("db error")
print(e)
pass
else:
for row in rows:
linktext = ""
if "http" in row[4]:
TRIAGE += '<tr><td><a href="%s">%s</a></td>' % (row[4], html.escape(xstr(row[1])))
linktext = '<a href="%s">Link to test</a>' % (row[4])
else:
TRIAGE += '<tr><td>%s</td>' % (html.escape(xstr(row[1])))
linktext = "Externally reported by " + html.escape(xstr(row[4]))
TRIAGE += '<td><div style="overflow: auto; width:30vw; height:300px;">%s</div></td>' % (html.escape(row[2]).replace('\n', '<br>'))
TRIAGE += '<td><div style="overflow: auto; width:50vw; height:300px;">%s</div></td>' % (html.escape(xstr(row[3])).replace('\n', '<br>'))
TRIAGE += "<td>%s</td</tr>" % (linktext)
all_items = {'NEWCRASHID':newid_str, 'REPORT': REPORTS, 'TRIAGE':TRIAGE, 'BASENAMEURL':basenameurl}
return template.format(**all_items)
def convert_new_crash(dbconn, form):
template="""<html><head><title>Converting new crash report</title></head>
<body>
<H2>Converting crashreport #{NEWCRASHID}</H2>
<table border=1>
<form method="post" action=\"#{BASENAMEURL}\">
<h2>Matched {TRACECOUNT} crash traces:</h2>
<tr><th>ID</th><th>Crash Reason</th><th>Crashing Function</th><th>Matched Backtrace</th><th>Matched Reports Count</th><th>Last report time</th></tr>
{REPORTS}
</table>
<textarea name=\"inlogs\" style=\"display:none;\" readonly>{INLOGS}</textarea>
<textarea name=\"infullbt\" style=\"display:none;\" readonly>{INFULLBT}</textarea>
<input type=\"hidden\" name=\"testline\" value=\"{TESTLINE}\"/>
<input type=\"hidden\" name=\"bugdescription\" value=\"{BUGDESCRIPTION}\"/>
<input type=\"hidden\" name=\"extrainfo\" value=\"{EXTRAINFO}\"/>
<input type=\"hidden\" name=\"newid\" value=\"{NEWCRASHID}\"/>
<input type=\"hidden\" name=\"btline\" value=\"{BTLINE}\"/>
<input type=\"hidden\" name=\"deletereport\" value=\"{DELETEREPORT}\"/>
<input type=\"hidden\" name=\"confirm\" value=\"yes\"/>
<input type=\"submit\" name=\"newconvert_submit\" value=\"Confirm Adding to Known bugs\"/>
</form>
<p>
<a href=\"{BASENAMEURL}?newid={NEWCRASHID}\">Return to view ID {NEWCRASHID}</a> | <a href=\"{BASENAMEURL}\">Return to new crashes list</a>
</body>
</html>
"""
newid_str = form.getfirst("newid")
if not newid_str or not newid_str.isdigit():
return "Error, not numeric id"
newid = int(newid_str)
testline = form.getfirst("testline", "")
bugdescription = form.getfirst("bugdescription", "")
extrainfo = form.getfirst("extrainfo", "")
inlogs = form.getfirst("inlogs", "")
infullbt = form.getfirst("infullbt", "")
confirm = form.getfirst("confirm", "")
btline_str = form.getfirst("btline", "")
deletereport = form.getfirst("deletereport", "")
if not bugdescription and deletereport != "yes":
return "Error! Bug description cannot be empty and not deleting"
if bugdescription and deletereport == "yes":
return "Error! Cannot assign bug numbers to reports you are deleting"
# Get backtrace info
try:
cur = dbconn.cursor()
cur.execute("SELECT backtrace, func, reason FROM new_crashes WHERE id=%s", [ newid ])
if cur.rowcount == 0:
return "No such id!"
row = cur.fetchone()
backtrace = row[0].splitlines()
func = xstr(row[1])
reason = row[2]
cur.execute("SELECT count(*) FROM triage WHERE newcrash_id=%s", [ newid ])
triagereports = cur.fetchone()[0]
cur.close()
except psycopg2.DatabaseError:
return "Db error"
if not btline_str:
btlines = len(backtrace)
else:
tmp = btline_str.replace("cutat", "")
if not tmp.isdigit():
return "Wrong bt cutat value"
btlines = int(tmp) + 1
if btlines > len(backtrace) + 1 or btlines < 2:
if btlines != len(backtrace): # for small backtraces it's ok
return "Cannot cut too low or too high"
backtrace = backtrace[:btlines]
# Now see how many matches we have
btline = '\n'.join(backtrace)
SELECTline = "SELECT new_crashes.id, new_crashes.reason, new_crashes.func, new_crashes.backtrace, count(triage.newcrash_id) as hitcount, max(triage.created_at) as last_seen FROM new_crashes, triage WHERE triage.newcrash_id=new_crashes.id AND new_crashes.reason=%s AND strpos(new_crashes.backtrace, %s) = 1"
SELECTvars = [ reason, btline ]
EXTRACONDS = ""
EXTRACONDvars = []
if func:
EXTRACONDS += " AND new_crashes.func=%s"
EXTRACONDvars.append(func)
if testline:
EXTRACONDS += " AND strpos(triage.testline, %s) > 0"
EXTRACONDvars.append(testline)
if inlogs:
inlogs_lines = []
for line in inlogs.splitlines():
line = line.strip()
EXTRACONDS += " AND strpos(triage.testlogs, %s) > 0"
EXTRACONDvars.append(line)
inlogs_lines.append(line)
inlogs_cleaned = '\n'.join(inlogs_lines)
else:
inlogs_cleaned = ""
if infullbt:
infullbt_lines = []
for line in infullbt.splitlines():
line = line.strip()
EXTRACONDS += " AND strpos(triage.fullcrash, %s) > 0"
EXTRACONDvars.append(line)
infullbt_lines.append(line)
infullbt_cleaned = '\n'.join(infullbt_lines)
else:
infullbt_cleaned = ""
SELECTline += EXTRACONDS
SELECTline += " group by new_crashes.id order by last_seen desc, hitcount desc"
SELECTvars += EXTRACONDvars
REPORTS = ""
try:
cur = dbconn.cursor()
cur.execute(SELECTline, SELECTvars)
if cur.rowcount == 0:
return "Cannot find anything matching: " + SELECTline + " " + str(SELECTvars)
TRACECOUNT = cur.rowcount
rows = cur.fetchall()
cur.close()
REPORTS = newreport_rows_to_table(rows)
except psycopg2.DatabaseError as e:
REPORTS = "DB Error " + str(e)
TRACECOUNT = 0
if confirm != "yes":
all_items = {'NEWCRASHID':newid_str, 'REPORTS': REPORTS, 'TRACECOUNT':TRACECOUNT, 'INLOGS':inlogs_cleaned, 'BUGDESCRIPTION':bugdescription, 'EXTRAINFO':extrainfo, 'TESTLINE':testline, 'INFULLBT':infullbt_cleaned, 'BTLINE':btline_str, 'DELETEREPORT':deletereport, 'BASENAMEURL':basenameurl}
return template.format(**all_items)
elif not is_delete_allowed():
return "Actual deleting on external scripts is disabled"
template = """<html><head><title>Converting new crash report</title></head>
<body>
<H2>Converting crashreport #{NEWCRASHID}</H2>
{MALOOREPORT}
<a href=\"{BASENAMEURL}\">Return to new crashes list</a>
</body>
</html>
"""
# Assemble array of newbug IDs affected in a form that postgres understands (1, 2,3, ...)
ids = []
for row in rows:
ids.append(str(row[0]))
NEWIDS = '(' + ', '.join(ids) + ')'
malooreport = ""
# This was our second pass, we now need to insert the data into known crashes
# Or if it was a delete request, don't create anything, just delete
if deletereport != 'yes':
if not add_known_crash(testline, reason, func, btline, inlogs_cleaned, infullbt_cleaned, bugdescription, extrainfo, DBCONN=dbconn):
return "Failed to add new known crash"
malooreport += "<h2>Maloo update report</h2>"
malooreport += "<table border=1><tr><th>maloo link</th><th>Update result</th></tr>"
# Now we need to gather all links and post the vetter result to maloo:
try:
reporter = mymaloo_bugreporter.maloo_poster()
cur = dbconn.cursor()
cur.execute('SELECT triage.link, triage.testline FROM triage, new_crashes WHERE triage.newcrash_id=new_crashes.id AND newcrash_id in ' + NEWIDS + EXTRACONDS, EXTRACONDvars)
rows = cur.fetchall()
cur.close()
for row in rows:
link = row[0]
# we could have excluded it with select, but that's probably
# not all that important with small numbers we have here
# and I need to do extra hoops to save old values and stuff
# in EXTRACONDS and EXTRACONDvars
if link.startswith('https://testing.whamcloud.com'):
print("marking " + link + " " + str(row[1]) + "<br>")
sys.stdout.flush()
if "tag" not in extrainfo:
res = reporter.associate_bug_by_url(link, bugdescription, row[1])
malooreport += '<tr><td><a href="%s">%s</a></td><td>' % (link, link)
if res:
malooreport += "Success"
print("Success")
else:
malooreport += "Error: " + reporter.error
print("error: " + reporter.error)
else:
print("Skipping tag ", bugdescription)
print("<br>")
sys.stdout.flush()
malooreport += '</td></tr>'
except psycopg2.DatabaseError as e:
malooreport += "DB Error " + str(e)
print(str(e))
malooreport += "</table>"
# and remove all matching reports.
try:
cur = dbconn.cursor()
cur.execute('DELETE FROM triage USING new_crashes WHERE newcrash_id in ' + NEWIDS + EXTRACONDS, EXTRACONDvars)
dbconn.commit()
# Now we need to see if any new crashes have zero triage reports left
# and nuke those
cur.execute('DELETE FROM new_crashes WHERE id in ' + NEWIDS + ' AND NOT EXISTS (SELECT 1 FROM triage WHERE triage.newcrash_id=new_crashes.id)')
dbconn.commit()
cur.close()
except psycopg2.DatabaseError as e:
return "DB Error on delete " + str(e)
all_items = {'NEWCRASHID':', '.join(ids), 'BASENAMEURL':basenameurl, 'MALOOREPORT':malooreport}
return template.format(**all_items)
if __name__ == "__main__":
form = cgi.FieldStorage()
if not form or form.getfirst("count") or form.getfirst("sort"):
result = print_new_crashes(dbconn, form)
elif form.getfirst("newconvert_submit"):
result = convert_new_crash(dbconn, form)
elif form.getfirst("newid"):
result = examine_one_new_crash(dbconn, form.getfirst("newid"))
print(result)
dbconn.close()