Skip to content

Commit

Permalink
scheme redesign
Browse files Browse the repository at this point in the history
  • Loading branch information
@simagix
simagix committed Feb 20, 2023
1 parent 45930d1 commit 3d19a51
Show file tree
Hide file tree
Showing 13 changed files with 341 additions and 229 deletions.
5 changes: 5 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,5 +30,10 @@ if you choose to view in the legacy format without a browser, use the command be

For additional usages and integration details, see [developer's guide](README_DEV.md).

## A Smart Log Analyzer
How smart Hatchet is? A picture is worth a thousand words.

![Sage Says](sage_says.png)

## License
[Apache-2.0 License](LICENSE)
10 changes: 0 additions & 10 deletions README_DEV.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,6 @@ SELECT SUBSTR(date, 1, 16), COUNT(op), op, ns
GROUP by SUBSTR(date, 1, 16), op, ns;
```

### Query Long Lasting Connection Duration and Relen
```sqlite3
SELECT ip, context, STRFTIME('%s', SUBSTR(etm,1,19))-STRFTIME('%s', SUBSTR(btm,1,19)) dur, reslen
FROM (
SELECT MAX(a.date) etm, MIN(a.date) btm, a.context context, b.ip, SUM(a.reslen) reslen
FROM mongod_1b3d5f7 a, mongod_1b3d5f7_clients b WHERE a.ip = b.ip GROUP BY a.context
)
WHERE dur > 0 AND reslen > 0 order by dur DESC, reslen DESC limit 23;
```

## Use SQLite3 API
Different drivers are supported for most popular programming languages including Golang, NodeJS, Java, Python, and C#.

Expand Down
81 changes: 54 additions & 27 deletions audit_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package hatchet
import (
"fmt"
"html/template"
"math/rand"
"strings"
"time"

Expand All @@ -20,16 +21,21 @@ import (
func GetAuditTablesTemplate() (*template.Template, error) {
html := headers + getContentHTML()
html += `{{$name := .Hatchet}}
<table style='border:none;'>
<tr><td style='border:none; vertical-align: top; padding: 5px;'>
<img class='rotate23' src='data:image/png;base64,{{ getTheGuyImage }}'></img></td>
<td class='summary'>{{getInfoSummary .Info}}<p/>{{getStatsSummary .Data}}</td></tr>
</table>
<div style='margin: 5px 5px; width=100%; clear: left;'>
<table style='border: none; margin: 10px 10px; width=100%; clear: left;' width='100%'>
{{$flag := coinToss}}
<tr><td style='border:none; vertical-align: top; padding: 5px; background-color: #F3F7F4;'>
<img class='rotate23' src='data:image/png;base64,{{ assignConsultant $flag }}'></img></td>
<td class='summary'>
{{getInfoSummary .Info $flag}}<p/>{{getStatsSummary .Data}}</td>
</tr>
</table>
</div>
{{if hasData .Data "exception"}}
<h3><button class='btn'
<table style='float: left; margin: 10px 10px; clear: left;'>
<caption><button class='btn'
onClick="javascript:location.href='/hatchets/{{.Hatchet}}/logs/all?severity=W'; return false;">
<i class='fa fa-search'></i></button>Exceptions</h3>
<table>
<i class='fa fa-search'></i></button>Exceptions</caption>
<tr><th></th><th>Severity</th><th>Total</th></tr>
{{range $n, $val := index .Data "exception"}}
<tr><td align=right>{{add $n 1}}</td>
Expand All @@ -42,10 +48,10 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
{{if hasData .Data "failed"}}
<h3><button class='btn'
<table style='float: left; margin: 10px 10px;'>
<caption><button class='btn'
onClick="javascript:location.href='/hatchets/{{.Hatchet}}/logs/all?context=failed'; return false;">
<i class='fa fa-search'></i></button>Failed Operations</h3>
<table>
<i class='fa fa-search'></i></button>Failed Operations</caption>
<tr><th></th><th>Failed Operation</th><th>Total</th></tr>
{{range $n, $val := index .Data "failed"}}
<tr><td align=right>{{add $n 1}}</td>
Expand All @@ -59,10 +65,10 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
{{if hasData .Data "op"}}
<h3><button class='btn'
<table style='float: left; margin: 10px 10px; clear: left;'>
<caption><button class='btn'
onClick="javascript:location.href='/hatchets/{{.Hatchet}}/charts/ops?type=stats'; return false;">
<i class='fa fa-area-chart'></i></button>Operations Stats</h3>
<table>
<i class='fa fa-area-chart'></i></button>Operations Stats</caption>
<tr><th></th><th>Operation</th><th>Total</th></tr>
{{range $n, $val := index .Data "op"}}
<tr><td align=right>{{add $n 1}}</td>
Expand All @@ -75,10 +81,10 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
{{if hasData .Data "ip"}}
<h3><button class='btn'
<table style='float: left; margin: 10px 10px;'>
<caption><button class='btn'
onClick="javascript:location.href='/hatchets/{{.Hatchet}}/charts/connections?type=accepted'; return false;">
<i class='fa fa-pie-chart'></i></button>Stats by IPs</h3>
<table>
<i class='fa fa-pie-chart'></i></button>Stats by IPs</caption>
<tr><th></th><th>IP</th><th>Accepted Connections</th><th>Response Length</th></tr>
{{range $n, $val := index .Data "ip"}}
<tr><td align=right>{{add $n 1}}</td>
Expand All @@ -91,10 +97,10 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
{{if hasData .Data "ns"}}
<h3><button class='btn'
<table style='float: left; margin: 10px 10px; clear: left;'>
<caption><button class='btn'
onClick="javascript:location.href='/hatchets/{{.Hatchet}}/charts/reslen-ns?ns='; return false;">
<i class='fa fa-pie-chart'></i></button>Stats by Namespaces</h3>
<table>
<i class='fa fa-pie-chart'></i></button>Stats by Namespaces</caption>
<tr><th></th><th>Namespace</th><th>Accessed</th><th>Response Length</th></tr>
{{range $n, $val := index .Data "ns"}}
<tr><td align=right>{{add $n 1}}</td>
Expand All @@ -107,8 +113,8 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
{{if hasData .Data "duration"}}
<h3><span style="font-size: 16px; padding: 5px 5px;"><i class="fa fa-shield"></i></span>Top N Long Lasting Connections</h3>
<table>
<table style='float: left; margin: 10px 10px; clear: left;'>
<caption><span style="font-size: 16px; padding: 5px 5px;"><i class="fa fa-shield"></i></span>Top N Long Lasting Connections</caption>
<tr><th></th><th>Context</th><th>Duration</th></tr>
{{range $n, $val := index .Data "duration"}}
{{if lt $n 23}}
Expand All @@ -122,7 +128,7 @@ func GetAuditTablesTemplate() (*template.Template, error) {
{{end}}
</table>
{{end}}
<div align='center'><hr/><p/>@simagix</div>
<div style='clear: left;' align='center'><hr/><p/>@simagix</div>
`
html += "</body></html>"
return template.New("hatchet").Funcs(template.FuncMap{
Expand All @@ -143,13 +149,21 @@ func GetAuditTablesTemplate() (*template.Template, error) {
}
return toks[0]
},
"getTheGuyImage": func() string {
"assignConsultant": func(sage bool) string {
if sage {
return SAGE_PNG
}
return SIMONE_PNG
},
"getFormattedNumber": func(numbers []int, i int) string {
printer := message.NewPrinter(language.English)
return printer.Sprintf("%v", numbers[i])
},
"coinToss": func() bool {
rand.Seed(time.Now().UnixNano())
randomNum := rand.Intn(2)
return (randomNum%2 == 0)
},
"getDurationFromSeconds": func(s int) string {
return gox.GetDurationFromSeconds(float64(s))
},
Expand All @@ -162,8 +176,11 @@ func GetAuditTablesTemplate() (*template.Template, error) {
"getFormattedSize": func(numbers []int, i int) string {
return gox.GetStorageSize(numbers[i])
},
"getInfoSummary": func(info HatchetInfo) template.HTML {
var html = "Hey there! My name is Simone and I am your assistant today. "
"getInfoSummary": func(info HatchetInfo, sage bool) template.HTML {
var html = "Hey there! My name is <i>Simone</i> and here is the summary I've prepared for you. "
if sage {
html = "Hello, my name is <i>Sage</i> and I'd like to share my thoughts with you on the findings. "
}
if info.Version == "" {
html += "There is not enough information in the log to determine what MongoDB version is used."
} else {
Expand Down Expand Up @@ -299,12 +316,22 @@ func GetAuditTablesTemplate() (*template.Template, error) {
if seconds < (10 * 60) { // should be calculated with duration
html += printer.Sprintf("The total impact time from slowest operations was %s. ", gox.GetDurationFromSeconds(seconds))
} else if seconds < (60 * 60) {
html += printer.Sprintf("The total impact time from slowest operations was, ouch,<span style='color: orange;'>%s</span>. ", gox.GetDurationFromSeconds(seconds))
html += printer.Sprintf("The total impact time from slowest operations was, ouch, <span style='color: orange;'>%s</span>. ", gox.GetDurationFromSeconds(seconds))
} else {
totalImpact = seconds
}
}
}
} else if key == "collscan" && len(docs) > 0 {
html += "Let's move to the performance evaluation. "
for _, doc := range docs {
if doc.Name == "count" {
html += printer.Sprintf(`I found <span style='color: orange;'>%d</span> with <mark><i>COLLSCAN</i> </mark>plan summary. `, doc.Values[0])
} else if doc.Name == "totalMilli" {
seconds := float64(doc.Values[0]) / 1000
html += printer.Sprintf(`The <i>COLLSCAN</i> caused a total of <span style='color: orange;'>%s</span> wasted. `, gox.GetDurationFromSeconds(seconds))
}
}
}
}
if totalImpact > 0 {
Expand Down
21 changes: 15 additions & 6 deletions charts_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,14 @@ func GetChartTemplate(chartType string) (*template.Template, error) {
html += getConnectionsChart()
}
html += `
<div align=left>
<input type='datetime-local' id='start' value='{{.Start}}'></input>
<input type='datetime-local' id='end' value='{{.End}}'></input>
<button onClick="refreshChart(); return false;" class="button">Refresh</button>
<div id='hatchetChart' align='center' width='100%'/>
</div></body></html>`
<div style="float: left; width: 100%; clear: left;">
<input type='datetime-local' id='start' value='{{.Start}}'></input>
<input type='datetime-local' id='end' value='{{.End}}'></input>
<button onClick="refreshChart(); return false;" class="button">Refresh</button>
</div>
<div id='hatchetChart' style="width: 100%; clear: left;"></div>
</body></html>`

return template.New("hatchet").Funcs(template.FuncMap{
"descr": func(v OpCount) template.HTML {
Expand Down Expand Up @@ -84,6 +86,7 @@ func getOpStatsChart() string {
// 'hAxis': { textPosition: 'none' },
'hAxis': { slantedText: true, slantedTextAngle: 30 },
'vAxis': {title: '{{.VAxisLabel}}', minValue: 0},
'width': '100%',
'height': 480,
'titleTextStyle': {'fontSize': 20},
{{if eq $ctype "ops"}}
Expand Down Expand Up @@ -123,6 +126,7 @@ func getPieChart() string {
var options = {
'backgroundColor': { 'fill': 'transparent' },
'title': '{{.Chart.Title}}',
'width': '100%',
'height': 480,
'titleTextStyle': {'fontSize': 20},
'slices': {},
Expand Down Expand Up @@ -169,11 +173,16 @@ func getConnectionsChart() string {
'title': '{{.Chart.Title}}',
'hAxis': { slantedText: true, slantedTextAngle: 30 },
'vAxis': {title: 'Count', minValue: 0},
'width': '100%',
'height': 480,
'titleTextStyle': {'fontSize': 20},
'legend': { 'position': 'right' } };
// Instantiate and draw our chart, passing in some options.
{{if eq $ctype "connections-time"}}
var chart = new google.visualization.LineChart(document.getElementById('hatchetChart'));
{{else}}
var chart = new google.visualization.ColumnChart(document.getElementById('hatchetChart'));
{{end}}
chart.draw(data, options);
}
</script>
Expand Down
1 change: 1 addition & 0 deletions images.go
View file

Large diffs are not rendered by default.

29 changes: 28 additions & 1 deletion legacy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ func AddLegacyString(doc *Logv2Info) error {
b, _ := bson.MarshalExtJSON(attr.Value, false, false)
arr = append(arr, string(b))
if doc.Msg == "client metadata" {

data, ok := attr.Value.(bson.D)
if ok {
driver, ok := data.Map()["driver"].(bson.D)
Expand All @@ -99,6 +98,34 @@ func AddLegacyString(doc *Logv2Info) error {
} else {
arr = append(arr, fmt.Sprintf("%v:%v", attr.Key, toLegacyString(attr.Value)))
}

// extra effort of retrieving driver info from COMMAND
if attr.Key == "command" {
remote := RemoteClient{}
_client, ok := attr.Value.(bson.D).Map()["$client"].(bson.D)
if ok {
driver, ok := _client.Map()["driver"].(bson.D)
if ok {
remote.Driver, _ = driver.Map()["name"].(string)
remote.Version, _ = driver.Map()["version"].(string)
}
mongos, ok := _client.Map()["mongos"].(bson.D)
if ok {
remote.IP, ok = mongos.Map()["client"].(string)
if ok {
if strings.Contains(remote.IP, ":") {
toks := strings.Split(remote.IP, ":")
remote.IP = toks[0]
}
}
} else {
log.Println("key 'mongos' under 'attr.command.$client' not found, report an issue at https://github.com/simagix/hatchet/issues")
}
}
if remote.IP != "" {
doc.Client = &remote
}
}
}
}

Expand Down
26 changes: 14 additions & 12 deletions logs_template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ func GetLogTableTemplate(attr string) (*template.Template, error) {
},
"highlightLog": func(log string, params ...string) template.HTML {
return template.HTML(highlightLog(log, params...))
},
"formatDateTime": func(str string) string {
return strings.Replace(str, "T", " ", 1)
}}).Parse(html)
}

Expand Down Expand Up @@ -90,51 +93,50 @@ func getSlowOpsLogsTable() string {
{{range $n, $value := .Logs}}
<tr>
<td align='right'>{{ add $n 1 }}</td>
<td>{{ $value.Timestamp }}</td>
<td>{{ formatDateTime $value.Timestamp }}</td>
<td>{{ $value.Severity }}</td>
<td>{{ $value.Component }}</td>
<td>{{ $value.Context }}</td>
<td>{{ highlightLog $value.Message }}</td>
</tr>
{{end}}
</table>
<div align='center'><hr/><p/>@simagix</div>
<div style='clear: left;' align='center'><hr/><p/>@simagix</div>
</div>
`
return template
}

func getLegacyLogsTable() string {
template := `
<br/>
<div style="float: left;">
<div style="float: left; margin-right: 20px; clear: left;">
<label><i class="fa fa-leaf"></i></label>
<select id='component'>
<option value=''>select a component</option>
{{getComponentOptions .Component}}
</select>
</div>
</div>
<div style="float: left; padding: 0px 0px 0px 20px;">
<div style="float: left; margin-right: 20px;">
<label><i class="fa fa-exclamation"></i></label>
<select id='severity'>
<option value=''>select a severity</option>
{{getSeverityOptions .Severity}}
</select>
</div>
</div>
<div style="float: left; padding: 0px 0px 0px 20px;">
<div style="float: left; margin-right: 20px;">
<label><i class="fa fa-search"></i></label>
<input id='context' type='text' value='{{.Context}}' size='30'/>
<button id="find" onClick="findLogs()" class="button" style="float: right;">Find</button>
</div>
</div>
<p/>
<div>
{{ if .Logs }}
{{if .HasMore}}
<button onClick="javascript:location.href='{{.URL}}'; return false;"
class="btn" style="float: right;"><i class="fa fa-arrow-right"></i></button>
class="btn" style="float: right; clear: right"><i class="fa fa-arrow-right"></i></button>
{{end}}
<table width='100%'>
<tr>
Expand All @@ -150,7 +152,7 @@ func getLegacyLogsTable() string {
{{range $n, $value := .Logs}}
<tr>
<td align='right'>{{ add $n $seq }}</td>
<td>{{ $value.Timestamp }}</td>
<td>{{ formatDateTime $value.Timestamp }}</td>
<td>{{ $value.Severity }}</td>
<td>{{ $value.Component }}</td>
<td>{{ $value.Context }}</td>
Expand All @@ -160,7 +162,7 @@ func getLegacyLogsTable() string {
</table>
{{if .HasMore}}
<button onClick="javascript:location.href='{{.URL}}'; return false;"
class="btn" style="float: right;"><i class="fa fa-arrow-right"></i></button>
class="btn" style="float: right; clear: right;"><i class="fa fa-arrow-right"></i></button>
{{end}}
<div align='center'><hr/><p/>@simagix</div>
{{end}}
Expand Down
Loading

0 comments on commit 3d19a51

Please sign in to comment.