Skip to content
This repository was archived by the owner on Aug 23, 2023. It is now read-only.

Optimize kafka format #876

Merged
merged 70 commits into from
Apr 11, 2018
Merged
Changes from 1 commit
Commits
Show all changes
70 commits
Select commit Hold shift + click to select a range
9bab237
schema: fix build
Dieterbe Mar 13, 2018
f1788b7
schema: refactor benchmarks
Dieterbe Mar 13, 2018
75ff34f
schema: benchmarks for msgp MetricData individually
Dieterbe Mar 14, 2018
8df7656
schema: better func name, more realistic timestamps, set Id realistic…
Dieterbe Mar 14, 2018
cf82d06
schema: implementation + benchmarks for msgp optimized MetricPoint id's
Dieterbe Mar 14, 2018
404f5d0
schema: rework MetricPoint2 as embedding MetricPoint1
Dieterbe Mar 17, 2018
53138e4
input plugins: accept new format (mainly kafka)
Dieterbe Mar 16, 2018
761a9ee
schema: add Key and MKey types
Dieterbe Mar 16, 2018
d3ed9ef
schema: add an efficient way to represent a rollup archive
Dieterbe Mar 17, 2018
b9bf957
schema: add AMKey (archive-aware multitenant key)
Dieterbe Mar 18, 2018
7bddec1
cache: update to identify metrics by AMKey
Dieterbe Mar 16, 2018
34fef01
cache: make unit tests work (WARN: cache itself doesn't work yet)
Dieterbe Mar 17, 2018
8b69884
mdata: make AggMetric(s), Aggregator, CWR id's use Key, MKey, AMKey n…
Dieterbe Mar 17, 2018
e217ec5
input: use MKey
Dieterbe Mar 18, 2018
2dd003d
store: update for reads/writes via AMKey
Dieterbe Mar 18, 2018
a1e29af
notifier: call into idx and AggMetrics using MKey
Dieterbe Mar 18, 2018
16a6a8c
remove old idx.MetricID stuff
Dieterbe Mar 18, 2018
5bcd639
schema: MetricDefinition use MKey instead of string id
Dieterbe Mar 18, 2018
e04411d
idx: simplify
Dieterbe Mar 19, 2018
fa7dd52
idx: remove no longer needed safety precaution
Dieterbe Mar 19, 2018
4040889
WIP: update idx
Dieterbe Mar 18, 2018
fc5812b
idx & input: changing approach. no need for composite msg.Point type
Dieterbe Mar 19, 2018
abb3313
typo
Dieterbe Mar 19, 2018
1bbc409
schema: finalize new MetricPoint type
Dieterbe Mar 19, 2018
48a68aa
cache: use new format
replay Mar 19, 2018
eadcc93
ccache: gofmt -w -s
Dieterbe Mar 30, 2018
bdf3312
schema: GetAMKey helper
Dieterbe Mar 20, 2018
36fd782
cache: simplify test keys
Dieterbe Mar 20, 2018
3388891
schema: encode/decode methods for MetricPoint messages
Dieterbe Mar 21, 2018
22e4770
input/idx: update for new MetricPoint type and messages
Dieterbe Mar 21, 2018
2571396
api: update to adopt new Key types
Dieterbe Mar 21, 2018
d10bf90
WIP: fix tools
Dieterbe Mar 22, 2018
b903812
enable multiple graphite's so you can check data for >1 orgs
Dieterbe Mar 27, 2018
128372c
enable prometheus monitoring for host and container stats
Dieterbe Mar 27, 2018
f7e3ee8
update for new fakemetrics stats
Dieterbe Mar 27, 2018
8e4f577
cleaner way to query both orgs on graphite + MT directly
Dieterbe Mar 28, 2018
78e998a
more useful viz
Dieterbe Mar 28, 2018
efb075b
fix alloc regression in CassandraIdx.deleteOldIfMoved
Dieterbe Mar 28, 2018
333bcc5
fix alloc regression in CassandraIdx.updateCassandraIfStale
Dieterbe Mar 28, 2018
6a94384
optimize CasIdx.deleteDef
Dieterbe Mar 28, 2018
b78b5ad
idx: optimize UpdateMaybe and AddOrUpdate calls
Dieterbe Mar 28, 2018
d25066e
benchmark script
Dieterbe Mar 28, 2018
9b803e1
move gettest(A)Mkey helpers to reusable lib
Dieterbe Mar 29, 2018
d7724be
update test.Get[A]Mkey to support large ints
Dieterbe Mar 29, 2018
8b3ff43
WIP: fix tests
Dieterbe Mar 29, 2018
f4f54ef
disable chaos tests when running `make test`
Dieterbe Mar 29, 2018
d129a8a
schema: add IsSpanValid() function
Dieterbe Mar 30, 2018
59c2a04
mdata/conf: make sure aggspans for AggMetrics' AMKeys are always valid
Dieterbe Mar 30, 2018
0cd6d09
schema: add AMKeyFromString function
Dieterbe Mar 30, 2018
578aafe
notifier: use AMKeyFromString
Dieterbe Mar 30, 2018
60c1b56
mt-store-cat: fix and cleanups, relying on MKey and AMKey[FromString]
Dieterbe Mar 30, 2018
e0d3f54
tools: fix the last ones. mt-whisper-importer-*
Dieterbe Mar 30, 2018
a39a1e4
schema: provide easy access to Method and Span of Archive
Dieterbe Apr 2, 2018
0da479e
mdata: cleanup notifier parsing relying on Archive
Dieterbe Apr 2, 2018
e9c6b8b
bench.sh: also compare with/without compression
Dieterbe Apr 2, 2018
baf256a
easy testing along with `fakemetrics storageconf`
Dieterbe Apr 3, 2018
1a5ab80
update docker-cluster as well for multi-org testing
Dieterbe Apr 4, 2018
47e7a14
update docker-cluster as well for aggregation testing
Dieterbe Apr 4, 2018
9db941f
mt-schemas-explain: also show built-in default
Dieterbe Apr 4, 2018
6c7db1e
cleanup vendor
Dieterbe Apr 4, 2018
445e85b
rename MemoryIdx.Update to UpdateArchive
Dieterbe Apr 10, 2018
49f2631
UpdateMaybe -> Update and better comments
Dieterbe Apr 10, 2018
f05deab
use official graphite image
Dieterbe Apr 10, 2018
01dabf9
cleanup input handler metrics
Dieterbe Apr 10, 2018
96a9579
schema: deprecate Metric field in favor of Name
Dieterbe Apr 10, 2018
8dec28c
deprecate Metric field in favor of Name
Dieterbe Apr 10, 2018
eabd1bb
schema: clean up struct tags, remove references to elasticsearch
Dieterbe Apr 10, 2018
d23d807
./scripts/dev/tools-to-doc.sh > docs/tools.md
Dieterbe Apr 10, 2018
45c02a0
shield hot Debug call with loglevel check
Dieterbe Apr 10, 2018
a6295dd
remove non-snappy tests again
Dieterbe Apr 10, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
easy testing along with fakemetrics storageconf
Dieterbe committed Apr 10, 2018
commit baf256afd810827a0aa66a12e2bc4e577d2f2863
20 changes: 20 additions & 0 deletions docker/docker-dev-custom-cfg-kafka/storage-aggregation.conf
Original file line number Diff line number Diff line change
@@ -11,6 +11,26 @@
#
# see https://github.com/grafana/metrictank/blob/master/docs/consolidation.md for related info.

[sum]
pattern = .*sum
xFilesFactor = 0.5
aggregationMethod = sum

[min]
pattern = .*min
xFilesFactor = 0.5
aggregationMethod = min

[max]
pattern = .*max
xFilesFactor = 0.5
aggregationMethod = max

[last]
pattern = .*lst
xFilesFactor = 0.5
aggregationMethod = last

[default]
pattern = .*
xFilesFactor = 0.5
6 changes: 5 additions & 1 deletion docker/docker-dev-custom-cfg-kafka/storage-schemas.conf
Original file line number Diff line number Diff line change
@@ -50,7 +50,11 @@
# This example has 3 retention definitions, the first and last override some default options (to use 10minutely and 2hourly chunks and only keep one of them in memory
# and the last rollup is marked as not ready yet for querying.

[raw]
pattern = fakemetrics.raw
retentions = 1s:6h:2min:2

[default]
pattern = .*
retentions = 1s:35d:2min:2
retentions = 1s:6h:2min:2,1min:35d:6h:1
# reorderBuffer = 20
275 changes: 275 additions & 0 deletions docker/extra/dashboards/fake-metrics-storage-conf.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,275 @@
{
"__inputs": [],
"__requires": [
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "4.6.2"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"hideControls": false,
"id": null,
"links": [],
"refresh": false,
"rows": [
{
"collapse": false,
"height": 262,
"panels": [
{
"aliasColors": {
"fakemetrics.raw.1.default": "#e5ac0e",
"fakemetrics.raw.1.max": "#e24d42",
"fakemetrics.raw.1.min": "#65c5db",
"fakemetrics.raw.1.sum": "#890f02",
"sum": "#bf1b00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"id": 1,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [
],
"spaceLength": 10,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(fakemetrics.raw.1.*, 3)"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "fakemetrics.raw",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 10,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "Dashboard Row",
"titleSize": "h6"
},
{
"collapse": false,
"height": 250,
"panels": [
{
"aliasColors": {
"default": "#7eb26d",
"fakemetrics.raw.1.default": "#e5ac0e",
"fakemetrics.raw.1.max": "#e24d42",
"fakemetrics.raw.1.min": "#65c5db",
"fakemetrics.raw.1.sum": "#890f02",
"sum": "#bf1b00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"id": 2,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [
],
"spaceLength": 10,
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(fakemetrics.agg.1.*, 3)"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "fakemetrics.agg",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 10,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"repeat": null,
"repeatIteration": null,
"repeatRowId": null,
"showTitle": false,
"title": "Dashboard Row",
"titleSize": "h6"
}
],
"schemaVersion": 14,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"current": {
"text": "default",
"value": "default"
},
"hide": 0,
"label": null,
"name": "datasource",
"options": [],
"query": "graphite",
"refresh": 1,
"regex": "",
"type": "datasource"
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Fakemetrics storageconf",
"version": 1
}