Skip to content

Commit

Permalink
Closes #168 - Implement data retention in storage layer (WIP) (#182)
Browse files Browse the repository at this point in the history
  • Loading branch information
@s4kibs4mi
s4kibs4mi authored Jun 17, 2021
1 parent 7331758 commit af9a745
Show file tree
Hide file tree
Showing 19 changed files with 985 additions and 286 deletions.
97 changes: 54 additions & 43 deletions benchmark/grafana-provisioning/dashboards/main.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1300,7 +1300,7 @@
"renderer": "flot",
"seriesOverrides": [
{
"alias": "ram_evictions_used_perc",
"alias": "evictions_used_perc",
"yaxis": 2
}
],
Expand All @@ -1310,7 +1310,7 @@
"targets": [
{
"exemplar": true,
"expr": "ram_evictions_alloc_bytes{}",
"expr": "evictions_alloc_bytes{}",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
Expand All @@ -1319,7 +1319,7 @@
},
{
"exemplar": true,
"expr": "ram_evictions_total_bytes{}",
"expr": "evictions_total_bytes{}",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
Expand Down Expand Up @@ -1389,14 +1389,14 @@
"y": 10
},
"hiddenSeries": false,
"id": 14,
"id": 12,
"interval": "1",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"show": true,
"total": false,
"values": false
},
Expand All @@ -1414,8 +1414,12 @@
"renderer": "flot",
"seriesOverrides": [
{
"alias": "ram_evictions_used_perc",
"yaxis": 1
"alias": "evictions",
"yaxis": 2
},
{
"alias": "evictions_count",
"yaxis": 2
}
],
"spaceLength": 10,
Expand All @@ -1424,29 +1428,27 @@
"targets": [
{
"exemplar": true,
"expr": "ram_evictions_used_perc{}",
"expr": "{__name__=~\"evictions_timer\", instance=\"pyroscope:4040\"}",
"interval": "",
"legendFormat": "{{__name__}}",
"queryType": "randomWalk",
"refId": "A"
},
{
"exemplar": true,
"expr": "{__name__=~\"evictions_count\", instance=\"pyroscope:4040\"}",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
"queryType": "randomWalk",
"refId": "D"
}
],
"thresholds": [
{
"$$hashKey": "object:104",
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 0.25,
"yaxis": "left"
"refId": "B"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "RAM Usage %",
"title": "RAM Evictions Timer",
"tooltip": {
"shared": true,
"sort": 2,
Expand All @@ -1463,20 +1465,20 @@
"yaxes": [
{
"$$hashKey": "object:42",
"format": "percentunit",
"format": "ns",
"label": null,
"logBase": 1,
"max": "1",
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:43",
"format": "percentunit",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"min": null,
"show": true
}
],
Expand Down Expand Up @@ -1504,14 +1506,14 @@
"y": 10
},
"hiddenSeries": false,
"id": 12,
"id": 20,
"interval": "1",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"show": true,
"total": false,
"values": false
},
Expand All @@ -1529,7 +1531,11 @@
"renderer": "flot",
"seriesOverrides": [
{
"alias": "ram_evictions",
"alias": "evictions",
"yaxis": 2
},
{
"alias": "write_back_count",
"yaxis": 2
}
],
Expand All @@ -1539,15 +1545,15 @@
"targets": [
{
"exemplar": true,
"expr": "{__name__=~\"ram_evictions_timer\", instance=\"pyroscope:4040\"}",
"expr": "{__name__=~\"write_back_timer\", instance=\"pyroscope:4040\"}",
"interval": "",
"legendFormat": "{{__name__}}",
"queryType": "randomWalk",
"refId": "A"
},
{
"exemplar": true,
"expr": "{__name__=~\"ram_evictions\", instance=\"pyroscope:4040\"}",
"expr": "{__name__=~\"write_back_count\", instance=\"pyroscope:4040\"}",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
Expand All @@ -1559,7 +1565,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "RAM Evictions Timer",
"title": "RAM Persistence Timer",
"tooltip": {
"shared": true,
"sort": 2,
Expand Down Expand Up @@ -1682,18 +1688,18 @@
"gridPos": {
"h": 5,
"w": 3,
"x": 12,
"x": 6,
"y": 15
},
"hiddenSeries": false,
"id": 20,
"id": 14,
"interval": "1",
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"show": true,
"total": false,
"values": false
},
Expand All @@ -1711,7 +1717,11 @@
"renderer": "flot",
"seriesOverrides": [
{
"alias": "ram_evictions",
"alias": "evictions_used_perc",
"yaxis": 1
},
{
"alias": "retention_count",
"yaxis": 2
}
],
Expand All @@ -1721,27 +1731,28 @@
"targets": [
{
"exemplar": true,
"expr": "{__name__=~\"ram_persistence_timer\", instance=\"pyroscope:4040\"}",
"expr": "retention_timer",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
"queryType": "randomWalk",
"refId": "A"
"refId": "D"
},
{
"exemplar": true,
"expr": "{__name__=~\"ram_persistence\", instance=\"pyroscope:4040\"}",
"expr": "retention_count",
"hide": false,
"interval": "",
"legendFormat": "{{__name__}}",
"queryType": "randomWalk",
"refId": "B"
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "RAM Persistence Timer",
"title": "Retention",
"tooltip": {
"shared": true,
"sort": 2,
Expand Down Expand Up @@ -1771,7 +1782,7 @@
"label": null,
"logBase": 1,
"max": null,
"min": null,
"min": "0",
"show": true
}
],
Expand All @@ -1789,7 +1800,7 @@
"list": []
},
"time": {
"from": "now-5m",
"from": "now-15m",
"to": "now"
},
"timepicker": {
Expand Down Expand Up @@ -1820,5 +1831,5 @@
"timezone": "browser",
"title": "main",
"uid": "65gjqY3Mk",
"version": 1
"version": 10
}
98 changes: 51 additions & 47 deletions pkg/agent/upstream/remote/remote_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,58 +18,62 @@ import (

var _ = Describe("remote.Remote", func() {
Describe("Upload", func() {
It("uploads data to an http server", func(done Done) {
wg := sync.WaitGroup{}
wg.Add(3)
var timestampsMutex sync.Mutex
timestamps := []time.Time{}
myHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer GinkgoRecover()
It("uploads data to an http server", func() {
done := make(chan interface{})
func() {
wg := sync.WaitGroup{}
wg.Add(3)
var timestampsMutex sync.Mutex
timestamps := []time.Time{}
myHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer GinkgoRecover()

timestampsMutex.Lock()
timestamps = append(timestamps, time.Now())
timestampsMutex.Unlock()
_, err := ioutil.ReadAll(r.Body)
Expect(err).ToNot(HaveOccurred())
timestampsMutex.Lock()
timestamps = append(timestamps, time.Now())
timestampsMutex.Unlock()
_, err := ioutil.ReadAll(r.Body)
Expect(err).ToNot(HaveOccurred())

fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
wg.Done()
})
fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path))
wg.Done()
})

mockServer := &http.Server{
Addr: ":50001",
Handler: myHandler,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
go mockServer.ListenAndServe()
mockServer := &http.Server{
Addr: ":50001",
Handler: myHandler,
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
MaxHeaderBytes: 1 << 20,
}
go mockServer.ListenAndServe()

cfg := RemoteConfig{
AuthToken: "",
UpstreamThreads: 4,
UpstreamAddress: "http://localhost:50001",
UpstreamRequestTimeout: 3 * time.Second,
}
r, err := New(cfg, logrus.New())
cfg := RemoteConfig{
AuthToken: "",
UpstreamThreads: 4,
UpstreamAddress: "http://localhost:50001",
UpstreamRequestTimeout: 3 * time.Second,
}
r, err := New(cfg, logrus.New())

t := transporttrie.New()
for i := 0; i < 3; i++ {
r.Upload(&upstream.UploadJob{
Name: "test{}",
StartTime: testing.SimpleTime(0),
EndTime: testing.SimpleTime(10),
SpyName: "debugspy",
SampleRate: 100,
Units: "samples",
Trie: t,
})
}
t := transporttrie.New()
for i := 0; i < 3; i++ {
r.Upload(&upstream.UploadJob{
Name: "test{}",
StartTime: testing.SimpleTime(0),
EndTime: testing.SimpleTime(10),
SpyName: "debugspy",
SampleRate: 100,
Units: "samples",
Trie: t,
})
}

Expect(err).To(BeNil())
wg.Wait()
r.Stop()
close(done)
}, 3)
Expect(err).To(BeNil())
wg.Wait()
r.Stop()
close(done)
}()
Eventually(done, 5).Should(BeClosed())
})
})
})
Loading

0 comments on commit af9a745

Please sign in to comment.