Skip to content

Commit 7f2fd2c

Browse files
replayreplay
authored
Merge pull request grafana#959 from bloomberg/isNonNull
Add isNonNull function
2 parents 4c22f98 + 970e18e commit 7f2fd2c

File tree

4 files changed

+280
-0
lines changed

4 files changed

+280
-0
lines changed

‎docs/graphite.md

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ divideSeriesLists(dividends, divisors) seriesList | | Stable
4040
exclude(seriesList, pattern) seriesList | | Stable
4141
grep(seriesList, pattern) seriesList | | Stable
4242
groupByTags(seriesList, func, tagList) seriesList | | Stable
43+
isNonNull(seriesList) seriesList | | Stable
4344
maxSeries(seriesList) series | max | Stable
4445
minSeries(seriesList) series | min | Stable
4546
multiplySeries(seriesList) series | | Stable

‎expr/func_isnonnull.go

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package expr
2+
3+
import (
4+
"fmt"
5+
"math"
6+
7+
"github.com/grafana/metrictank/api/models"
8+
schema "gopkg.in/raintank/schema.v1"
9+
)
10+
11+
type FuncIsNonNull struct {
12+
in GraphiteFunc
13+
}
14+
15+
func NewIsNonNull() GraphiteFunc {
16+
return &FuncIsNonNull{}
17+
}
18+
19+
func (s *FuncIsNonNull) Signature() ([]Arg, []Arg) {
20+
return []Arg{
21+
ArgSeriesList{val: &s.in}}, []Arg{ArgSeriesList{}}
22+
}
23+
24+
func (s *FuncIsNonNull) Context(context Context) Context {
25+
return context
26+
}
27+
28+
func (s *FuncIsNonNull) Exec(cache map[Req][]models.Series) ([]models.Series, error) {
29+
series, err := s.in.Exec(cache)
30+
if err != nil {
31+
return nil, err
32+
}
33+
34+
out := make([]models.Series, len(series))
35+
for i, serie := range series {
36+
transformed := &out[i]
37+
transformed.Target = fmt.Sprintf("isNonNull(%s)", serie.Target)
38+
transformed.QueryPatt = fmt.Sprintf("isNonNull(%s)", serie.QueryPatt)
39+
transformed.Tags = make(map[string]string, len(serie.Tags)+1)
40+
transformed.Datapoints = pointSlicePool.Get().([]schema.Point)
41+
transformed.Interval = serie.Interval
42+
transformed.Consolidator = serie.Consolidator
43+
transformed.QueryCons = serie.QueryCons
44+
45+
for k, v := range serie.Tags {
46+
transformed.Tags[k] = v
47+
}
48+
transformed.Tags["isNonNull"] = "1"
49+
for _, p := range serie.Datapoints {
50+
if math.IsNaN(p.Val) {
51+
p.Val = 0
52+
} else {
53+
p.Val = 1
54+
}
55+
transformed.Datapoints = append(transformed.Datapoints, p)
56+
}
57+
cache[Req{}] = append(cache[Req{}], *transformed)
58+
}
59+
60+
return out, nil
61+
}

‎expr/func_isnonnull_test.go

+217
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
package expr
2+
3+
import (
4+
"strconv"
5+
"testing"
6+
7+
"github.com/grafana/metrictank/api/models"
8+
"github.com/grafana/metrictank/test"
9+
"gopkg.in/raintank/schema.v1"
10+
)
11+
12+
var aIsNonNull = []schema.Point{
13+
{Val: 1, Ts: 10},
14+
{Val: 1, Ts: 20},
15+
{Val: 1, Ts: 30},
16+
{Val: 0, Ts: 40},
17+
{Val: 0, Ts: 50},
18+
{Val: 1, Ts: 60},
19+
}
20+
21+
var bIsNonNull = []schema.Point{
22+
{Val: 1, Ts: 10},
23+
{Val: 1, Ts: 20},
24+
{Val: 1, Ts: 30},
25+
{Val: 0, Ts: 40},
26+
{Val: 1, Ts: 50},
27+
{Val: 0, Ts: 60},
28+
}
29+
30+
var cdIsNonNull = []schema.Point{
31+
{Val: 1, Ts: 10},
32+
{Val: 1, Ts: 20},
33+
{Val: 1, Ts: 30},
34+
{Val: 1, Ts: 40},
35+
{Val: 1, Ts: 50},
36+
{Val: 1, Ts: 60},
37+
}
38+
39+
func TestIsNonNullSingle(t *testing.T) {
40+
testIsNonNull(
41+
"identity",
42+
[]models.Series{
43+
{
44+
Interval: 10,
45+
QueryPatt: "a",
46+
Datapoints: getCopy(a),
47+
},
48+
},
49+
[]models.Series{
50+
{
51+
Interval: 10,
52+
QueryPatt: "isNonNull(a)",
53+
Datapoints: getCopy(aIsNonNull),
54+
},
55+
},
56+
t,
57+
)
58+
}
59+
60+
func TestIsNonNullSingleAllNonNull(t *testing.T) {
61+
testIsNonNull(
62+
"identity-counter8bit",
63+
[]models.Series{
64+
{
65+
Interval: 10,
66+
QueryPatt: "counter8bit",
67+
Datapoints: getCopy(d),
68+
},
69+
},
70+
[]models.Series{
71+
{
72+
Interval: 10,
73+
QueryPatt: "isNonNull(counter8bit)",
74+
Datapoints: getCopy(cdIsNonNull),
75+
},
76+
},
77+
t,
78+
)
79+
}
80+
81+
func TestIsNonNullMulti(t *testing.T) {
82+
testIsNonNull(
83+
"multiple-series",
84+
[]models.Series{
85+
{
86+
Interval: 10,
87+
QueryPatt: "a",
88+
Datapoints: getCopy(a),
89+
},
90+
{
91+
Interval: 10,
92+
QueryPatt: "b.*",
93+
Datapoints: getCopy(b),
94+
},
95+
{
96+
Interval: 10,
97+
QueryPatt: "c.foo{bar,baz}",
98+
Datapoints: getCopy(c),
99+
},
100+
{
101+
Interval: 10,
102+
QueryPatt: "movingAverage(bar, '1min')",
103+
Datapoints: getCopy(d),
104+
},
105+
},
106+
[]models.Series{
107+
{
108+
QueryPatt: "isNonNull(a)",
109+
Datapoints: getCopy(aIsNonNull),
110+
},
111+
{
112+
QueryPatt: "isNonNull(b.*)",
113+
Datapoints: getCopy(bIsNonNull),
114+
},
115+
{
116+
QueryPatt: "isNonNull(c.foo{bar,baz})",
117+
Datapoints: getCopy(cdIsNonNull),
118+
},
119+
{
120+
QueryPatt: "isNonNull(movingAverage(bar, '1min'))",
121+
Datapoints: getCopy(cdIsNonNull),
122+
},
123+
},
124+
t,
125+
)
126+
}
127+
128+
func testIsNonNull(name string, in []models.Series, out []models.Series, t *testing.T) {
129+
f := NewIsNonNull()
130+
f.(*FuncIsNonNull).in = NewMock(in)
131+
gots, err := f.Exec(make(map[Req][]models.Series))
132+
if err != nil {
133+
t.Fatalf("case %q: err should be nil. got %q", name, err)
134+
}
135+
if len(gots) != len(out) {
136+
t.Fatalf("case %q: isNonNull len output expected %d, got %d", name, len(out), len(gots))
137+
}
138+
for i, g := range gots {
139+
exp := out[i]
140+
if g.QueryPatt != exp.QueryPatt {
141+
t.Fatalf("case %q: expected target %q, got %q", name, exp.QueryPatt, g.QueryPatt)
142+
}
143+
if len(g.Datapoints) != len(exp.Datapoints) {
144+
t.Fatalf("case %q: len output expected %d, got %d", name, len(exp.Datapoints), len(g.Datapoints))
145+
}
146+
for j, p := range g.Datapoints {
147+
if (p.Val == exp.Datapoints[j].Val) && p.Ts == exp.Datapoints[j].Ts {
148+
continue
149+
}
150+
t.Fatalf("case %q: output point %d - expected %v got %v", name, j, exp.Datapoints[j], p)
151+
}
152+
}
153+
}
154+
155+
func BenchmarkIsNonNull10k_1NoNulls(b *testing.B) {
156+
benchmarkIsNonNull(b, 1, test.RandFloats10k, test.RandFloats10k)
157+
}
158+
func BenchmarkIsNonNull10k_10NoNulls(b *testing.B) {
159+
benchmarkIsNonNull(b, 10, test.RandFloats10k, test.RandFloats10k)
160+
}
161+
func BenchmarkIsNonNull10k_100NoNulls(b *testing.B) {
162+
benchmarkIsNonNull(b, 100, test.RandFloats10k, test.RandFloats10k)
163+
}
164+
func BenchmarkIsNonNull10k_1000NoNulls(b *testing.B) {
165+
benchmarkIsNonNull(b, 1000, test.RandFloats10k, test.RandFloats10k)
166+
}
167+
168+
func BenchmarkIsNonNull10k_1SomeSeriesHalfNulls(b *testing.B) {
169+
benchmarkIsNonNull(b, 1, test.RandFloats10k, test.RandFloatsWithNulls10k)
170+
}
171+
func BenchmarkIsNonNull10k_10SomeSeriesHalfNulls(b *testing.B) {
172+
benchmarkIsNonNull(b, 10, test.RandFloats10k, test.RandFloatsWithNulls10k)
173+
}
174+
func BenchmarkIsNonNull10k_100SomeSeriesHalfNulls(b *testing.B) {
175+
benchmarkIsNonNull(b, 100, test.RandFloats10k, test.RandFloatsWithNulls10k)
176+
}
177+
func BenchmarkIsNonNull10k_1000SomeSeriesHalfNulls(b *testing.B) {
178+
benchmarkIsNonNull(b, 1000, test.RandFloats10k, test.RandFloatsWithNulls10k)
179+
}
180+
181+
func BenchmarkIsNonNull10k_1AllSeriesHalfNulls(b *testing.B) {
182+
benchmarkIsNonNull(b, 1, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
183+
}
184+
func BenchmarkIsNonNull10k_10AllSeriesHalfNulls(b *testing.B) {
185+
benchmarkIsNonNull(b, 10, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
186+
}
187+
func BenchmarkIsNonNull10k_100AllSeriesHalfNulls(b *testing.B) {
188+
benchmarkIsNonNull(b, 100, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
189+
}
190+
func BenchmarkIsNonNull10k_1000AllSeriesHalfNulls(b *testing.B) {
191+
benchmarkIsNonNull(b, 1000, test.RandFloatsWithNulls10k, test.RandFloatsWithNulls10k)
192+
}
193+
194+
func benchmarkIsNonNull(b *testing.B, numSeries int, fn0, fn1 func() []schema.Point) {
195+
var input []models.Series
196+
for i := 0; i < numSeries; i++ {
197+
series := models.Series{
198+
QueryPatt: strconv.Itoa(i),
199+
}
200+
if i%2 == 0 {
201+
series.Datapoints = fn0()
202+
} else {
203+
series.Datapoints = fn1()
204+
}
205+
input = append(input, series)
206+
}
207+
b.ResetTimer()
208+
for i := 0; i < b.N; i++ {
209+
f := NewIsNonNull()
210+
f.(*FuncIsNonNull).in = NewMock(input)
211+
got, err := f.Exec(make(map[Req][]models.Series))
212+
if err != nil {
213+
b.Fatalf("%s", err)
214+
}
215+
results = got
216+
}
217+
}

‎expr/funcs.go

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ func init() {
6161
"exclude": {NewExclude, true},
6262
"grep": {NewGrep, true},
6363
"groupByTags": {NewGroupByTags, true},
64+
"isNonNull": {NewIsNonNull, true},
6465
"max": {NewAggregateConstructor("max", crossSeriesMax), true},
6566
"maxSeries": {NewAggregateConstructor("max", crossSeriesMax), true},
6667
"min": {NewAggregateConstructor("min", crossSeriesMin), true},

0 commit comments

Comments
 (0)