Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix cloud thresholds #894

Merged
merged 3 commits into from
Jan 15, 2019
Merged

Fix cloud thresholds #894

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
070fd1a
Report threshold failure correctly to output cloud fix #849
mstoykov Jan 14, 2019
b1e7088
Refactor/rename Thredhold(s)' fields/methods and some documentation
mstoykov Jan 14, 2019
d5a2d2f
Add release note for fixing cloud threshold reporting #849
mstoykov Jan 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions release notes/upcoming.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ Description of feature.
* JS: Consistently report setup/teardown timeouts as such and switch the error message to be more
expressive (#890)
* JS: Correctly exit with non zero exit code when setup or teardown timeouts (#892)
* Thresholds: When outputting metrics to the Load Impact cloud, fix the incorrect reporting of
threshold statuses at the end of the test (#894)
4 changes: 2 additions & 2 deletions stats/cloud/collector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,8 +479,8 @@ func (c *Collector) testFinished() {
for name, thresholds := range c.thresholds {
thresholdResults[name] = make(map[string]bool)
for _, t := range thresholds {
thresholdResults[name][t.Source] = t.Failed
if t.Failed {
thresholdResults[name][t.Source] = t.LastFailed
if t.LastFailed {
testTainted = true
}
}
Expand Down
67 changes: 40 additions & 27 deletions stats/thresholds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,17 +45,23 @@ func init() {
jsEnv = pgm
}

// Threshold is a representation of a single threshold for a single metric
type Threshold struct {
Source string
Failed bool
AbortOnFail bool
// Source is the text based source of the threshold
Source string
// LastFailed is a makrer if the last testing of this threshold failed
LastFailed bool
// AbortOnFail marks if a given threshold fails that the whole test should be aborted
AbortOnFail bool
// AbortGracePeriod is a the minimum amount of time a test should be running before a failing
// this threshold will abort the test
AbortGracePeriod types.NullDuration

pgm *goja.Program
rt *goja.Runtime
}

func NewThreshold(src string, rt *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) {
func newThreshold(src string, newThreshold *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) {
pgm, err := goja.Compile("__threshold__", src, true)
if err != nil {
return nil, err
Expand All @@ -66,36 +72,34 @@ func NewThreshold(src string, rt *goja.Runtime, abortOnFail bool, gracePeriod ty
AbortOnFail: abortOnFail,
AbortGracePeriod: gracePeriod,
pgm: pgm,
rt: rt,
rt: newThreshold,
}, nil
}

func (t Threshold) RunNoTaint() (bool, error) {
func (t Threshold) runNoTaint() (bool, error) {
v, err := t.rt.RunProgram(t.pgm)
if err != nil {
return false, err
}
return v.ToBoolean(), nil
}

func (t *Threshold) Run() (bool, error) {
b, err := t.RunNoTaint()
if !b {
t.Failed = true
}
func (t *Threshold) run() (bool, error) {
b, err := t.runNoTaint()
t.LastFailed = !b
return b, err
}

type ThresholdConfig struct {
type thresholdConfig struct {
Threshold string `json:"threshold"`
AbortOnFail bool `json:"abortOnFail"`
AbortGracePeriod types.NullDuration `json:"delayAbortEval"`
}

//used internally for JSON marshalling
type rawThresholdConfig ThresholdConfig
type rawThresholdConfig thresholdConfig

func (tc *ThresholdConfig) UnmarshalJSON(data []byte) error {
func (tc *thresholdConfig) UnmarshalJSON(data []byte) error {
//shortcircuit unmarshalling for simple string format
if err := json.Unmarshal(data, &tc.Threshold); err == nil {
return nil
Expand All @@ -105,37 +109,39 @@ func (tc *ThresholdConfig) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, rawConfig)
}

func (tc ThresholdConfig) MarshalJSON() ([]byte, error) {
func (tc thresholdConfig) MarshalJSON() ([]byte, error) {
if tc.AbortOnFail {
return json.Marshal(rawThresholdConfig(tc))
}
return json.Marshal(tc.Threshold)
}

// Thresholds is the combination of all Thresholds for a given metric
type Thresholds struct {
Runtime *goja.Runtime
Thresholds []*Threshold
Abort bool
}

// NewThresholds returns Thresholds objects representing the provided source strings
func NewThresholds(sources []string) (Thresholds, error) {
tcs := make([]ThresholdConfig, len(sources))
tcs := make([]thresholdConfig, len(sources))
for i, source := range sources {
tcs[i].Threshold = source
}

return NewThresholdsWithConfig(tcs)
return newThresholdsWithConfig(tcs)
}

func NewThresholdsWithConfig(configs []ThresholdConfig) (Thresholds, error) {
func newThresholdsWithConfig(configs []thresholdConfig) (Thresholds, error) {
rt := goja.New()
if _, err := rt.RunProgram(jsEnv); err != nil {
return Thresholds{}, errors.Wrap(err, "builtin")
}

ts := make([]*Threshold, len(configs))
for i, config := range configs {
t, err := NewThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod)
t, err := newThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod)
if err != nil {
return Thresholds{}, errors.Wrapf(err, "%d", i)
}
Expand All @@ -145,7 +151,7 @@ func NewThresholdsWithConfig(configs []ThresholdConfig) (Thresholds, error) {
return Thresholds{rt, ts, false}, nil
}

func (ts *Thresholds) UpdateVM(sink Sink, t time.Duration) error {
func (ts *Thresholds) updateVM(sink Sink, t time.Duration) error {
ts.Runtime.Set("__sink__", sink)
f := sink.Format(t)
for k, v := range f {
Expand All @@ -154,10 +160,10 @@ func (ts *Thresholds) UpdateVM(sink Sink, t time.Duration) error {
return nil
}

func (ts *Thresholds) RunAll(t time.Duration) (bool, error) {
func (ts *Thresholds) runAll(t time.Duration) (bool, error) {
succ := true
for i, th := range ts.Thresholds {
b, err := th.Run()
b, err := th.run()
if err != nil {
return false, errors.Wrapf(err, "%d", i)
}
Expand All @@ -175,32 +181,39 @@ func (ts *Thresholds) RunAll(t time.Duration) (bool, error) {
return succ, nil
}

// Run processes all the thresholds with the provided Sink at the provided time and returns if any
// of them fails
func (ts *Thresholds) Run(sink Sink, t time.Duration) (bool, error) {
if err := ts.UpdateVM(sink, t); err != nil {
if err := ts.updateVM(sink, t); err != nil {
return false, err
}
return ts.RunAll(t)
return ts.runAll(t)
}

// UnmarshalJSON is implementation of json.Unmarshaler
func (ts *Thresholds) UnmarshalJSON(data []byte) error {
var configs []ThresholdConfig
var configs []thresholdConfig
if err := json.Unmarshal(data, &configs); err != nil {
return err
}
newts, err := NewThresholdsWithConfig(configs)
newts, err := newThresholdsWithConfig(configs)
if err != nil {
return err
}
*ts = newts
return nil
}

// MarshalJSON is implementation of json.Marshaler
func (ts Thresholds) MarshalJSON() ([]byte, error) {
configs := make([]ThresholdConfig, len(ts.Thresholds))
configs := make([]thresholdConfig, len(ts.Thresholds))
for i, t := range ts.Thresholds {
configs[i].Threshold = t.Source
configs[i].AbortOnFail = t.AbortOnFail
configs[i].AbortGracePeriod = t.AbortGracePeriod
}
return json.Marshal(configs)
}

var _ json.Unmarshaler = &Thresholds{}
var _ json.Marshaler = &Thresholds{}
36 changes: 18 additions & 18 deletions stats/thresholds_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@ func TestNewThreshold(t *testing.T) {
rt := goja.New()
abortOnFail := false
gracePeriod := types.NullDurationFrom(2 * time.Second)
th, err := NewThreshold(src, rt, abortOnFail, gracePeriod)
th, err := newThreshold(src, rt, abortOnFail, gracePeriod)
assert.NoError(t, err)

assert.Equal(t, src, th.Source)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
assert.NotNil(t, th.pgm)
assert.Equal(t, rt, th.rt)
assert.Equal(t, abortOnFail, th.AbortOnFail)
Expand All @@ -48,40 +48,40 @@ func TestNewThreshold(t *testing.T) {

func TestThresholdRun(t *testing.T) {
t.Run("true", func(t *testing.T) {
th, err := NewThreshold(`1+1==2`, goja.New(), false, types.NullDuration{})
th, err := newThreshold(`1+1==2`, goja.New(), false, types.NullDuration{})
assert.NoError(t, err)

t.Run("no taint", func(t *testing.T) {
b, err := th.RunNoTaint()
b, err := th.runNoTaint()
assert.NoError(t, err)
assert.True(t, b)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
})

t.Run("taint", func(t *testing.T) {
b, err := th.Run()
b, err := th.run()
assert.NoError(t, err)
assert.True(t, b)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
})
})

t.Run("false", func(t *testing.T) {
th, err := NewThreshold(`1+1==4`, goja.New(), false, types.NullDuration{})
th, err := newThreshold(`1+1==4`, goja.New(), false, types.NullDuration{})
assert.NoError(t, err)

t.Run("no taint", func(t *testing.T) {
b, err := th.RunNoTaint()
b, err := th.runNoTaint()
assert.NoError(t, err)
assert.False(t, b)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
})

t.Run("taint", func(t *testing.T) {
b, err := th.Run()
b, err := th.run()
assert.NoError(t, err)
assert.False(t, b)
assert.True(t, th.Failed)
assert.True(t, th.LastFailed)
})
})
}
Expand All @@ -99,7 +99,7 @@ func TestNewThresholds(t *testing.T) {
assert.Len(t, ts.Thresholds, 2)
for i, th := range ts.Thresholds {
assert.Equal(t, sources[i], th.Source)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
assert.False(t, th.AbortOnFail)
assert.NotNil(t, th.pgm)
assert.Equal(t, ts.Runtime, th.rt)
Expand All @@ -114,16 +114,16 @@ func TestNewThresholdsWithConfig(t *testing.T) {
assert.Len(t, ts.Thresholds, 0)
})
t.Run("two", func(t *testing.T) {
configs := []ThresholdConfig{
configs := []thresholdConfig{
{`1+1==2`, false, types.NullDuration{}},
{`1+1==4`, true, types.NullDuration{}},
}
ts, err := NewThresholdsWithConfig(configs)
ts, err := newThresholdsWithConfig(configs)
assert.NoError(t, err)
assert.Len(t, ts.Thresholds, 2)
for i, th := range ts.Thresholds {
assert.Equal(t, configs[i].Threshold, th.Source)
assert.False(t, th.Failed)
assert.False(t, th.LastFailed)
assert.Equal(t, configs[i].AbortOnFail, th.AbortOnFail)
assert.NotNil(t, th.pgm)
assert.Equal(t, ts.Runtime, th.rt)
Expand All @@ -134,7 +134,7 @@ func TestNewThresholdsWithConfig(t *testing.T) {
func TestThresholdsUpdateVM(t *testing.T) {
ts, err := NewThresholds(nil)
assert.NoError(t, err)
assert.NoError(t, ts.UpdateVM(DummySink{"a": 1234.5}, 0))
assert.NoError(t, ts.updateVM(DummySink{"a": 1234.5}, 0))
assert.Equal(t, 1234.5, ts.Runtime.Get("a").ToFloat())
}

Expand Down Expand Up @@ -170,7 +170,7 @@ func TestThresholdsRunAll(t *testing.T) {

assert.NoError(t, err)

b, err := ts.RunAll(runDuration)
b, err := ts.runAll(runDuration)

if data.err {
assert.Error(t, err)
Expand Down