Skip to content

Commit

Permalink
improvement(perf): add validation rules for latency decorator
Browse files Browse the repository at this point in the history
Added validation rules for results sent by
`latency_calculator_decorator` to Argus.
Each workload and result name (nemesis, predefined step) may set own
rules.

Current rules were created based on existing results - to pass typical
good results.

closes: #9237
  • Loading branch information
soyacz committed Dec 3, 2024
1 parent 51f564e commit ca452ea
Show file tree
Hide file tree
Showing 21 changed files with 420 additions and 25 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
latency_decorator_error_thresholds:
write:
_mgmt_repair_cli:
duration:
fixed_limit: 10800
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 2500
decommission_nodes:
duration:
fixed_limit: 1800
replace_node:
duration:
fixed_limit: 3600

read:
_mgmt_repair_cli:
duration:
fixed_limit: 3600
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 3000
decommission_nodes:
duration:
fixed_limit: 1800
replace_node:
duration:
fixed_limit: 3000

mixed:
_mgmt_repair_cli:
duration:
fixed_limit: 5000
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 2500
decommission_nodes:
duration:
fixed_limit: 1800
replace_node:
duration:
fixed_limit: 3200
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
latency_decorator_error_thresholds:
write:
_mgmt_repair_cli:
duration:
fixed_limit: 10800 # 3h this one typically will fail
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 7200
decommission_nodes:
duration:
fixed_limit: 8000
replace_node:
duration:
fixed_limit: 3600

read:
_mgmt_repair_cli:
duration:
fixed_limit: 10800 # 3h this one typically will fail
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 2600
decommission_nodes:
duration:
fixed_limit: 2600
replace_node:
duration:
fixed_limit: 2600

mixed:
_mgmt_repair_cli:
duration:
fixed_limit: 10800 # 3h this one typically will fail
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 2800
decommission_nodes:
duration:
fixed_limit: 2600
replace_node:
duration:
fixed_limit: 2600
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
latency_decorator_error_thresholds:
write:
_mgmt_repair_cli:
duration:
fixed_limit: 28000
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 10000
decommission_nodes:
duration:
fixed_limit: 10000
replace_node:
duration:
fixed_limit: 2500

read:
_mgmt_repair_cli:
duration:
fixed_limit: 3000
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 2500
decommission_nodes:
duration:
fixed_limit: 2800
replace_node:
duration:
fixed_limit: 1500

mixed:
_mgmt_repair_cli:
duration:
fixed_limit: 5000
_terminate_and_wait:
duration:
fixed_limit: 500
add_new_nodes:
duration:
fixed_limit: 3000
decommission_nodes:
duration:
fixed_limit: 3600
replace_node:
duration:
fixed_limit: 1700
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
latency_decorator_error_thresholds:
write:
"200000":
P90 write:
fixed_limit: 5
P99 write:
fixed_limit: 10
"300000":
P99 write:
fixed_limit: 1000
"400000":
P99 write:
fixed_limit: 1000
unthrottled:
P90 write:
fixed_limit: null
P99 write:
fixed_limit: null
Throughput write:
best_pct: 10

read:
"150000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 1
"300000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 1
"450000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 3
"600000":
P90 read:
fixed_limit: 1.5
P99 read:
fixed_limit: 50
"700000":
P90 read:
fixed_limit: 3
P99 read:
fixed_limit: 50
unthrottled:
P90 read:
fixed_limit: null
P99 read:
fixed_limit: null
Throughput read:
best_pct: 10

mixed:
"50000":
P90 write:
fixed_limit: 1
P90 read:
fixed_limit: 1
P99 write:
fixed_limit: 2.5
P99 read:
fixed_limit: 2.5
"150000":
P90 write:
fixed_limit: 1
P90 read:
fixed_limit: 1.7
P99 write:
fixed_limit: 3
P99 read:
fixed_limit: 3
"300000":
P90 write:
fixed_limit: 3
P90 read:
fixed_limit: 3
P99 write:
fixed_limit: 5
P99 read:
fixed_limit: 5
"450000":
P90 write:
fixed_limit: 3
P90 read:
fixed_limit: 4
P99 write:
fixed_limit: 15
P99 read:
fixed_limit: 15
unthrottled:
P90 write:
fixed_limit: null
P90 read:
fixed_limit: null
P99 write:
fixed_limit: null
P99 read:
fixed_limit: null
Throughput write:
best_pct: 10
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
latency_decorator_error_thresholds:
write:
"200000":
P90 write:
fixed_limit: 5
P99 write:
fixed_limit: 10
"300000":
P90 write:
fixed_limit: 1000
P99 write:
fixed_limit: 1000
"400000":
P90 write:
fixed_limit: 1000
P99 write:
fixed_limit: 1000
unthrottled:
P90 write:
fixed_limit: null
P99 write:
fixed_limit: null
Throughput write:
best_pct: 10

read:
"150000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 1
"300000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 1
"450000":
P90 read:
fixed_limit: 1
P99 read:
fixed_limit: 3
"600000":
P90 read:
fixed_limit: 1.5
P99 read:
fixed_limit: 50
"700000":
P90 read:
fixed_limit: 3
P99 read:
fixed_limit: 50
unthrottled:
P90 read:
fixed_limit: null
P99 read:
fixed_limit: null
Throughput read:
best_pct: 10

mixed:
"50000":
P90 write:
fixed_limit: 1
P90 read:
fixed_limit: 1
P99 write:
fixed_limit: 2.5
P99 read:
fixed_limit: 2.5
"150000":
P90 write:
fixed_limit: 1
P90 read:
fixed_limit: 1.7
P99 write:
fixed_limit: 3
P99 read:
fixed_limit: 3
"300000":
P90 write:
fixed_limit: 3
P90 read:
fixed_limit: 3
P99 write:
fixed_limit: 5
P99 read:
fixed_limit: 5
"450000":
P90 write:
fixed_limit: 3
P90 read:
fixed_limit: 4
P99 write:
fixed_limit: 15
P99 read:
fixed_limit: 15
unthrottled:
P90 write:
fixed_limit: null
P90 read:
fixed_limit: null
P99 write:
fixed_limit: null
P99 read:
fixed_limit: null
Throughput write:
best_pct: 10
24 changes: 24 additions & 0 deletions defaults/test_default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -268,3 +268,27 @@ zero_token_instance_type_db: 'i4i.large'
use_zero_nodes: false

latte_schema_parameters: {}

latency_decorator_error_thresholds:
write:
default:
P90 write:
fixed_limit: 5
P99 write:
fixed_limit: 10
read:
default:
P90 read:
fixed_limit: 5
P99 read:
fixed_limit: 10
mixed:
default:
P90 write:
fixed_limit: 5
P90 read:
fixed_limit: 5
P99 write:
fixed_limit: 10
P99 read:
fixed_limit: 10
Loading

0 comments on commit ca452ea

Please sign in to comment.