diff --git a/.editorconfig b/.editorconfig index c893398..191dd92 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,4 +18,8 @@ trim_trailing_whitespace = false [{*.js,*.j2}] indent_style = space +indent_size = 2 + +[{*.yaml, *.yml}] +indent_style = space indent_size = 2 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9956232..970c939 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,5 @@ mysql/data .env nebula-bench.db .vscode -output \ No newline at end of file +output +third/*/data \ No newline at end of file diff --git a/README.md b/README.md index 12f667f..2bf2fab 100644 --- a/README.md +++ b/README.md @@ -111,8 +111,11 @@ python3 run.py stress run # run all scenarios with 10 virtual users, every scenario lasts 3 seconds. python3 run.py stress run -vu 10 -d 3 +# list all stress test scenarios +python3 run.py stress scenarios + # run go.Go1Step scenarios with 10 virtual users, every scenario lasts 3 seconds. -python3 run.py stress run -vu 10 -d 3 -s go.Go1Step +python3 run.py stress run -vu 10 -d 3 -scenario go.Go1Step ``` k6 config file, summary result and outputs are in `output` folder. e.g. diff --git a/README_cn.md b/README_cn.md index 4f9b60f..5cde7e5 100644 --- a/README_cn.md +++ b/README_cn.md @@ -103,8 +103,11 @@ python3 run.py stress run # run all scenarios with 10 virtual users, every scenario lasts 3 seconds. python3 run.py stress run -vu 10 -d 3 +# list all stress test scenarios +python3 run.py stress scenarios + # run go.Go1Step scenarios with 10 virtual users, every scenario lasts 3 seconds. -python3 run.py stress run -vu 10 -d 3 -s go.Go1Step +python3 run.py stress run -vu 10 -d 3 -scenario go.Go1Step ``` k6 config file, summary result and outputs are in `output` folder. e.g. diff --git a/env b/env index f16f160..37f107a 100644 --- a/env +++ b/env @@ -4,4 +4,4 @@ #NEBULA_PASSWORD=nebula #NEBULA_ADDRESS=127.0.0.1:9669 #NEBULA_MAX_CONNECTION=100 - +#INFLUXDB_URL=http://192.168.8.60:8086/k6 diff --git a/nebula_bench/cli.py b/nebula_bench/cli.py index 4c95c02..2f0b435 100644 --- a/nebula_bench/cli.py +++ b/nebula_bench/cli.py @@ -5,7 +5,7 @@ from nebula_bench.utils import logger from nebula_bench.controller import NebulaController from nebula_bench.utils import run_process -from nebula_bench.stress import StressFactory +from nebula_bench.stress import StressFactory, load_scenarios SH_COMMAND = "/bin/bash" @@ -136,7 +136,7 @@ def stress(): @click.option( "-d", "--duration", default=60, help="duration for every scenario, unit: second, default: 60" ) -@click.option("-s", "--scenarioes", default="all", help="special scenarioes, e.g. go.Go1Step") +@click.option("-scenario", default="all", help="run special scenario, e.g. go.Go1Step") @click.option("-c", "--controller", default="k6", help="using which test tool") @click.option( "--dry-run", @@ -145,7 +145,7 @@ def stress(): help="Dry run, just dump stress testing config file, default: False", ) def run( - folder, address, user, password, space, vid_type, scenarioes, controller, vu, duration, dry_run + folder, address, user, password, space, vid_type, scenario, controller, vu, duration, dry_run ): stress = StressFactory.gen_stress( _type=controller, @@ -155,7 +155,7 @@ def run( password=password, space=space, vid_type=vid_type, - scenarios=scenarioes, + scenarios=scenario, vu=vu, duration=duration, dry_run=dry_run, @@ -163,3 +163,14 @@ def run( stress.run() pass + + +@stress.command() +def scenarios(): + click.echo("All scenarios as below:") + + scenarios = load_scenarios("all") + for s in scenarios: + module = s.__module__.split(".")[-1] + name = s.__name__ + click.echo("\t{}.{}".format(module, name)) diff --git a/nebula_bench/setting.py b/nebula_bench/setting.py index c1a5d88..088dc97 100644 --- a/nebula_bench/setting.py +++ b/nebula_bench/setting.py @@ -27,3 +27,4 @@ NEBULA_MAX_CONNECTION = 400 SQLALCHEMY_URI = os.environ.get("SQLALCHEMY_URI") or "sqlite:///./nebula-bench.db" +INFLUXDB_URL = os.environ.get("INFLUXDB_URL") or "http://127.0.0.1:8086/k6" diff --git a/nebula_bench/stress.py b/nebula_bench/stress.py index e19dac9..857e54f 100644 --- a/nebula_bench/stress.py +++ b/nebula_bench/stress.py @@ -3,12 +3,24 @@ import inspect from pathlib import Path +import click + from nebula_bench.utils import load_class, jinja_dump, run_process from nebula_bench.common.base import BaseScenario from nebula_bench.utils import logger from nebula_bench import setting +def load_scenarios(scenarios): + if scenarios.strip().upper() == "ALL": + r = load_class("nebula_bench.scenarios", True, BaseScenario) + else: + r = load_class("nebula_bench.scenarios", False, BaseScenario, scenarios) + + r = [x for x in r if x.abstract == False] + return r + + class Stress(object): def __init__( self, @@ -35,15 +47,7 @@ def __init__( self.vu = vu self.duration = duration self.dry_run = dry_run - self.load_scenarios(scenarios) - - def load_scenarios(self, scenarios): - if scenarios.strip().upper() == "ALL": - self.scenarios = load_class("nebula_bench.scenarios", True, BaseScenario) - else: - self.scenarios = load_class("nebula_bench.scenarios", False, BaseScenario, scenarios) - - self.scenarios = [x for x in self.scenarios if x.abstract == False] + self.scenarios = load_scenarios(scenarios) logger.info("total stress test scenarios is {}".format(len(self.scenarios))) # dump config file @@ -141,9 +145,15 @@ def run(self): str(self.vu), "-d", "{}s".format(self.duration), + "--summary-trend-stats", + "min,avg,med,max,p(90),p(95),p(99)", + "--out", + "influxdb={}".format(setting.INFLUXDB_URL), "--summary-export", "{}/result_{}.json".format(self.output_folder, scenario.name), ] + click.echo("run command as below:") + click.echo(" ".join(command)) if self.dry_run is not None and self.dry_run: continue run_process(command) diff --git a/third/exporter/docker-compose.yaml b/third/exporter/docker-compose.yaml new file mode 100644 index 0000000..1d08867 --- /dev/null +++ b/third/exporter/docker-compose.yaml @@ -0,0 +1,35 @@ + +version: '3.7' + + +services: + node-exporter: + image: prom/node-exporter + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - --collector.filesystem.ignored-mount-points + - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)" + network_mode: host + restart: always + + process-exporter: + image: ncabatoff/process-exporter + volumes: + - /proc:/host/proc + - ./filename.yml:/config/filename.yml + command: + - --procfs + - /host/proc + - -config.path + - /config/filename.yml + + network_mode: host + + + restart: always + privileged: true diff --git a/third/exporter/filename.yml b/third/exporter/filename.yml new file mode 100644 index 0000000..08dee8b --- /dev/null +++ b/third/exporter/filename.yml @@ -0,0 +1,6 @@ +process_names: + - name: "qa-61-{{.Comm}}" + comm: + - nebula-metad + - nebula-storaged + - nebula-graphd diff --git a/third/images/k6_result_1.jpg b/third/images/k6_result_1.jpg new file mode 100644 index 0000000..a79c49b Binary files /dev/null and b/third/images/k6_result_1.jpg differ diff --git a/third/images/node_exporter_1.jpg b/third/images/node_exporter_1.jpg new file mode 100644 index 0000000..ff06876 Binary files /dev/null and b/third/images/node_exporter_1.jpg differ diff --git a/third/images/node_exporter_2.jpg b/third/images/node_exporter_2.jpg new file mode 100644 index 0000000..1a5b580 Binary files /dev/null and b/third/images/node_exporter_2.jpg differ diff --git a/third/images/process_exporter_1.jpg b/third/images/process_exporter_1.jpg new file mode 100644 index 0000000..cdbdeac Binary files /dev/null and b/third/images/process_exporter_1.jpg differ diff --git a/mysql/docker-compose.yaml b/third/mysql/docker-compose.yaml similarity index 84% rename from mysql/docker-compose.yaml rename to third/mysql/docker-compose.yaml index ab8126a..464a47a 100644 --- a/mysql/docker-compose.yaml +++ b/third/mysql/docker-compose.yaml @@ -2,8 +2,8 @@ version: '3.4' services: mysql: image: mysql:5.6 - ports: - - 3306:3306 + # ports: + # - 3306:3306 environment: - MYSQL_ROOT_PASSWORD=admin - TZ=Asiz/Shanghai @@ -12,3 +12,4 @@ services: - ./mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf:rw - ./data:/var/lib/mysql + network_mode: host diff --git a/mysql/mysqld.cnf b/third/mysql/mysqld.cnf similarity index 100% rename from mysql/mysqld.cnf rename to third/mysql/mysqld.cnf diff --git a/third/promethues/docker-compose.yaml b/third/promethues/docker-compose.yaml new file mode 100644 index 0000000..aceead3 --- /dev/null +++ b/third/promethues/docker-compose.yaml @@ -0,0 +1,58 @@ + +version: '3.7' + + +networks: + monitor: + +services: + prometheus: + image: prom/prometheus:v2.1.0 + user: root + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./data/prometheus:/prometheus:rw + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/usr/share/prometheus/console_libraries' + - '--web.console.templates=/usr/share/prometheus/consoles' + - '--storage.tsdb.retention=15d' + ports: + - 9090:9090 + # links: + # - alertmanager:alertmanager + # depends_on: + networks: + - monitor + restart: always + + grafana: + image: grafana/grafana + user: root + depends_on: + - prometheus + ports: + - 3000:3000 + + networks: + - monitor + restart: always + + volumes: + - ./data/grafana:/var/lib/grafana:rw + + + influxdb: + image: influxdb:1.8 + container_name: influxdb + ports: + - "8086:8086" + - "8083:8083" + environment: + - INFLUXDB_DATA_ENGINE=tsm1 + - INFLUXDB_DB=k6 + + volumes: + # Data persistency + - ./data/influxdb:/var/lib/influxdb \ No newline at end of file diff --git a/third/promethues/k6_influxdb.json b/third/promethues/k6_influxdb.json new file mode 100644 index 0000000..ea6060a --- /dev/null +++ b/third/promethues/k6_influxdb.json @@ -0,0 +1,1648 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for visualizing results from the k6.io load testing tool, using the InfluxDB exporter", + "editable": true, + "gnetId": 2587, + "graphTooltip": 2, + "id": 6, + "iteration": 1626863115469, + "links": [], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 1, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Active VUs", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "vus", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Users", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 1 + }, + "hiddenSeries": false, + "id": 17, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "Requests per Second", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "iterations", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:78", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:79", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 7, + "interval": "1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "Num Errors", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "errors", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "count" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Errors Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 1 + }, + "hiddenSeries": false, + "id": 10, + "interval": ">1s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "total": true, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Num Errors", + "color": "#BF1B00" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "alias": "$tag_check", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "check" + ], + "type": "tag" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "checks", + "orderByTime": "ASC", + "policy": "default", + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Checks Per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [], + "repeat": "Measurement", + "title": "$Measurement", + "type": "row" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 11, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT mean(\"value\") FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (avg)", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 14, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (max)", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 15, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT median(\"value\") FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (med)", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 16, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (min)", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 12, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM $Measurement WHERE $timeFilter", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (p90)", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "InfluxDB", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "µs" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 9 + }, + "id": 13, + "interval": null, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.0.6", + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM $Measurement WHERE $timeFilter ", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (p95)", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "InfluxDB", + "description": "Grouped by 1 sec intervals", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + }, + "height": "250px", + "hiddenSeries": false, + "id": 5, + "interval": ">1s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.0.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "max", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT max(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "C", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + }, + { + "alias": "p95", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 95) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "D", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + 95 + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "p90", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT percentile(\"value\", 90) FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [ + "90" + ], + "type": "percentile" + } + ] + ], + "tags": [] + }, + { + "alias": "min", + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "none" + ], + "type": "fill" + } + ], + "measurement": "/^$Measurement$/", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT min(\"value\") FROM /^$Measurement$/ WHERE $timeFilter and value > 0 GROUP BY time($__interval) fill(none)", + "rawQuery": true, + "refId": "E", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "min" + } + ] + ], + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "$Measurement (over time)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:196", + "format": "µs", + "label": null, + "logBase": 2, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:197", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "rgb(0, 234, 255)", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "InfluxDB", + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + }, + "heatmap": {}, + "height": "250px", + "hideZeroBuckets": false, + "highlightCards": true, + "id": 8, + "interval": ">1s", + "legend": { + "show": false + }, + "links": [], + "reverseYBuckets": false, + "targets": [ + { + "dsType": "influxdb", + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "http_req_duration", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"value\" FROM $Measurement WHERE $timeFilter and value > 0", + "rawQuery": true, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "field" + }, + { + "params": [], + "type": "mean" + } + ] + ], + "tags": [] + } + ], + "title": "$Measurement (over time)", + "tooltip": { + "show": true, + "showHistogram": true + }, + "tooltipDecimals": null, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "µs", + "logBase": 2, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 20, + "panels": [], + "repeat": null, + "title": "Dashboard Row", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "Measurement", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "latency", + "value": "latency" + }, + { + "selected": false, + "text": "responseTime", + "value": "responseTime" + } + ], + "query": "latency,responseTime", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "k6 Load Testing Results - nebula", + "uid": "doglkIZ7z", + "version": 2 +} \ No newline at end of file diff --git a/third/promethues/prometheus.yml b/third/promethues/prometheus.yml new file mode 100644 index 0000000..266dc12 --- /dev/null +++ b/third/promethues/prometheus.yml @@ -0,0 +1,38 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +# alerting: +# alertmanagers: +# - static_configs: +# - targets: ['192.168.159.129:9093'] +# - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + - "node_down.yml" + # - "first_rules.yml" + # - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: "prometheus" + static_configs: + - targets: ["192.168.8.152:9090"] + + - job_name: "node" + scrape_interval: 8s + static_configs: + - targets: + - 192.168.8.217:9100 + - 192.168.8.217:9256 + - 192.168.8.218:9100 + - 192.168.8.218:9256 + - 192.168.8.219:9100 + - 192.168.8.219:9256 + \ No newline at end of file diff --git a/third/readme.md b/third/readme.md new file mode 100644 index 0000000..f2113c6 --- /dev/null +++ b/third/readme.md @@ -0,0 +1,82 @@ +# Monitor Readme + +在压测过程中，监控是不可少的环节，监控可以帮助我们： + +1. 根据不同的监控结果，印证推测，不断调整测试方案。 +2. 将更详尽的过程结果记录，方便报告和总结。 + +## 监控类型 + +总体上有服务端监控和客户端监控。 + +### 服务端监控 + +组件使用 + +* Promethues +* Node exporter +* Process exporter + +### 客户端监控 + +组件使用 + +* Node exporter +* Influxdb，用来收集 qps、latency、response time 等数据。 + +## 部署步骤 + +### Promethues + +* 将 promethues 复制到任意一个节点 +* prometheus.yml 中，设置要监控的 nebula 节点，默认 node exporter 端口 9100，process exporter 端口 9256，然后启动 docker-compose。 + +```bash +docker-compose up -d +``` + +默认 promethues 的数据保留 15 天，如果需要更改，请自行修改 docker-compose 中 `--storage.tsdb.retention=15d'` 配置。 + +### Exporter + +* 分别在 nebula 节点，执行压测的机器上，复制 exporter 目录。 +* 修改 `filename.yml` 中的 name，方便和其他机器做区分，然后启动 docker-compose。 + +```bash +docker-compose up -d +``` + +### Grafana + +为了方便，grafana 默认放在 promethues 的 docker-compose.yml 中，启动 prometheus 时，会自动启动。 + +* 登录 grafana，默认端口 3000，用户名/密码是: admin/admin。 +* 添加 prometheus 数据源。 +* import dashboard。 + * `249` process exporter + * `1860` node exporter + +### InfluxDB + +为了方便，influxdb 默认放在 promethues 的 docker-compose.yml 中，启动 prometheus 时，会自动启动。 +docker-compose 中，配置了默认数据库为 `k6`。 + +* 在 `.env` 中，配置 influxdb 的地址，如：`http://192.168.8.60:8086/k6`。 +* 登录 grafana，添加 influxdb 数据源。 +* 添加 dashboard，json 文件见 [k6_influxdb.json](./promethues/k6_influxdb.json)。 +* 执行压力测试，查看 grafana 的图。 + +## 效果 + +### node exporter + +![node_exporter_1](./images/node_exporter_1.jpg) +![node_exporter_2](./images/node_exporter_2.jpg) + +### process exporter + +![process_exporter_1](./images/process_exporter_1.jpg) + +### k6 testing results + +![k6_result_1](./images/k6_result_1.jpg)