diff --git a/.github/license-checker.yml b/.github/license-checker.yml new file mode 100644 index 00000000000..f0087cc8eb8 --- /dev/null +++ b/.github/license-checker.yml @@ -0,0 +1,26 @@ +header: + license: + spdx-id: Apache-2.0 + copyright-owner: TiKV Project Authors. + paths-ignore: + - 'config/' + - 'dev/' + - 'docs/' + - 'metrics/' + - 'LICENSE' + - 'Makefile' + - 'pom.xml' + - 'shell.nix' + - '.ci/' + - '.gitignore' + - '.gitattributes' + - '.github/' + - '**/*.md' + - '**/*.properties' + - '**/*.json' + - '**/*.pem' + - '**/*.crt' + - '**/*.g4' + - 'src/main/java/io/grpc/' + - 'src/main/java/io/netty/' + comment: on-failure diff --git a/dev/README.md b/dev/README.md index b0eccd6057b..e924397398f 100644 --- a/dev/README.md +++ b/dev/README.md @@ -1,8 +1,6 @@ -# TiSpark Dev Tools Guide +# TiKV Java Client Dev Tools -## Formatting - -### Java Format +## Code Formatting TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https://github.com/coveooss/fmt-maven-plugin) which follows Google's code styleguide. It is also checked on CI before build. @@ -18,4 +16,4 @@ TiKV Java Client formats its code using [Google-Java-Format Maven Plugin](https: ```shell script ./dev/javafmt - ``` \ No newline at end of file + ``` diff --git a/scripts/proto.sh b/dev/proto.sh similarity index 100% rename from scripts/proto.sh rename to dev/proto.sh diff --git a/pom.xml b/pom.xml index 4efa43f8db4..c0e14dc0507 100644 --- a/pom.xml +++ b/pom.xml @@ -278,7 +278,7 @@ clone proto files - ${basedir}/scripts/proto.sh + ${basedir}/dev/proto.sh validate diff --git a/scripts/rstats.py b/scripts/rstats.py new file mode 100755 index 00000000000..4c483fb8ab6 --- /dev/null +++ b/scripts/rstats.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +#!coding:utf-8 + +# Copyright 2022 TiKV Project Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import json +import argparse + +def main(): + pattern = r'.*SlowLog:.*' + slowstr = "SlowLog:" + grpc_pattern = "gRPC tikvpb.Tikv" + backoff_pattern = "backoff " + + args = parse_args() + items = [] + with open(args.slowlog, encoding = 'utf-8') as f: + for line in f.readlines(): + matched = re.match(pattern, line, re.M|re.I) + if matched is not None: + log = json.loads(line[(line.index(slowstr) + len(slowstr)):]) + item = { + 'req': log['func'], + 'start': log['start'], + 'tot_lat': latency_ms(log), + 'tot_grpc': 0, + 'tot_bo': 0, + } + items.append(item) + for span in log['spans']: + if grpc_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_grpc'] += latency_ms(span) + elif backoff_pattern in span['name'] and span['duration'] != 'N/A': + item['tot_bo'] += latency_ms(span) + + if args.order == "total": + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + elif args.order == "grpc": + items = sorted(items, key=lambda d: d['tot_grpc'], reverse=True) + elif args.order == "backoff": + items = sorted(items, key=lambda d: d['tot_bo'], reverse=True) + else: + print("unsupported order option, use default value: total") + items = sorted(items, key=lambda d: d['tot_lat'], reverse=True) + + fmtStr = "{:<12} {:<14} {:<14} {:<20} {:<20}" + print(fmtStr.format("Request", "Start", "Total Lat(ms)", "Total gRPC Lat(ms)", "Total Backoff Lat(ms)")) + for item in items: + print(fmtStr.format(item['req'], item['start'], item['tot_lat'], item['tot_grpc'], item['tot_bo'])) + +def latency_ms(span): + return int(span['duration'][:len(span['duration'])-2]) + + +def parse_args(): + parser = argparse.ArgumentParser(description="rstats: A TiKV Java Client Request Stats Analyzer") + parser.add_argument("-o", dest="order", default="total", help="order the output, default: total. accepted value: total, grpc, backoff") + parser.add_argument("slowlog", help="slow log file") + return parser.parse_args() + +if __name__ == '__main__': + main() +