From e8c354ccd0d47e74cc89a2ef37969675bb410b64 Mon Sep 17 00:00:00 2001 From: ii2day Date: Tue, 23 May 2023 20:09:41 +0800 Subject: [PATCH] optimize DNS client performance Signed-off-by: ii2day --- charts/templates/configmap.yaml | 1 + charts/values.yaml | 3 + go.mod | 2 - go.sum | 4 - pkg/loadRequest/loadDns/dns.go | 240 ++----------- pkg/loadRequest/loadDns/dns_reporter.go | 65 ++++ pkg/loadRequest/loadDns/dns_requester.go | 222 ++++++++++++ pkg/loadRequest/loadDns/dns_suite_test.go | 3 + pkg/loadRequest/loadDns/dns_test.go | 8 +- pkg/pluginManager/netdns/agentExecuteTask.go | 16 +- pkg/types/types.go | 1 + .../andres-erbsen/clock/.travis.yml | 7 - vendor/github.com/andres-erbsen/clock/LICENSE | 21 -- .../github.com/andres-erbsen/clock/README.md | 104 ------ .../github.com/andres-erbsen/clock/clock.go | 317 ------------------ vendor/go.uber.org/ratelimit/.gitignore | 6 - vendor/go.uber.org/ratelimit/CHANGELOG.md | 23 -- vendor/go.uber.org/ratelimit/LICENSE | 21 -- vendor/go.uber.org/ratelimit/Makefile | 46 --- vendor/go.uber.org/ratelimit/README.md | 46 --- .../go.uber.org/ratelimit/limiter_atomic.go | 110 ------ .../ratelimit/limiter_mutexbased.go | 88 ----- vendor/go.uber.org/ratelimit/ratelimit.go | 135 -------- vendor/modules.txt | 6 - 24 files changed, 338 insertions(+), 1157 deletions(-) create mode 100644 pkg/loadRequest/loadDns/dns_reporter.go create mode 100644 pkg/loadRequest/loadDns/dns_requester.go delete mode 100644 vendor/github.com/andres-erbsen/clock/.travis.yml delete mode 100644 vendor/github.com/andres-erbsen/clock/LICENSE delete mode 100644 vendor/github.com/andres-erbsen/clock/README.md delete mode 100644 vendor/github.com/andres-erbsen/clock/clock.go delete mode 100644 vendor/go.uber.org/ratelimit/.gitignore delete mode 100644 vendor/go.uber.org/ratelimit/CHANGELOG.md delete mode 100644 vendor/go.uber.org/ratelimit/LICENSE delete mode 100644 vendor/go.uber.org/ratelimit/Makefile delete mode 100644 vendor/go.uber.org/ratelimit/README.md delete mode 100644 vendor/go.uber.org/ratelimit/limiter_atomic.go delete mode 100644 vendor/go.uber.org/ratelimit/limiter_mutexbased.go delete mode 100644 vendor/go.uber.org/ratelimit/ratelimit.go diff --git a/charts/templates/configmap.yaml b/charts/templates/configmap.yaml index 00054e37..08728ad2 100644 --- a/charts/templates/configmap.yaml +++ b/charts/templates/configmap.yaml @@ -22,6 +22,7 @@ data: nethttp_defaultMaxIdleConnsPerHost: {{ .Values.feature.nethttp_defaultMaxIdleConnsPerHost }} nethttp_defaultRequest_DurationInSecond: {{ .Values.feature.nethttp_defaultRequest_DurationInSecond }} nethttp_defaultRequest_PerRequestTimeoutInMS: {{ .Values.feature.nethttp_defaultRequest_PerRequestTimeoutInMS }} + netdns_defaultConcurrency: {{ .Values.feature.netdns_defaultConcurrency }} multusPodAnnotationKey: {{ .Values.feature.multusPodAnnotationKey }} crdMaxHistory: {{ .Values.feature.crdMaxHistory }} {{- if .Values.feature.enableIPv4 }} diff --git a/charts/values.yaml b/charts/values.yaml index 9b16a5e8..595dd493 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -55,6 +55,9 @@ feature: ## @param feature.nethttp_defaultFail_MeanDelayInMs mean delay in ms for kind nethttp nethttp_defaultFail_MeanDelayInMs: 2000 + ## @param feature.netdns_defaultConcurrency concurrency for kind netdns + netdns_defaultConcurrency: 50 + ## @param feature.taskPollIntervalInSecond the interval to poll the task in controller and agent pod taskPollIntervalInSecond: 5 diff --git a/go.mod b/go.mod index 2f647d94..20d5bd01 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( go.opentelemetry.io/otel/metric v0.34.0 go.opentelemetry.io/otel/sdk v1.11.1 go.opentelemetry.io/otel/sdk/metric v0.33.0 - go.uber.org/ratelimit v0.2.0 go.uber.org/zap v1.24.0 golang.org/x/net v0.10.0 google.golang.org/grpc v1.53.0 @@ -48,7 +47,6 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/Masterminds/sprig/v3 v3.2.2 // indirect - github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 84c8c86c..317359bd 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI= -github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= @@ -538,8 +536,6 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA= -go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= diff --git a/pkg/loadRequest/loadDns/dns.go b/pkg/loadRequest/loadDns/dns.go index e05c03cf..839a8139 100644 --- a/pkg/loadRequest/loadDns/dns.go +++ b/pkg/loadRequest/loadDns/dns.go @@ -4,15 +4,12 @@ package loadDns import ( - "context" "fmt" "github.com/miekg/dns" - "github.com/spidernet-io/spiderdoctor/pkg/lock" - "github.com/spidernet-io/spiderdoctor/pkg/utils/stats" - "go.uber.org/ratelimit" "go.uber.org/zap" - "sync" "time" + + config "github.com/spidernet-io/spiderdoctor/pkg/types" ) type RequestProtocol string @@ -38,218 +35,45 @@ type DnsRequestData struct { DurationInSecond int } -// ------------------ - -type DelayMetric struct { - // Mean is the mean request latency. - Mean string `json:"mean"` - // P50 is the 50th percentile request latency. - P50 string `json:"50th"` - // P90 is the 90th percentile request latency. - P90 string `json:"90th"` - // P95 is the 95th percentile request latency. - P95 string `json:"95th"` - // P99 is the 99th percentile request latency. - P99 string `json:"99th"` - // Max is the maximum observed request latency. - Max string `json:"max"` - // Min is the minimum observed request latency. - Min string `json:"min"` -} - -// final metric -type DnsMetrics struct { - StartTime time.Time - EndTime time.Time - Duration string - - TargetDomain string - DnsServer string - DnsMethod string - - // succeed to query the ip - SucceedCount int - // failed to get response , or not get ip in the dns response - FailedCount int - TotalCount int - SuccessRate float64 - - // when succeed to get response - ReplyCode map[string]int - // error to send request, such as timeout - ErrorMap map[string]int - - DnsAnswer []dns.RR - - // delay information for success request - DelayForSuccess DelayMetric -} - -// metric for one request -type dnsMetric struct { - e error - rtt time.Duration - msg *dns.Msg -} - -func executeRequestOnce(c *dns.Client, conn *dns.Conn, msg *dns.Msg) *dnsMetric { - r := dnsMetric{} - r.msg, r.rtt, r.e = c.ExchangeWithConn(msg, conn) - return &r -} - -func ParseMetrics(final *DnsMetrics, validVals []float32) (*DnsMetrics, error) { - var e error - var t float32 - - final.SuccessRate = float64(final.SucceedCount) / float64(final.TotalCount) - - // delay - if final.SucceedCount > 0 { - t, e = stats.Mean(validVals) - if e != nil { - return nil, fmt.Errorf("failed to parse mean delay, error=%v", e) - } - final.DelayForSuccess.Mean = parseTime(time.Duration(t)) - - t, e = stats.Max(validVals) - if e != nil { - return nil, fmt.Errorf("failed to parse max delay, error=%v", e) - } - final.DelayForSuccess.Max = parseTime(time.Duration(t)) - - t, e = stats.Min(validVals) - if e != nil { - return nil, fmt.Errorf("failed to parse min delay, error=%v", e) - } - final.DelayForSuccess.Min = parseTime(time.Duration(t)) - - t, e = stats.Percentile(validVals, 50) - if e != nil { - return nil, fmt.Errorf("failed to parse 50 Percentile, error=%v", e) - } - final.DelayForSuccess.P50 = parseTime(time.Duration(t)) - - t, e = stats.Percentile(validVals, 90) - if e != nil { - return nil, fmt.Errorf("failed to parse 90 Percentile, error=%v", e) - } - final.DelayForSuccess.P90 = parseTime(time.Duration(t)) - - t, e = stats.Percentile(validVals, 95) - if e != nil { - return nil, fmt.Errorf("failed to parse 95 Percentile, error=%v", e) - } - final.DelayForSuccess.P95 = parseTime(time.Duration(t)) - - t, e = stats.Percentile(validVals, 99) - if e != nil { - return nil, fmt.Errorf("failed to parse 99 Percentile, error=%v", e) - } - final.DelayForSuccess.P99 = parseTime(time.Duration(t)) - } - - return final, nil -} - -func DnsRequest(logger *zap.Logger, req *DnsRequestData) (result *DnsMetrics, err error) { - ServerAddress := req.DnsServerAddr - l := &lock.Mutex{} +func DnsRequest(logger *zap.Logger, reqData *DnsRequestData) (result *Metrics, err error) { - logger.Sugar().Infof("dns ServerAddress=%v, request=%v, ", ServerAddress, req) + logger.Sugar().Infof("dns ServerAddress=%v, request=%v, ", reqData.DnsServerAddr, reqData) - if _, ok := dns.IsDomainName(req.TargetDomain); !ok { - return nil, fmt.Errorf("invalid domain name: %v", req.TargetDomain) + if _, ok := dns.IsDomainName(reqData.TargetDomain); !ok { + return nil, fmt.Errorf("invalid domain name: %v", reqData.TargetDomain) } // if not fqdn, the dns library will report error, so convert the format - if !dns.IsFqdn(req.TargetDomain) { - req.TargetDomain = dns.Fqdn(req.TargetDomain) - logger.Sugar().Debugf("convert target domain to fqdn %v", req.TargetDomain) + if !dns.IsFqdn(reqData.TargetDomain) { + reqData.TargetDomain = dns.Fqdn(reqData.TargetDomain) + logger.Sugar().Debugf("convert target domain to fqdn %v", reqData.TargetDomain) } - rl := ratelimit.New(req.Qps) - var wg sync.WaitGroup - d := time.Duration(req.DurationInSecond) * time.Second - ctx, cancel := context.WithTimeout(context.Background(), d) - defer cancel() - var duration time.Duration - logger.Sugar().Infof("begin to request %v for duration %v ", req.TargetDomain, d.String()) - - // -------- send all request - start := time.Now() - counter := 0 + duration := time.Duration(reqData.DurationInSecond) * time.Second - c := new(dns.Client) - c.Net = string(req.Protocol) - c.Timeout = time.Duration(req.PerRequestTimeoutInMs) * time.Millisecond - msg := new(dns.Msg).SetQuestion(req.TargetDomain, req.DnsType) - conn, _ := c.Dial(ServerAddress) - c.SingleInflight = true - - final := &DnsMetrics{ - ErrorMap: map[string]int{}, - DnsAnswer: []dns.RR{}, - ReplyCode: map[string]int{}, + w := &Work{ + Concurrency: config.AgentConfig.Configmap.NetdnsDefaultConcurrency, + QPS: reqData.Qps, + Timeout: reqData.PerRequestTimeoutInMs, + Msg: new(dns.Msg).SetQuestion(reqData.TargetDomain, reqData.DnsType), + Protocol: string(reqData.Protocol), + ServerAddr: reqData.DnsServerAddr, } - - validVals := []float32{} - - p := func(wg *sync.WaitGroup) { - r := executeRequestOnce(c, conn, msg) - l.Lock() - final.TotalCount++ - if r.e != nil { - final.FailedCount++ - final.ErrorMap[r.e.Error()]++ - } else { - if len(r.msg.Answer) > 0 && r.msg.Rcode == dns.RcodeSuccess { - final.SucceedCount++ - validVals = append(validVals, float32(r.rtt)) - } else { - final.FailedCount++ - } - rcodeStr := dns.RcodeToString[r.msg.Rcode] - final.ReplyCode[rcodeStr]++ - } - l.Unlock() - wg.Done() + w.Init() + + // The monitoring task timed out + if duration > 0 { + go func() { + time.Sleep(duration) + w.Stop() + }() } + logger.Sugar().Infof("begin to request %v for duration %v ", w.ServerAddr, duration.String()) + w.Run() + logger.Sugar().Infof("finish all request %v for %s ", w.report.totalCount, w.ServerAddr) + // Collect metric reports + metrics := w.AggregateMetric() -LOOP: - for { - select { - case <-ctx.Done(): - cancel() - duration = time.Since(start) - break LOOP - - default: - rl.Take() - counter++ - wg.Add(1) - go p(&wg) - } - } - wg.Wait() - end := time.Now() - logger.Sugar().Infof("finish all %v requests for %v ", counter, req.TargetDomain) - //-------- parse final metric - r, e := ParseMetrics(final, validVals) - if e != nil { - return nil, fmt.Errorf("failed to parse metric, %v", e) - } - r.StartTime = start - r.EndTime = end - r.Duration = duration.String() - r.TargetDomain = req.TargetDomain - r.DnsServer = ServerAddress - r.DnsMethod = string(req.Protocol) - - logger.Sugar().Infof("result : %v ", r) - return r, nil - -} + logger.Sugar().Infof("result : %v ", metrics) + return metrics, nil -func parseTime(t time.Duration) string { - return fmt.Sprintf("%.6fms", t.Seconds()*1000) } diff --git a/pkg/loadRequest/loadDns/dns_reporter.go b/pkg/loadRequest/loadDns/dns_reporter.go new file mode 100644 index 00000000..7d492670 --- /dev/null +++ b/pkg/loadRequest/loadDns/dns_reporter.go @@ -0,0 +1,65 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package loadDns + +import ( + "github.com/miekg/dns" + "time" +) + +// We report for max 1M results. +const maxRes = 1000000 + +type report struct { + avgTotal float64 + average float64 + tps float64 + + results chan *result + done chan bool + total time.Duration + errorDist map[string]int + lats []float32 + totalCount int64 + successCount int64 + failedCount int64 + ReplyCode map[string]int +} + +func newReport(results chan *result) *report { + return &report{ + results: results, + done: make(chan bool, 1), + errorDist: make(map[string]int), + lats: make([]float32, 0, maxRes), + ReplyCode: make(map[string]int), + } +} + +func runReporter(r *report) { + // Loop will continue until channel is closed + for res := range r.results { + r.totalCount++ + if res.err != nil { + r.errorDist[res.err.Error()]++ + r.failedCount++ + } else { + r.avgTotal += res.duration.Seconds() + if len(r.lats) < maxRes { + r.lats = append(r.lats, float32(res.duration.Milliseconds())) + } + rcodeStr := dns.RcodeToString[res.msg.Rcode] + r.ReplyCode[rcodeStr]++ + r.successCount++ + } + } + // Signal reporter is done. + r.done <- true +} + +func (r *report) finalize(total time.Duration) { + r.total = total + r.tps = float64(r.totalCount) / r.total.Seconds() + r.average = r.avgTotal / float64(len(r.lats)) +} diff --git a/pkg/loadRequest/loadDns/dns_requester.go b/pkg/loadRequest/loadDns/dns_requester.go new file mode 100644 index 00000000..d4c44e19 --- /dev/null +++ b/pkg/loadRequest/loadDns/dns_requester.go @@ -0,0 +1,222 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package loadDns + +import ( + "github.com/miekg/dns" + "github.com/spidernet-io/spiderdoctor/pkg/utils/stats" + "sync" + "time" +) + +// Max size of the buffer of result channel. +const maxResult = 1000000 + +type result struct { + err error + duration time.Duration + msg *dns.Msg +} + +type Metrics struct { + StartTime time.Time `json:"start"` + EndTime time.Time `json:"end"` + TargetDomain string `json:"target_domain"` + DnsServer string `json:"dns_server"` + DnsMethod string `json:"Method"` + Duration string `json:"duration"` + Requests int64 `json:"requests"` + Success int64 `json:"success"` + Failed int64 `json:"failed"` + TPS float64 `json:"TPS"` + Latencies latencyDistribution `json:"latencies"` + Errors map[string]int `json:"errors"` + ReplyCode map[string]int `json:"reply_code"` +} + +type latencyDistribution struct { + // Mean is the mean request latency. + Mean float32 `json:"Mean_inMs"` + // P50 is the 50th percentile request latency. + P50 float32 `json:"P50_inMs"` + // P90 is the 90th percentile request latency. + P90 float32 `json:"P90_inMs"` + // P95 is the 95th percentile request latency. + P95 float32 `json:"P95_inMs"` + // P99 is the 99th percentile request latency. + P99 float32 `json:"P99_inMs"` + // Max is the maximum observed request latency. + Max float32 `json:"Max_inMs"` + // Min is the minimum observed request latency. + Min float32 `json:"Min_inMs"` +} + +type Work struct { + ServerAddr string + + Msg *dns.Msg + + Protocol string + + // C is the concurrency level, the number of concurrent workers to run. + Concurrency int + + // Timeout in seconds. + Timeout int + + // Qps is the rate limit in queries per second. + QPS int + + initOnce sync.Once + results chan *result + stopCh chan struct{} + start time.Duration + startTime time.Time + report *report +} + +// Init initializes internal data-structures +func (b *Work) Init() { + b.initOnce.Do(func() { + b.results = make(chan *result, maxResult) + b.stopCh = make(chan struct{}, b.Concurrency) + }) +} + +// Run makes all the requests, prints the summary. It blocks until +// all work is done. +func (b *Work) Run() { + b.Init() + b.startTime = time.Now() + b.start = time.Since(b.startTime) + b.report = newReport(b.results) + // Run the reporter first, it polls the result channel until it is closed. + go func() { + runReporter(b.report) + }() + + b.runWorkers() + b.Finish() +} + +func (b *Work) Stop() { + // Send stop signal so that workers can stop gracefully. + for i := 0; i < b.Concurrency; i++ { + b.stopCh <- struct{}{} + } +} + +func (b *Work) Finish() { + close(b.results) + total := b.now() - b.start + // Wait until the reporter is done. + <-b.report.done + b.report.finalize(total) +} + +func (b *Work) makeRequest(client *dns.Client, conn *dns.Conn, wg *sync.WaitGroup) { + defer wg.Done() + msg, rtt, err := client.ExchangeWithConn(b.Msg, conn) + b.results <- &result{ + duration: rtt, + err: err, + msg: msg, + } +} + +func (b *Work) runWorker() { + var ticker *time.Ticker + if b.QPS > 0 { + ticker = time.NewTicker(time.Duration(1e6*b.Concurrency/(b.QPS)) * time.Microsecond) + } + client := new(dns.Client) + client.Net = b.Protocol + client.Timeout = time.Duration(b.Timeout) * time.Millisecond + conn, _ := client.Dial(b.ServerAddr) + client.SingleInflight = true + wg := &sync.WaitGroup{} + for { + // Check if application is stopped. Do not send into a closed channel. + select { + case <-b.stopCh: + wg.Wait() + return + default: + if b.QPS > 0 { + <-ticker.C + } + wg.Add(1) + + // check connect close + // if close new connect + if conn == nil { + conn, _ = client.Dial(b.ServerAddr) + } + go b.makeRequest(client, conn, wg) + } + } +} + +func (b *Work) runWorkers() { + var wg sync.WaitGroup + wg.Add(b.Concurrency) + for i := 0; i < b.Concurrency; i++ { + go func() { + b.runWorker() + wg.Done() + }() + } + wg.Wait() + +} + +func (b *Work) now() time.Duration { return time.Since(b.startTime) } + +func (b *Work) AggregateMetric() *Metrics { + metric := &Metrics{} + metric.Requests = b.report.totalCount + metric.StartTime = b.startTime + metric.EndTime = b.startTime.Add(b.report.total) + metric.Duration = b.report.total.String() + metric.Failed = b.report.failedCount + metric.Success = b.report.successCount + metric.TPS = b.report.tps + latency := latencyDistribution{} + + t, _ := stats.Mean(b.report.lats) + latency.Mean = t + + t, _ = stats.Max(b.report.lats) + latency.Max = t + + t, _ = stats.Min(b.report.lats) + + latency.Min = t + + t, _ = stats.Percentile(b.report.lats, 50) + latency.P50 = t + + t, _ = stats.Percentile(b.report.lats, 90) + latency.P90 = t + + t, _ = stats.Percentile(b.report.lats, 95) + latency.P95 = t + + t, _ = stats.Percentile(b.report.lats, 99) + latency.P99 = t + + metric.Latencies = latency + + metric.Errors = b.report.errorDist + + metric.ReplyCode = b.report.ReplyCode + + metric.TargetDomain = b.Msg.Question[0].Name + + metric.DnsMethod = b.Protocol + + metric.DnsServer = b.ServerAddr + + return metric +} diff --git a/pkg/loadRequest/loadDns/dns_suite_test.go b/pkg/loadRequest/loadDns/dns_suite_test.go index c2f03108..12ac140d 100644 --- a/pkg/loadRequest/loadDns/dns_suite_test.go +++ b/pkg/loadRequest/loadDns/dns_suite_test.go @@ -7,6 +7,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + config "github.com/spidernet-io/spiderdoctor/pkg/types" ) func TestIppoolCR(t *testing.T) { @@ -16,4 +18,5 @@ func TestIppoolCR(t *testing.T) { var _ = BeforeSuite(func() { // nothing to do + config.AgentConfig.Configmap.NetdnsDefaultConcurrency = 50 }) diff --git a/pkg/loadRequest/loadDns/dns_test.go b/pkg/loadRequest/loadDns/dns_test.go index 22e4c044..311e7c4d 100644 --- a/pkg/loadRequest/loadDns/dns_test.go +++ b/pkg/loadRequest/loadDns/dns_test.go @@ -32,7 +32,7 @@ var _ = Describe("test dns ", Label("dns"), func() { log := logger.NewStdoutLogger("debug", "test") result, e := loadDns.DnsRequest(log, req) Expect(e).NotTo(HaveOccurred(), "failed to execute , error=%v", e) - Expect(result.FailedCount).To(Equal(0)) + Expect(int(result.Failed)).To(Equal(0)) Expect(len(result.ReplyCode)).To(Equal(1)) Expect(result.ReplyCode).Should(HaveKey(dns.RcodeToString[dns.RcodeSuccess])) @@ -62,7 +62,7 @@ var _ = Describe("test dns ", Label("dns"), func() { log := logger.NewStdoutLogger("debug", "test") result, e := loadDns.DnsRequest(log, req) Expect(e).NotTo(HaveOccurred(), "failed to execute , error=%v", e) - Expect(result.FailedCount).To(Equal(0)) + Expect(int(result.Failed)).To(Equal(0)) Expect(len(result.ReplyCode)).To(Equal(1)) Expect(result.ReplyCode).Should(HaveKey(dns.RcodeToString[dns.RcodeSuccess])) @@ -91,7 +91,7 @@ var _ = Describe("test dns ", Label("dns"), func() { log := logger.NewStdoutLogger("debug", "test") result, e := loadDns.DnsRequest(log, req) Expect(e).NotTo(HaveOccurred(), "failed to execute , error=%v", e) - Expect(result.SucceedCount).To(Equal(0)) + Expect(result.ReplyCode["NXDOMAIN"]).To(Equal(int(result.Requests))) Expect(len(result.ReplyCode)).To(Equal(1)) Expect(result.ReplyCode).Should(HaveKey(dns.RcodeToString[dns.RcodeNameError])) @@ -120,7 +120,7 @@ var _ = Describe("test dns ", Label("dns"), func() { log := logger.NewStdoutLogger("debug", "test") result, e := loadDns.DnsRequest(log, req) Expect(e).NotTo(HaveOccurred(), "failed to execute , error=%v", e) - Expect(result.FailedCount).To(Equal(0)) + Expect(int(result.Failed)).To(Equal(0)) Expect(len(result.ReplyCode)).To(Equal(1)) Expect(result.ReplyCode).Should(HaveKey(dns.RcodeToString[dns.RcodeSuccess])) diff --git a/pkg/pluginManager/netdns/agentExecuteTask.go b/pkg/pluginManager/netdns/agentExecuteTask.go index 00f7b5ec..454ead7b 100644 --- a/pkg/pluginManager/netdns/agentExecuteTask.go +++ b/pkg/pluginManager/netdns/agentExecuteTask.go @@ -18,17 +18,15 @@ import ( "strconv" "strings" "sync" - "time" ) -func ParseSuccessCondition(successCondition *crd.NetSuccessCondition, metricResult *loadDns.DnsMetrics) (failureReason string, err error) { - mean, _ := time.ParseDuration(metricResult.DelayForSuccess.Mean) +func ParseSuccessCondition(successCondition *crd.NetSuccessCondition, metricResult *loadDns.Metrics) (failureReason string, err error) { switch { - case successCondition.SuccessRate != nil && metricResult.SuccessRate < *(successCondition.SuccessRate): - failureReason = fmt.Sprintf("Success Rate %v is lower than request %v", metricResult.SuccessRate, *(successCondition.SuccessRate)) - case successCondition.MeanAccessDelayInMs != nil && mean.Milliseconds() > *(successCondition.MeanAccessDelayInMs): - failureReason = fmt.Sprintf("mean delay %v ms is bigger than request %v ms", mean.Milliseconds(), *(successCondition.MeanAccessDelayInMs)) + case successCondition.SuccessRate != nil && float64(metricResult.Success/metricResult.Requests) < *(successCondition.SuccessRate): + failureReason = fmt.Sprintf("Success Rate %v is lower than request %v", metricResult.Success/metricResult.Requests, *(successCondition.SuccessRate)) + case successCondition.MeanAccessDelayInMs != nil && int64(metricResult.Latencies.Mean) > *(successCondition.MeanAccessDelayInMs): + failureReason = fmt.Sprintf("mean delay %v ms is bigger than request %v ms", metricResult.Latencies.Mean, *(successCondition.MeanAccessDelayInMs)) default: failureReason = "" err = nil @@ -51,8 +49,8 @@ func SendRequestAndReport(logger *zap.Logger, targetName string, req *loadDns.Dn report["FailureReason"] = failureReason return } - report["MeanDelay"] = result.DelayForSuccess.Mean - report["SucceedRate"] = fmt.Sprintf("%v", result.SuccessRate) + report["MeanDelay"] = result.Latencies.Mean + report["SucceedRate"] = fmt.Sprintf("%v", result.Success/result.Requests) failureReason, err = ParseSuccessCondition(successCondition, result) if err != nil { diff --git a/pkg/types/types.go b/pkg/types/types.go index d7a2843f..ded82cf0 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -12,6 +12,7 @@ type ConfigmapConfig struct { NethttpDefaultMaxIdleConnsPerHost int `yaml:"nethttp_defaultMaxIdleConnsPerHost"` NethttpDefaultRequestDurationInSecond int `yaml:"nethttp_defaultRequest_DurationInSecond"` NethttpDefaultRequestPerRequestTimeoutInMS int `yaml:"nethttp_defaultRequest_PerRequestTimeoutInMS"` + NetdnsDefaultConcurrency int `yaml:"netdns_defaultConcurrency"` MultusPodAnnotationKey string `yaml:"multusPodAnnotationKey"` CrdMaxHistory int `yaml:"crdMaxHistory"` diff --git a/vendor/github.com/andres-erbsen/clock/.travis.yml b/vendor/github.com/andres-erbsen/clock/.travis.yml deleted file mode 100644 index ca785e51..00000000 --- a/vendor/github.com/andres-erbsen/clock/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -go: - - 1.3 - - 1.4 - - release - - tip -sudo: false diff --git a/vendor/github.com/andres-erbsen/clock/LICENSE b/vendor/github.com/andres-erbsen/clock/LICENSE deleted file mode 100644 index ddf4e001..00000000 --- a/vendor/github.com/andres-erbsen/clock/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Ben Johnson, Copyright (c) 2015 Yahoo Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/andres-erbsen/clock/README.md b/vendor/github.com/andres-erbsen/clock/README.md deleted file mode 100644 index f744e766..00000000 --- a/vendor/github.com/andres-erbsen/clock/README.md +++ /dev/null @@ -1,104 +0,0 @@ -clock [![Build Status](https://travis-ci.org/andres-erbsen/clock.svg)](https://travis-ci.org/andres-erbsen/clock) [![Coverage Status](https://coveralls.io/repos/andres-erbsen/clock/badge.png?branch=master)](https://coveralls.io/r/andres-erbsen/clock?branch=master) [![GoDoc](https://godoc.org/github.com/andres-erbsen/clock?status.png)](https://godoc.org/github.com/andres-erbsen/clock) ![Project status](http://img.shields.io/status/experimental.png?color=red) -===== - -Clock is a small library for mocking time in Go. It provides an interface -around the standard library's [`time`][time] package so that the application -can use the realtime clock while tests can use the mock clock. - -[time]: http://golang.org/pkg/time/ - - -## Usage - -### Realtime Clock - -Your application can maintain a `Clock` variable that will allow realtime and -mock clocks to be interchangable. For example, if you had an `Application` type: - -```go -import "github.com/andres-erbsen/clock" - -type Application struct { - Clock clock.Clock -} -``` - -You could initialize it to use the realtime clock like this: - -```go -var app Application -app.Clock = clock.New() -... -``` - -Then all timers and time-related functionality should be performed from the -`Clock` variable. - - -### Mocking time - -In your tests, you will want to use a `Mock` clock: - -```go -import ( - "testing" - - "github.com/andres-erbsen/clock" -) - -func TestApplication_DoSomething(t *testing.T) { - mock := clock.NewMock() - app := Application{Clock: mock} - ... -} -``` - -Now that you've initialized your application to use the mock clock, you can -adjust the time programmatically. The mock clock always starts from the Unix -epoch (midnight, Jan 1, 1970 UTC). - - -### Controlling time - -The mock clock provides the same functions that the standard library's `time` -package provides. For example, to find the current time, you use the `Now()` -function: - -```go -mock := clock.NewMock() - -// Find the current time. -mock.Now().UTC() // 1970-01-01 00:00:00 +0000 UTC - -// Move the clock forward. -mock.Add(2 * time.Hour) - -// Check the time again. It's 2 hours later! -mock.Now().UTC() // 1970-01-01 02:00:00 +0000 UTC -``` - -Timers and Tickers are also controlled by this same mock clock. They will only -execute when the clock is moved forward: - -``` -mock := clock.NewMock() -count := 0 - -// Kick off a timer to increment every 1 mock second. -go func() { - ticker := clock.Ticker(1 * time.Second) - for { - <-ticker.C - count++ - } -}() -runtime.Gosched() - -// Move the clock forward 10 second. -mock.Add(10 * time.Second) - -// This prints 10. -fmt.Println(count) -``` - - diff --git a/vendor/github.com/andres-erbsen/clock/clock.go b/vendor/github.com/andres-erbsen/clock/clock.go deleted file mode 100644 index b58b7032..00000000 --- a/vendor/github.com/andres-erbsen/clock/clock.go +++ /dev/null @@ -1,317 +0,0 @@ -package clock - -import ( - "sort" - "sync" - "time" -) - -// Clock represents an interface to the functions in the standard library time -// package. Two implementations are available in the clock package. The first -// is a real-time clock which simply wraps the time package's functions. The -// second is a mock clock which will only make forward progress when -// programmatically adjusted. -type Clock interface { - After(d time.Duration) <-chan time.Time - AfterFunc(d time.Duration, f func()) *Timer - Now() time.Time - Sleep(d time.Duration) - Tick(d time.Duration) <-chan time.Time - Ticker(d time.Duration) *Ticker - Timer(d time.Duration) *Timer -} - -// New returns an instance of a real-time clock. -func New() Clock { - return &clock{} -} - -// clock implements a real-time clock by simply wrapping the time package functions. -type clock struct{} - -func (c *clock) After(d time.Duration) <-chan time.Time { return time.After(d) } - -func (c *clock) AfterFunc(d time.Duration, f func()) *Timer { - return &Timer{timer: time.AfterFunc(d, f)} -} - -func (c *clock) Now() time.Time { return time.Now() } - -func (c *clock) Sleep(d time.Duration) { time.Sleep(d) } - -func (c *clock) Tick(d time.Duration) <-chan time.Time { return time.Tick(d) } - -func (c *clock) Ticker(d time.Duration) *Ticker { - t := time.NewTicker(d) - return &Ticker{C: t.C, ticker: t} -} - -func (c *clock) Timer(d time.Duration) *Timer { - t := time.NewTimer(d) - return &Timer{C: t.C, timer: t} -} - -// Mock represents a mock clock that only moves forward programmically. -// It can be preferable to a real-time clock when testing time-based functionality. -type Mock struct { - mu sync.Mutex - now time.Time // current time - timers clockTimers // tickers & timers -} - -// NewMock returns an instance of a mock clock. -// The current time of the mock clock on initialization is the Unix epoch. -func NewMock() *Mock { - return &Mock{now: time.Unix(0, 0)} -} - -// Add moves the current time of the mock clock forward by the duration. -// This should only be called from a single goroutine at a time. -func (m *Mock) Add(d time.Duration) { - // Calculate the final current time. - t := m.now.Add(d) - - // Continue to execute timers until there are no more before the new time. - for { - if !m.runNextTimer(t) { - break - } - } - - // Ensure that we end with the new time. - m.mu.Lock() - m.now = t - m.mu.Unlock() - - // Give a small buffer to make sure the other goroutines get handled. - gosched() -} - -// Sets the current time of the mock clock to a specific one. -// This should only be called from a single goroutine at a time. -func (m *Mock) Set(t time.Time) { - // Continue to execute timers until there are no more before the new time. - for { - if !m.runNextTimer(t) { - break - } - } - - // Ensure that we end with the new time. - m.mu.Lock() - m.now = t - m.mu.Unlock() - - // Give a small buffer to make sure the other goroutines get handled. - gosched() -} - -// runNextTimer executes the next timer in chronological order and moves the -// current time to the timer's next tick time. The next time is not executed if -// it's next time if after the max time. Returns true if a timer is executed. -func (m *Mock) runNextTimer(max time.Time) bool { - m.mu.Lock() - - // Sort timers by time. - sort.Sort(m.timers) - - // If we have no more timers then exit. - if len(m.timers) == 0 { - m.mu.Unlock() - return false - } - - // Retrieve next timer. Exit if next tick is after new time. - t := m.timers[0] - if t.Next().After(max) { - m.mu.Unlock() - return false - } - - // Move "now" forward and unlock clock. - m.now = t.Next() - m.mu.Unlock() - - // Execute timer. - t.Tick(m.now) - return true -} - -// After waits for the duration to elapse and then sends the current time on the returned channel. -func (m *Mock) After(d time.Duration) <-chan time.Time { - return m.Timer(d).C -} - -// AfterFunc waits for the duration to elapse and then executes a function. -// A Timer is returned that can be stopped. -func (m *Mock) AfterFunc(d time.Duration, f func()) *Timer { - t := m.Timer(d) - t.C = nil - t.fn = f - return t -} - -// Now returns the current wall time on the mock clock. -func (m *Mock) Now() time.Time { - m.mu.Lock() - defer m.mu.Unlock() - return m.now -} - -// Sleep pauses the goroutine for the given duration on the mock clock. -// The clock must be moved forward in a separate goroutine. -func (m *Mock) Sleep(d time.Duration) { - <-m.After(d) -} - -// Tick is a convenience function for Ticker(). -// It will return a ticker channel that cannot be stopped. -func (m *Mock) Tick(d time.Duration) <-chan time.Time { - return m.Ticker(d).C -} - -// Ticker creates a new instance of Ticker. -func (m *Mock) Ticker(d time.Duration) *Ticker { - m.mu.Lock() - defer m.mu.Unlock() - ch := make(chan time.Time, 1) - t := &Ticker{ - C: ch, - c: ch, - mock: m, - d: d, - next: m.now.Add(d), - } - m.timers = append(m.timers, (*internalTicker)(t)) - return t -} - -// Timer creates a new instance of Timer. -func (m *Mock) Timer(d time.Duration) *Timer { - ch := make(chan time.Time, 1) - t := &Timer{ - C: ch, - c: ch, - mock: m, - next: m.Now().Add(d), - } - m.addTimer((*internalTimer)(t)) - return t -} - -func (m *Mock) addTimer(t *internalTimer) { - m.mu.Lock() - defer m.mu.Unlock() - m.timers = append(m.timers, t) -} - -func (m *Mock) removeClockTimer(t clockTimer) bool { - m.mu.Lock() - defer m.mu.Unlock() - ret := false - for i, timer := range m.timers { - if timer == t { - ret = true - copy(m.timers[i:], m.timers[i+1:]) - m.timers[len(m.timers)-1] = nil - m.timers = m.timers[:len(m.timers)-1] - break - } - } - sort.Sort(m.timers) - return ret -} - -// clockTimer represents an object with an associated start time. -type clockTimer interface { - Next() time.Time - Tick(time.Time) -} - -// clockTimers represents a list of sortable timers. -type clockTimers []clockTimer - -func (a clockTimers) Len() int { return len(a) } -func (a clockTimers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a clockTimers) Less(i, j int) bool { return a[i].Next().Before(a[j].Next()) } - -// Timer represents a single event. -// The current time will be sent on C, unless the timer was created by AfterFunc. -type Timer struct { - C <-chan time.Time - c chan time.Time - timer *time.Timer // realtime impl, if set - next time.Time // next tick time - mock *Mock // mock clock, if set - fn func() // AfterFunc function, if set -} - -// Stop turns off the timer. -func (t *Timer) Stop() bool { - if t.timer != nil { - return t.timer.Stop() - } - return t.mock.removeClockTimer((*internalTimer)(t)) -} - -// Reset changes the timer to expire after duration d. It returns true if the -// timer had been active, false if the timer had expired or been stopped. -func (t *Timer) Reset(d time.Duration) bool { - if t.timer != nil { - return t.timer.Reset(d) - } - ret := t.mock.removeClockTimer((*internalTimer)(t)) - t.next = t.mock.Now().Add(d) - t.mock.addTimer((*internalTimer)(t)) - return ret -} - -type internalTimer Timer - -func (t *internalTimer) Next() time.Time { return t.next } -func (t *internalTimer) Tick(now time.Time) { - if t.fn != nil { - t.fn() - } else { - select { - case t.c <- now: - default: - } - } - t.mock.removeClockTimer((*internalTimer)(t)) - gosched() -} - -// Ticker holds a channel that receives "ticks" at regular intervals. -type Ticker struct { - C <-chan time.Time - c chan time.Time - ticker *time.Ticker // realtime impl, if set - next time.Time // next tick time - mock *Mock // mock clock, if set - d time.Duration // time between ticks -} - -// Stop turns off the ticker. -func (t *Ticker) Stop() { - if t.ticker != nil { - t.ticker.Stop() - } else { - t.mock.removeClockTimer((*internalTicker)(t)) - } -} - -type internalTicker Ticker - -func (t *internalTicker) Next() time.Time { return t.next } -func (t *internalTicker) Tick(now time.Time) { - select { - case t.c <- now: - default: - } - t.next = now.Add(t.d) - gosched() -} - -// Sleep momentarily so that other goroutines can process. -func gosched() { time.Sleep(1 * time.Millisecond) } diff --git a/vendor/go.uber.org/ratelimit/.gitignore b/vendor/go.uber.org/ratelimit/.gitignore deleted file mode 100644 index aa346ac4..00000000 --- a/vendor/go.uber.org/ratelimit/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/bin -/vendor -cover.html -cover.out - -*.swp diff --git a/vendor/go.uber.org/ratelimit/CHANGELOG.md b/vendor/go.uber.org/ratelimit/CHANGELOG.md deleted file mode 100644 index b2b89cf7..00000000 --- a/vendor/go.uber.org/ratelimit/CHANGELOG.md +++ /dev/null @@ -1,23 +0,0 @@ -# Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) -and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). - -## v0.2.0 - 2021-03-02 -### Added -- Allow configuring the limiter with custom slack. - [#64](https://github.com/uber-go/ratelimit/pull/64) -- Allow configuring the limiter per arbitrary time duration. - [#54](https://github.com/uber-go/ratelimit/pull/54) -### Changed -- Switched from Glide to Go Modules. -### Fixed -- Fix not working slack. - [#60](https://github.com/uber-go/ratelimit/pull/60) - -## v0.1.0 -### Fixed -- Changed the import path for `go.uber.org/atomic` to its newer, canonical - import path. - [#18](https://github.com/uber-go/ratelimit/issues/18) diff --git a/vendor/go.uber.org/ratelimit/LICENSE b/vendor/go.uber.org/ratelimit/LICENSE deleted file mode 100644 index 0f3edc86..00000000 --- a/vendor/go.uber.org/ratelimit/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Uber Technologies, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. \ No newline at end of file diff --git a/vendor/go.uber.org/ratelimit/Makefile b/vendor/go.uber.org/ratelimit/Makefile deleted file mode 100644 index 5bab5d77..00000000 --- a/vendor/go.uber.org/ratelimit/Makefile +++ /dev/null @@ -1,46 +0,0 @@ -# Directory to put `go install`ed binaries in. -export GOBIN ?= $(shell pwd)/bin - -GO_FILES := $(shell \ - find . '(' -path '*/.*' -o -path './vendor' ')' -prune \ - -o -name '*.go' -print | cut -b3-) - -.PHONY: bench -bench: - go test -bench=. ./... - -bin/golint: tools/go.mod - @cd tools && go install golang.org/x/lint/golint - -bin/staticcheck: tools/go.mod - @cd tools && go install honnef.co/go/tools/cmd/staticcheck - -.PHONY: build -build: - go build ./... - -.PHONY: cover -cover: - go test -coverprofile=cover.out -coverpkg=./... -v ./... - go tool cover -html=cover.out -o cover.html - -.PHONY: gofmt -gofmt: - $(eval FMT_LOG := $(shell mktemp -t gofmt.XXXXX)) - @gofmt -e -s -l $(GO_FILES) > $(FMT_LOG) || true - @[ ! -s "$(FMT_LOG)" ] || (echo "gofmt failed:" | cat - $(FMT_LOG) && false) - -.PHONY: golint -golint: bin/golint - @$(GOBIN)/golint -set_exit_status ./... - -.PHONY: lint -lint: gofmt golint staticcheck - -.PHONY: staticcheck -staticcheck: bin/staticcheck - @$(GOBIN)/staticcheck ./... - -.PHONY: test -test: - go test -race ./... diff --git a/vendor/go.uber.org/ratelimit/README.md b/vendor/go.uber.org/ratelimit/README.md deleted file mode 100644 index a05a2a88..00000000 --- a/vendor/go.uber.org/ratelimit/README.md +++ /dev/null @@ -1,46 +0,0 @@ -# Go rate limiter [![GoDoc][doc-img]][doc] [![Coverage Status][cov-img]][cov] ![test][test-img] - -This package provides a Golang implementation of the leaky-bucket rate limit algorithm. -This implementation refills the bucket based on the time elapsed between -requests instead of requiring an interval clock to fill the bucket discretely. - -Create a rate limiter with a maximum number of operations to perform per second. -Call Take() before each operation. Take will sleep until you can continue. - -```go -import ( - "fmt" - "time" - - "go.uber.org/ratelimit" -) - -func main() { - rl := ratelimit.New(100) // per second - - prev := time.Now() - for i := 0; i < 10; i++ { - now := rl.Take() - fmt.Println(i, now.Sub(prev)) - prev = now - } - - // Output: - // 0 0 - // 1 10ms - // 2 10ms - // 3 10ms - // 4 10ms - // 5 10ms - // 6 10ms - // 7 10ms - // 8 10ms - // 9 10ms -} -``` - -[cov-img]: https://codecov.io/gh/uber-go/ratelimit/branch/master/graph/badge.svg?token=zhLeUjjrm2 -[cov]: https://codecov.io/gh/uber-go/ratelimit -[doc-img]: https://pkg.go.dev/badge/go.uber.org/ratelimit -[doc]: https://pkg.go.dev/go.uber.org/ratelimit -[test-img]: https://github.com/uber-go/ratelimit/workflows/test/badge.svg diff --git a/vendor/go.uber.org/ratelimit/limiter_atomic.go b/vendor/go.uber.org/ratelimit/limiter_atomic.go deleted file mode 100644 index 745aa4cb..00000000 --- a/vendor/go.uber.org/ratelimit/limiter_atomic.go +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright (c) 2016,2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package ratelimit // import "go.uber.org/ratelimit" - -import ( - "time" - - "sync/atomic" - "unsafe" -) - -type state struct { - last time.Time - sleepFor time.Duration -} - -type atomicLimiter struct { - state unsafe.Pointer - //lint:ignore U1000 Padding is unused but it is crucial to maintain performance - // of this rate limiter in case of collocation with other frequently accessed memory. - padding [56]byte // cache line size - state pointer size = 64 - 8; created to avoid false sharing. - - perRequest time.Duration - maxSlack time.Duration - clock Clock -} - -// newAtomicBased returns a new atomic based limiter. -func newAtomicBased(rate int, opts ...Option) *atomicLimiter { - // TODO consider moving config building to the implementation - // independent code. - config := buildConfig(opts) - perRequest := config.per / time.Duration(rate) - l := &atomicLimiter{ - perRequest: perRequest, - maxSlack: -1 * time.Duration(config.slack) * perRequest, - clock: config.clock, - } - - initialState := state{ - last: time.Time{}, - sleepFor: 0, - } - atomic.StorePointer(&l.state, unsafe.Pointer(&initialState)) - return l -} - -// Take blocks to ensure that the time spent between multiple -// Take calls is on average time.Second/rate. -func (t *atomicLimiter) Take() time.Time { - var ( - newState state - taken bool - interval time.Duration - ) - for !taken { - now := t.clock.Now() - - previousStatePointer := atomic.LoadPointer(&t.state) - oldState := (*state)(previousStatePointer) - - newState = state{ - last: now, - sleepFor: oldState.sleepFor, - } - - // If this is our first request, then we allow it. - if oldState.last.IsZero() { - taken = atomic.CompareAndSwapPointer(&t.state, previousStatePointer, unsafe.Pointer(&newState)) - continue - } - - // sleepFor calculates how much time we should sleep based on - // the perRequest budget and how long the last request took. - // Since the request may take longer than the budget, this number - // can get negative, and is summed across requests. - newState.sleepFor += t.perRequest - now.Sub(oldState.last) - // We shouldn't allow sleepFor to get too negative, since it would mean that - // a service that slowed down a lot for a short period of time would get - // a much higher RPS following that. - if newState.sleepFor < t.maxSlack { - newState.sleepFor = t.maxSlack - } - if newState.sleepFor > 0 { - newState.last = newState.last.Add(newState.sleepFor) - interval, newState.sleepFor = newState.sleepFor, 0 - } - taken = atomic.CompareAndSwapPointer(&t.state, previousStatePointer, unsafe.Pointer(&newState)) - } - t.clock.Sleep(interval) - return newState.last -} diff --git a/vendor/go.uber.org/ratelimit/limiter_mutexbased.go b/vendor/go.uber.org/ratelimit/limiter_mutexbased.go deleted file mode 100644 index 1408f1c5..00000000 --- a/vendor/go.uber.org/ratelimit/limiter_mutexbased.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2016,2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package ratelimit // import "go.uber.org/ratelimit" - -import ( - "sync" - "time" -) - -type mutexLimiter struct { - sync.Mutex - last time.Time - sleepFor time.Duration - perRequest time.Duration - maxSlack time.Duration - clock Clock -} - -// newMutexBased returns a new atomic based limiter. -func newMutexBased(rate int, opts ...Option) *mutexLimiter { - // TODO consider moving config building to the implementation - // independent code. - config := buildConfig(opts) - perRequest := config.per / time.Duration(rate) - l := &mutexLimiter{ - perRequest: perRequest, - maxSlack: -1 * time.Duration(config.slack) * perRequest, - clock: config.clock, - } - return l -} - -// Take blocks to ensure that the time spent between multiple -// Take calls is on average time.Second/rate. -func (t *mutexLimiter) Take() time.Time { - t.Lock() - defer t.Unlock() - - now := t.clock.Now() - - // If this is our first request, then we allow it. - if t.last.IsZero() { - t.last = now - return t.last - } - - // sleepFor calculates how much time we should sleep based on - // the perRequest budget and how long the last request took. - // Since the request may take longer than the budget, this number - // can get negative, and is summed across requests. - t.sleepFor += t.perRequest - now.Sub(t.last) - - // We shouldn't allow sleepFor to get too negative, since it would mean that - // a service that slowed down a lot for a short period of time would get - // a much higher RPS following that. - if t.sleepFor < t.maxSlack { - t.sleepFor = t.maxSlack - } - - // If sleepFor is positive, then we should sleep now. - if t.sleepFor > 0 { - t.clock.Sleep(t.sleepFor) - t.last = now.Add(t.sleepFor) - t.sleepFor = 0 - } else { - t.last = now - } - - return t.last -} diff --git a/vendor/go.uber.org/ratelimit/ratelimit.go b/vendor/go.uber.org/ratelimit/ratelimit.go deleted file mode 100644 index b5b16e57..00000000 --- a/vendor/go.uber.org/ratelimit/ratelimit.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2016,2020 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package ratelimit // import "go.uber.org/ratelimit" - -import ( - "time" - - "github.com/andres-erbsen/clock" -) - -// Note: This file is inspired by: -// https://github.com/prashantv/go-bench/blob/master/ratelimit - -// Limiter is used to rate-limit some process, possibly across goroutines. -// The process is expected to call Take() before every iteration, which -// may block to throttle the goroutine. -type Limiter interface { - // Take should block to make sure that the RPS is met. - Take() time.Time -} - -// Clock is the minimum necessary interface to instantiate a rate limiter with -// a clock or mock clock, compatible with clocks created using -// github.com/andres-erbsen/clock. -type Clock interface { - Now() time.Time - Sleep(time.Duration) -} - -// config configures a limiter. -type config struct { - clock Clock - slack int - per time.Duration -} - -// New returns a Limiter that will limit to the given RPS. -func New(rate int, opts ...Option) Limiter { - return newAtomicBased(rate, opts...) -} - -// buildConfig combines defaults with options. -func buildConfig(opts []Option) config { - c := config{ - clock: clock.New(), - slack: 10, - per: time.Second, - } - - for _, opt := range opts { - opt.apply(&c) - } - return c -} - -// Option configures a Limiter. -type Option interface { - apply(*config) -} - -type clockOption struct { - clock Clock -} - -func (o clockOption) apply(c *config) { - c.clock = o.clock -} - -// WithClock returns an option for ratelimit.New that provides an alternate -// Clock implementation, typically a mock Clock for testing. -func WithClock(clock Clock) Option { - return clockOption{clock: clock} -} - -type slackOption int - -func (o slackOption) apply(c *config) { - c.slack = int(o) -} - -// WithoutSlack configures the limiter to be strict and not to accumulate -// previously "unspent" requests for future bursts of traffic. -var WithoutSlack Option = slackOption(0) - -// WithSlack configures custom slack. -// Slack allows the limiter to accumulate "unspent" requests -// for future bursts of traffic. -func WithSlack(slack int) Option { - return slackOption(slack) -} - -type perOption time.Duration - -func (p perOption) apply(c *config) { - c.per = time.Duration(p) -} - -// Per allows configuring limits for different time windows. -// -// The default window is one second, so New(100) produces a one hundred per -// second (100 Hz) rate limiter. -// -// New(2, Per(60*time.Second)) creates a 2 per minute rate limiter. -func Per(per time.Duration) Option { - return perOption(per) -} - -type unlimited struct{} - -// NewUnlimited returns a RateLimiter that is not limited. -func NewUnlimited() Limiter { - return unlimited{} -} - -func (unlimited) Take() time.Time { - return time.Now() -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 40e62ea1..07449b27 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,9 +7,6 @@ github.com/Masterminds/semver/v3 # github.com/Masterminds/sprig/v3 v3.2.2 ## explicit; go 1.13 github.com/Masterminds/sprig/v3 -# github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 -## explicit -github.com/andres-erbsen/clock # github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d ## explicit; go 1.13 github.com/asaskevich/govalidator @@ -431,9 +428,6 @@ go.uber.org/atomic # go.uber.org/multierr v1.8.0 ## explicit; go 1.14 go.uber.org/multierr -# go.uber.org/ratelimit v0.2.0 -## explicit; go 1.14 -go.uber.org/ratelimit # go.uber.org/zap v1.24.0 ## explicit; go 1.19 go.uber.org/zap