From 5cf9ccf6719fbe06ce2cccfa7a65cdb55f96eb53 Mon Sep 17 00:00:00 2001 From: Nish Krishnan Date: Thu, 13 Jan 2022 14:44:28 -0800 Subject: [PATCH] Replace gostats with tally (#162) --- go.mod | 9 +- go.sum | 18 +- .../controllers/events/events_controller.go | 14 +- .../events/events_controller_e2e_test.go | 4 +- .../events/events_controller_test.go | 44 +++-- server/controllers/jobs_controller.go | 18 +- server/events/apply_command_runner_test.go | 4 +- server/events/command_context.go | 4 +- server/events/command_runner.go | 18 +- server/events/command_runner_test.go | 4 +- .../instrumented_project_command_builder.go | 42 ++--- .../instrumented_project_command_runner.go | 16 +- .../instrumented_pull_closed_executor.go | 20 +-- server/events/models/models.go | 6 +- server/events/project_command_builder.go | 6 +- server/events/project_command_builder_test.go | 22 +-- .../events/project_command_context_builder.go | 12 +- server/events/vcs/instrumented_client.go | 165 +++++++++--------- server/events/yaml/valid/global_cfg.go | 10 ++ ...rumented_project_command_output_handler.go | 55 ------ server/lyft/aws/sns/writer.go | 16 +- .../audit_project_commands_wrapper_test.go | 10 +- server/lyft/scheduled/executor_service.go | 77 ++++---- server/lyft/scheduled/runtime_stats.go | 128 ++++++++++++++ server/metrics/debug.go | 93 ++++++++++ server/metrics/scope.go | 63 +++++++ server/server.go | 80 +++++---- 27 files changed, 620 insertions(+), 338 deletions(-) delete mode 100644 server/handlers/instrumented_project_command_output_handler.go create mode 100644 server/lyft/scheduled/runtime_stats.go create mode 100644 server/metrics/debug.go create mode 100644 server/metrics/scope.go diff --git a/go.mod b/go.mod index c54fa34e3..a97308226 100644 --- a/go.mod +++ b/go.mod @@ -54,11 +54,9 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jstemmer/go-junit-report v0.9.1 // indirect - github.com/kelseyhightower/envconfig v1.4.1-0.20200624135755-c974cae29cf5 // indirect github.com/klauspost/compress v1.11.2 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 // indirect - github.com/lyft/gostats v0.4.5 github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-colorable v0.0.9 // indirect github.com/mattn/go-isatty v0.0.4 // indirect @@ -101,7 +99,7 @@ require ( github.com/zclconf/go-cty v1.5.1 // indirect go.etcd.io/bbolt v1.3.6 go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.7.0 // indirect + go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.0 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 @@ -130,5 +128,10 @@ require ( require ( github.com/antlr/antlr4 v0.0.0-20201206235148-c87e55b61113 // indirect github.com/blang/semver v3.5.1+incompatible // indirect + github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2 github.com/nikunjy/rules v0.0.0-20200120082459-0b7c4dc9dc86 // indirect + github.com/twmb/murmur3 v1.1.6 // indirect + github.com/uber-go/tally v3.4.3+incompatible ) + +require github.com/onsi/ginkgo v1.14.0 // indirect diff --git a/go.sum b/go.sum index 56408c1fd..3529f4040 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,8 @@ github.com/bradleyfalzon/ghinstallation v1.1.1 h1:pmBXkxgM1WeF8QYvDLT5kuQiHMcmf+ github.com/bradleyfalzon/ghinstallation v1.1.1/go.mod h1:vyCmHTciHx/uuyN82Zc3rXN3X2KTK8nUTCrTMwAhcug= github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5 h1:osZyZB7J4kE1tKLeaUjV6+uZVBfS835T0I/RxmwWw1w= github.com/briandowns/spinner v0.0.0-20170614154858-48dbb65d7bd5/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= +github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2 h1:GgJnJEJYymy/lx+1zXOO2TvGPRQJJ9vz4onxnA9gF3k= +github.com/cactus/go-statsd-client/statsd v0.0.0-20200623234511-94959e3146b2/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -274,9 +276,6 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kelseyhightower/envconfig v1.4.1-0.20200624135755-c974cae29cf5 h1:TW7YQVw105jR4B+n/tAlkKNbYFSdWRK1RyLL5YYruso= -github.com/kelseyhightower/envconfig v1.4.1-0.20200624135755-c974cae29cf5/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.2 h1:MiK62aErc3gIiVEtyzKfeOHgW7atJb5g/KNX5m3c2nQ= @@ -294,8 +293,6 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 h1:MNApn+Z+fIT4NPZopPfCc1obT6aY3SVM6DOctz1A9ZU= github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018/go.mod h1:sFlOUpQL1YcjhFVXhg1CG8ZASEs/Mf1oVb6H75JL/zg= -github.com/lyft/gostats v0.4.5 h1:2KYdjsz+RfxeJAWqOpYUlyXgFgYd0SxPFUSFm9PdXM4= -github.com/lyft/gostats v0.4.5/go.mod h1:7ECvCenOq0N9BE6Tn+dtwFoS6xAwbvsx4tVSETO/hmc= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= @@ -380,7 +377,6 @@ github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXk github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.6.1-0.20200528085638-6699a89a232f h1:qqqIhBDFUBrbMezIyJkKWIpf+E5CdObleGMjW1s19Hg= github.com/sirupsen/logrus v1.6.1-0.20200528085638-6699a89a232f/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -413,6 +409,10 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thomaspoignant/go-feature-flag v0.18.4 h1:LJ5dns5HRMzXATggfXKHanrGriN0DfQ5cm2N7U9g/0A= github.com/thomaspoignant/go-feature-flag v0.18.4/go.mod h1:S/z9nx718SSumYgGL0LK71QRl3lQvFLAaQlPzexgMus= +github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= +github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= +github.com/uber-go/tally v3.4.3+incompatible h1:Oq25FXV8cWHPRo+EPeNdbN3LfuozC9mDK2/4vZ1k38U= +github.com/uber-go/tally v3.4.3+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU= github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= @@ -444,8 +444,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -460,7 +461,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200403201458-baeed622b8d8/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= @@ -775,14 +775,12 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= diff --git a/server/controllers/events/events_controller.go b/server/controllers/events/events_controller.go index f12c3b5d4..499bef187 100644 --- a/server/controllers/events/events_controller.go +++ b/server/controllers/events/events_controller.go @@ -20,7 +20,6 @@ import ( "strings" "github.com/google/go-github/v31/github" - stats "github.com/lyft/gostats" "github.com/mcdafydd/go-azuredevops/azuredevops" "github.com/microcosm-cc/bluemonday" "github.com/pkg/errors" @@ -30,6 +29,7 @@ import ( "github.com/runatlantis/atlantis/server/events/vcs/bitbucketcloud" "github.com/runatlantis/atlantis/server/events/vcs/bitbucketserver" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" gitlab "github.com/xanzy/go-gitlab" ) @@ -49,7 +49,7 @@ type VCSEventsController struct { CommandRunner events.CommandRunner PullCleaner events.PullCleaner Logger logging.SimpleLogging - Scope stats.Scope + Scope tally.Scope Parser events.EventParsing CommentParser events.CommentParsing ApplyDisabled bool @@ -158,7 +158,7 @@ func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Re githubReqID := "X-Github-Delivery=" + r.Header.Get("X-Github-Delivery") logger := e.Logger.With("gh-request-id", githubReqID) - scope := e.Scope.Scope("github.event") + scope := e.Scope.SubScope("github.event") logger.Debug("request valid") @@ -169,10 +169,10 @@ func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Re switch event := event.(type) { case *github.IssueCommentEvent: resp = e.HandleGithubCommentEvent(event, githubReqID, logger) - scope = scope.Scope(fmt.Sprintf("comment.%s", *event.Action)) + scope = scope.SubScope(fmt.Sprintf("comment.%s", *event.Action)) case *github.PullRequestEvent: resp = e.HandleGithubPullRequestEvent(logger, event, githubReqID) - scope = scope.Scope(fmt.Sprintf("pr.%s", *event.Action)) + scope = scope.SubScope(fmt.Sprintf("pr.%s", *event.Action)) default: resp = HttpResponse{ body: fmt.Sprintf("Ignoring unsupported event %s", githubReqID), @@ -181,13 +181,13 @@ func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Re if resp.err.code != 0 { logger.Err("error handling gh post code: %d err: %s", resp.err.code, resp.err.err.Error()) - scope.NewCounter(fmt.Sprintf("error_%d", resp.err.code)).Inc() + scope.Counter(fmt.Sprintf("error_%d", resp.err.code)).Inc(1) w.WriteHeader(resp.err.code) fmt.Fprintln(w, resp.err.err.Error()) return } - scope.NewCounter(fmt.Sprintf("success_%d", http.StatusOK)).Inc() + scope.Counter(fmt.Sprintf("success_%d", http.StatusOK)).Inc(1) w.WriteHeader(http.StatusOK) fmt.Fprintln(w, resp.body) } diff --git a/server/controllers/events/events_controller_e2e_test.go b/server/controllers/events/events_controller_e2e_test.go index c67387e03..2f7c6f80e 100644 --- a/server/controllers/events/events_controller_e2e_test.go +++ b/server/controllers/events/events_controller_e2e_test.go @@ -16,7 +16,6 @@ import ( "github.com/google/go-github/v31/github" "github.com/hashicorp/go-getter" "github.com/hashicorp/go-version" - stats "github.com/lyft/gostats" . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" @@ -40,6 +39,7 @@ import ( handlermocks "github.com/runatlantis/atlantis/server/handlers/mocks" "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/lyft/feature" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" ) @@ -895,7 +895,7 @@ func setupE2E(t *testing.T, repoDir string) (events_controllers.VCSEventsControl WorkingDir: workingDir, PreWorkflowHookRunner: mockPreWorkflowHookRunner, } - statsScope := stats.NewStore(stats.NewNullSink(), false) + statsScope, _, err := metrics.NewLoggingScope(logger, "atlantis") projectCommandBuilder := events.NewProjectCommandBuilder( userConfig.EnablePolicyChecksFlag, diff --git a/server/controllers/events/events_controller_test.go b/server/controllers/events/events_controller_test.go index f67640028..2a5516898 100644 --- a/server/controllers/events/events_controller_test.go +++ b/server/controllers/events/events_controller_test.go @@ -26,7 +26,6 @@ import ( "testing" "github.com/google/go-github/v31/github" - stats "github.com/lyft/gostats" . "github.com/petergtz/pegomock" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" "github.com/runatlantis/atlantis/server/controllers/events/mocks" @@ -36,6 +35,7 @@ import ( "github.com/runatlantis/atlantis/server/events/models" vcsmocks "github.com/runatlantis/atlantis/server/events/vcs/mocks" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" gitlab "github.com/xanzy/go-gitlab" ) @@ -199,9 +199,11 @@ func TestPost_GitlabCommentNotAllowlisted(t *testing.T) { t.Log("when the event is a gitlab comment from a repo that isn't allowlisted we comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, CommentParser: &events.CommentParser{}, GitlabRequestParserValidator: &events_controllers.DefaultGitlabRequestParserValidator{}, Parser: &events.EventParser{}, @@ -228,9 +230,11 @@ func TestPost_GitlabCommentNotAllowlistedWithSilenceErrors(t *testing.T) { t.Log("when the event is a gitlab comment from a repo that isn't allowlisted and we are silencing errors, do not comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, CommentParser: &events.CommentParser{}, GitlabRequestParserValidator: &events_controllers.DefaultGitlabRequestParserValidator{}, Parser: &events.EventParser{}, @@ -258,9 +262,11 @@ func TestPost_GithubCommentNotAllowlisted(t *testing.T) { t.Log("when the event is a github comment from a repo that isn't allowlisted we comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, GithubRequestValidator: &events_controllers.DefaultGithubRequestValidator{}, CommentParser: &events.CommentParser{}, Parser: &events.EventParser{}, @@ -288,9 +294,11 @@ func TestPost_GithubCommentNotAllowlistedWithSilenceErrors(t *testing.T) { t.Log("when the event is a github comment from a repo that isn't allowlisted and we are silencing errors, do not comment with an error") RegisterMockTestingT(t) vcsClient := vcsmocks.NewMockClient() + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, GithubRequestValidator: &events_controllers.DefaultGithubRequestValidator{}, CommentParser: &events.CommentParser{}, Parser: &events.EventParser{}, @@ -486,10 +494,12 @@ func TestPost_AzureDevopsPullRequestIgnoreEvent(t *testing.T) { vcsmock := vcsmocks.NewMockClient() repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ TestingMode: true, - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, ApplyDisabled: false, AzureDevopsWebhookBasicUser: user, AzureDevopsWebhookBasicPassword: secret, @@ -643,6 +653,8 @@ func TestPost_BBServerPullClosed(t *testing.T) { pullCleaner := emocks.NewMockPullCleaner() allowlist, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") ec := &events_controllers.VCSEventsController{ PullCleaner: pullCleaner, Parser: &events.EventParser{ @@ -653,8 +665,8 @@ func TestPost_BBServerPullClosed(t *testing.T) { RepoAllowlistChecker: allowlist, SupportedVCSHosts: []models.VCSHostType{models.BitbucketServer}, VCSClient: nil, - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, } // Build HTTP request. @@ -772,10 +784,12 @@ func setup(t *testing.T) (events_controllers.VCSEventsController, *mocks.MockGit vcsmock := vcsmocks.NewMockClient() repoAllowlistChecker, err := events.NewRepoAllowlistChecker("*") Ok(t, err) + logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "null") e := events_controllers.VCSEventsController{ TestingMode: true, - Logger: logging.NewNoopLogger(t), - Scope: stats.NewStore(stats.NewNullSink(), false).Scope("null"), + Logger: logger, + Scope: scope, GithubRequestValidator: v, Parser: p, CommentParser: cp, diff --git a/server/controllers/jobs_controller.go b/server/controllers/jobs_controller.go index 9134620a4..7f9c5fe46 100644 --- a/server/controllers/jobs_controller.go +++ b/server/controllers/jobs_controller.go @@ -8,7 +8,6 @@ import ( "strconv" "github.com/gorilla/mux" - stats "github.com/lyft/gostats" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/controllers/templates" "github.com/runatlantis/atlantis/server/controllers/websocket" @@ -16,6 +15,7 @@ import ( "github.com/runatlantis/atlantis/server/events/metrics" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" ) type JobsController struct { @@ -26,7 +26,7 @@ type JobsController struct { ProjectJobsErrorTemplate templates.TemplateWriter Db *db.BoltDB WsMux *websocket.Multiplexor - StatsScope stats.Scope + StatsScope tally.Scope } type ProjectInfoKeyGenerator struct{} @@ -134,10 +134,10 @@ func (j *JobsController) getProjectJobs(w http.ResponseWriter, r *http.Request) } func (j *JobsController) GetProjectJobs(w http.ResponseWriter, r *http.Request) { - errorCounter := j.StatsScope.Scope("getprojectjobs").NewCounter(metrics.ExecutionErrorMetric) + errorCounter := j.StatsScope.SubScope("getprojectjobs").Counter(metrics.ExecutionErrorMetric) err := j.getProjectJobs(w, r) if err != nil { - errorCounter.Inc() + errorCounter.Inc(1) } } @@ -153,14 +153,14 @@ func (j *JobsController) getProjectJobsWS(w http.ResponseWriter, r *http.Request } func (j *JobsController) GetProjectJobsWS(w http.ResponseWriter, r *http.Request) { - jobsMetric := j.StatsScope.Scope("getprojectjobs") - errorCounter := jobsMetric.NewCounter(metrics.ExecutionErrorMetric) - executionTime := jobsMetric.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + jobsMetric := j.StatsScope.SubScope("getprojectjobs") + errorCounter := jobsMetric.Counter(metrics.ExecutionErrorMetric) + executionTime := jobsMetric.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() err := j.getProjectJobsWS(w, r) if err != nil { - errorCounter.Inc() + errorCounter.Inc(1) } } diff --git a/server/events/apply_command_runner_test.go b/server/events/apply_command_runner_test.go index 95c1b5be4..465176d45 100644 --- a/server/events/apply_command_runner_test.go +++ b/server/events/apply_command_runner_test.go @@ -5,13 +5,13 @@ import ( "testing" "github.com/google/go-github/v31/github" - stats "github.com/lyft/gostats" . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/core/locking" "github.com/runatlantis/atlantis/server/events" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/events/models/fixtures" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" ) func TestApplyCommandRunner_IsLocked(t *testing.T) { @@ -48,7 +48,7 @@ func TestApplyCommandRunner_IsLocked(t *testing.T) { logger := logging.NewNoopLogger(t) vcsClient := setup(t) - scopeNull := stats.NewStore(stats.NewNullSink(), false) + scopeNull, _, _ := metrics.NewLoggingScope(logger, "atlantis") pull := &github.PullRequest{ State: github.String("open"), diff --git a/server/events/command_context.go b/server/events/command_context.go index f18e3db7b..723ade459 100644 --- a/server/events/command_context.go +++ b/server/events/command_context.go @@ -13,9 +13,9 @@ package events import ( - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" ) // CommandTrigger represents the how the command was triggered @@ -38,7 +38,7 @@ type CommandContext struct { // See https://help.github.com/articles/about-pull-request-merges/. HeadRepo models.Repo Pull models.PullRequest - Scope stats.Scope + Scope tally.Scope // User is the user that triggered this command. User models.User Log logging.SimpleLogging diff --git a/server/events/command_runner.go b/server/events/command_runner.go index ee50f5b13..aeb4c34ab 100644 --- a/server/events/command_runner.go +++ b/server/events/command_runner.go @@ -18,7 +18,6 @@ import ( "strconv" "github.com/google/go-github/v31/github" - stats "github.com/lyft/gostats" "github.com/mcdafydd/go-azuredevops/azuredevops" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/events/metrics" @@ -28,6 +27,7 @@ import ( "github.com/runatlantis/atlantis/server/logging" "github.com/runatlantis/atlantis/server/lyft/feature" "github.com/runatlantis/atlantis/server/recovery" + "github.com/uber-go/tally" gitlab "github.com/xanzy/go-gitlab" ) @@ -100,7 +100,7 @@ type DefaultCommandRunner struct { EventParser EventParsing Logger logging.SimpleLogging GlobalCfg valid.GlobalCfg - StatsScope stats.Scope + StatsScope tally.Scope // AllowForkPRs controls whether we operate on pull requests from forks. AllowForkPRs bool // ParallelPoolSize controls the size of the wait group used to run @@ -140,9 +140,9 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo log.Err("Unable to fetch pull status, this is likely a bug.", err) } - scope := c.StatsScope.Scope("autoplan") - timer := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer timer.Complete() + scope := c.StatsScope.SubScope("autoplan") + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() ctx := &CommandContext{ User: user, @@ -188,13 +188,13 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead log := c.buildLogger(baseRepo.FullName, pullNum) defer c.logPanics(baseRepo, pullNum, log) - scope := c.StatsScope.Scope("comment") + scope := c.StatsScope.SubScope("comment") if cmd != nil { - scope = scope.Scope(cmd.Name.String()) + scope = scope.SubScope(cmd.Name.String()) } - timer := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer timer.Complete() + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() headRepo, pull, err := c.ensureValidRepoMetadata(baseRepo, maybeHeadRepo, maybePull, user, pullNum, log) if err != nil { diff --git a/server/events/command_runner_test.go b/server/events/command_runner_test.go index 99065176d..dea14cd2c 100644 --- a/server/events/command_runner_test.go +++ b/server/events/command_runner_test.go @@ -20,12 +20,12 @@ import ( "strings" "testing" - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/core/db" "github.com/runatlantis/atlantis/server/events/vcs" lyft_vcs "github.com/runatlantis/atlantis/server/events/vcs/lyft" "github.com/runatlantis/atlantis/server/events/yaml/valid" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" "github.com/google/go-github/v31/github" . "github.com/petergtz/pegomock" @@ -193,7 +193,7 @@ func setup(t *testing.T) *vcsmocks.MockClient { When(preWorkflowHooksCommandRunner.RunPreHooks(matchers.AnyPtrToEventsCommandContext())).ThenReturn(nil) globalCfg := valid.NewGlobalCfgFromArgs(valid.GlobalCfgArgs{}) - scope := stats.NewDefaultStore() + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") ch = events.DefaultCommandRunner{ VCSClient: vcsClient, diff --git a/server/events/instrumented_project_command_builder.go b/server/events/instrumented_project_command_builder.go index 5b7d680d9..c6eea0390 100644 --- a/server/events/instrumented_project_command_builder.go +++ b/server/events/instrumented_project_command_builder.go @@ -12,63 +12,63 @@ type InstrumentedProjectCommandBuilder struct { } func (b *InstrumentedProjectCommandBuilder) BuildApplyCommands(ctx *CommandContext, comment *CommentCommand) ([]models.ProjectCommandContext, error) { - scope := ctx.Scope.Scope("builder") + scope := ctx.Scope.SubScope("builder") - timer := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer timer.Complete() + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) projectCmds, err := b.ProjectCommandBuilder.BuildApplyCommands(ctx, comment) if err != nil { - executionError.Inc() + executionError.Inc(1) b.Logger.Err("Error building apply commands: %s", err) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return projectCmds, err } func (b *InstrumentedProjectCommandBuilder) BuildAutoplanCommands(ctx *CommandContext) ([]models.ProjectCommandContext, error) { - scope := ctx.Scope.Scope("builder") + scope := ctx.Scope.SubScope("builder") - timer := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer timer.Complete() + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) projectCmds, err := b.ProjectCommandBuilder.BuildAutoplanCommands(ctx) if err != nil { - executionError.Inc() + executionError.Inc(1) b.Logger.Err("Error building auto plan commands: %s", err) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return projectCmds, err } func (b *InstrumentedProjectCommandBuilder) BuildPlanCommands(ctx *CommandContext, comment *CommentCommand) ([]models.ProjectCommandContext, error) { - scope := ctx.Scope.Scope("builder") + scope := ctx.Scope.SubScope("builder") - timer := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer timer.Complete() + timer := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer timer.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) projectCmds, err := b.ProjectCommandBuilder.BuildPlanCommands(ctx, comment) if err != nil { - executionError.Inc() + executionError.Inc(1) b.Logger.Err("Error building plan commands: %s", err) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return projectCmds, err diff --git a/server/events/instrumented_project_command_runner.go b/server/events/instrumented_project_command_runner.go index 72da95f8d..b060b12f1 100644 --- a/server/events/instrumented_project_command_runner.go +++ b/server/events/instrumented_project_command_runner.go @@ -30,30 +30,30 @@ func RunAndEmitStats(commandName string, ctx models.ProjectCommandContext, execu ctx.Log = ctx.Log.WithHistory("project", ctx.ProjectName) logger := ctx.Log - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) - executionFailure := scope.NewCounter(metrics.ExecutionFailureMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) + executionFailure := scope.Counter(metrics.ExecutionFailureMetric) result := execute(ctx) if result.Error != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Error running %s operation: %s", commandName, result.Error.Error()) return result } if result.Failure != "" { - executionFailure.Inc() + executionFailure.Inc(1) logger.Err("Failure running %s operation: %s", commandName, result.Failure) return result } logger.Info("%s success. output available at: %s", commandName, ctx.Pull.URL) - executionSuccess.Inc() + executionSuccess.Inc(1) return result } diff --git a/server/events/instrumented_pull_closed_executor.go b/server/events/instrumented_pull_closed_executor.go index 671d81b9b..af9bbf45c 100644 --- a/server/events/instrumented_pull_closed_executor.go +++ b/server/events/instrumented_pull_closed_executor.go @@ -3,24 +3,24 @@ package events import ( "strconv" - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/events/metrics" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" ) type InstrumentedPullClosedExecutor struct { - scope stats.Scope + scope tally.Scope log logging.SimpleLogging cleaner PullCleaner } func NewInstrumentedPullClosedExecutor( - scope stats.Scope, log logging.SimpleLogging, cleaner PullCleaner, + scope tally.Scope, log logging.SimpleLogging, cleaner PullCleaner, ) PullCleaner { return &InstrumentedPullClosedExecutor{ - scope: scope.Scope("pullclosed.cleanup"), + scope: scope.SubScope("pullclosed.cleanup"), log: log, cleaner: cleaner, } @@ -32,22 +32,22 @@ func (e *InstrumentedPullClosedExecutor) CleanUpPull(repo models.Repo, pull mode "pull-num", strconv.Itoa(pull.Num), ) - executionSuccess := e.scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := e.scope.NewCounter(metrics.ExecutionErrorMetric) - executionTime := e.scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionSuccess := e.scope.Counter(metrics.ExecutionSuccessMetric) + executionError := e.scope.Counter(metrics.ExecutionErrorMetric) + executionTime := e.scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() log.Info("Initiating cleanup of pull data.") err := e.cleaner.CleanUpPull(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) log.Err("error during cleanup of pull data", err) return err } - executionSuccess.Inc() + executionSuccess.Inc(1) return nil } diff --git a/server/events/models/models.go b/server/events/models/models.go index 3b502ebb0..15913e26a 100644 --- a/server/events/models/models.go +++ b/server/events/models/models.go @@ -26,8 +26,8 @@ import ( "time" "github.com/hashicorp/go-version" - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/events/yaml/valid" @@ -384,7 +384,7 @@ type ProjectCommandContext struct { // Log is a logger that's been set up for this context. Log logging.SimpleLogging // Scope is the scope for reporting stats setup for this context - Scope stats.Scope + Scope tally.Scope // PullReqStatus holds state about the PR that requires additional computation outside models.PullRequest PullReqStatus PullReqStatus // CurrentProjectPlanStatus is the status of the current project prior to this command. @@ -440,7 +440,7 @@ func (p ProjectCommandContext) ProjectCloneDir() string { // SetScope sets the scope of the stats object field. Note: we deliberately set this on the value // instead of a pointer since we want scopes to mirror our function stack func (p ProjectCommandContext) SetScope(scope string) { - p.Scope = p.Scope.Scope(scope) //nolint + p.Scope = p.Scope.SubScope(scope) //nolint } // GetShowResultFileName returns the filename (not the path) to store the tf show result diff --git a/server/events/project_command_builder.go b/server/events/project_command_builder.go index 06cf72ebb..6cfa861e5 100644 --- a/server/events/project_command_builder.go +++ b/server/events/project_command_builder.go @@ -4,9 +4,9 @@ import ( "fmt" "os" - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/events/yaml/valid" "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/events/models" @@ -47,7 +47,7 @@ func NewProjectCommandBuilder( skipCloneNoChanges bool, EnableRegExpCmd bool, AutoplanFileList string, - scope stats.Scope, + scope tally.Scope, logger logging.SimpleLogging, ) ProjectCommandBuilder { return NewProjectCommandBuilderWithLimit( @@ -82,7 +82,7 @@ func NewProjectCommandBuilderWithLimit( skipCloneNoChanges bool, EnableRegExpCmd bool, AutoplanFileList string, - scope stats.Scope, + scope tally.Scope, logger logging.SimpleLogging, limit int, ) ProjectCommandBuilder { diff --git a/server/events/project_command_builder_test.go b/server/events/project_command_builder_test.go index 4efe30452..1394f1887 100644 --- a/server/events/project_command_builder_test.go +++ b/server/events/project_command_builder_test.go @@ -7,7 +7,6 @@ import ( "strings" "testing" - stats "github.com/lyft/gostats" . "github.com/petergtz/pegomock" "github.com/runatlantis/atlantis/server/events" "github.com/runatlantis/atlantis/server/events/matchers" @@ -17,6 +16,7 @@ import ( "github.com/runatlantis/atlantis/server/events/yaml" "github.com/runatlantis/atlantis/server/events/yaml/valid" "github.com/runatlantis/atlantis/server/logging" + "github.com/runatlantis/atlantis/server/metrics" . "github.com/runatlantis/atlantis/testing" ) @@ -119,7 +119,7 @@ projects: } logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewLoggingSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { t.Run(c.Description, func(t *testing.T) { @@ -384,7 +384,7 @@ projects: } logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { // NOTE: we're testing both plan and apply here. @@ -544,7 +544,7 @@ projects: } logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for name, c := range cases { t.Run(name, func(t *testing.T) { RegisterMockTestingT(t) @@ -657,7 +657,7 @@ func TestDefaultProjectCommandBuilder_BuildMultiApply(t *testing.T) { ApprovedReq: false, UnDivergedReq: false, } - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -741,8 +741,8 @@ projects: ApprovedReq: false, UnDivergedReq: false, } - scope := stats.NewStore(stats.NewNullSink(), false) logger := logging.NewNoopLogger(t) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -800,7 +800,7 @@ func TestDefaultProjectCommandBuilder_EscapeArgs(t *testing.T) { } logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for _, c := range cases { t.Run(strings.Join(c.ExtraArgs, " "), func(t *testing.T) { @@ -975,7 +975,7 @@ projects: } logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") for name, testCase := range testCases { t.Run(name, func(t *testing.T) { @@ -1071,7 +1071,7 @@ projects: ApprovedReq: false, UnDivergedReq: false, } - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") builder := events.NewProjectCommandBuilder( false, @@ -1115,7 +1115,7 @@ func TestDefaultProjectCommandBuilder_WithPolicyCheckEnabled_BuildAutoplanComman defer cleanup() logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") workingDir := mocks.NewMockWorkingDir() When(workingDir.Clone(matchers.AnyPtrToLoggingSimpleLogger(), matchers.AnyModelsRepo(), matchers.AnyModelsPullRequest(), AnyString())).ThenReturn(tmpDir, false, nil) @@ -1204,7 +1204,7 @@ func TestDefaultProjectCommandBuilder_BuildVersionCommand(t *testing.T) { ThenReturn(tmpDir, nil) logger := logging.NewNoopLogger(t) - scope := stats.NewStore(stats.NewNullSink(), false) + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") globalCfgArgs := valid.GlobalCfgArgs{ AllowRepoCfg: false, diff --git a/server/events/project_command_context_builder.go b/server/events/project_command_context_builder.go index 4c51142a3..59a11656b 100644 --- a/server/events/project_command_context_builder.go +++ b/server/events/project_command_context_builder.go @@ -6,19 +6,19 @@ import ( "github.com/hashicorp/go-version" "github.com/hashicorp/terraform-config-inspect/tfconfig" - stats "github.com/lyft/gostats" + "github.com/uber-go/tally" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/events/yaml/valid" ) -func NewProjectCommandContextBulder(policyCheckEnabled bool, commentBuilder CommentBuilder, scope stats.Scope) ProjectCommandContextBuilder { +func NewProjectCommandContextBulder(policyCheckEnabled bool, commentBuilder CommentBuilder, scope tally.Scope) ProjectCommandContextBuilder { projectCommandContextBuilder := &DefaultProjectCommandContextBuilder{ CommentBuilder: commentBuilder, } contextBuilderWithStats := &CommandScopedStatsProjectCommandContextBuilder{ ProjectCommandContextBuilder: projectCommandContextBuilder, - ProjectCounter: scope.NewCounter("projects"), + ProjectCounter: scope.Counter("projects"), } if policyCheckEnabled { @@ -57,7 +57,7 @@ type ProjectCommandContextBuilder interface { type CommandScopedStatsProjectCommandContextBuilder struct { ProjectCommandContextBuilder // Conciously making this global since it gets flushed periodically anyways - ProjectCounter stats.Counter + ProjectCounter tally.Counter } // BuildProjectContext builds the context and injects the appropriate command level scope after the fact. @@ -69,7 +69,7 @@ func (cb *CommandScopedStatsProjectCommandContextBuilder) BuildProjectContext( repoDir string, contextFlags *ContextFlags, ) (projectCmds []models.ProjectCommandContext) { - cb.ProjectCounter.Inc() + cb.ProjectCounter.Inc(1) cmds := cb.ProjectCommandContextBuilder.BuildProjectContext( ctx, cmdName, prjCfg, commentFlags, repoDir, contextFlags, @@ -207,7 +207,7 @@ func newProjectCommandContext(ctx *CommandContext, policySets valid.PolicySets, escapedCommentArgs []string, contextFlags *ContextFlags, - scope stats.Scope, + scope tally.Scope, pullStatus models.PullReqStatus, ) models.ProjectCommandContext { diff --git a/server/events/vcs/instrumented_client.go b/server/events/vcs/instrumented_client.go index 1c9a6c9d6..32253074a 100644 --- a/server/events/vcs/instrumented_client.go +++ b/server/events/vcs/instrumented_client.go @@ -5,15 +5,15 @@ import ( "strconv" "github.com/google/go-github/v31/github" - stats "github.com/lyft/gostats" + "github.com/uber-go/tally" "github.com/runatlantis/atlantis/server/events/metrics" "github.com/runatlantis/atlantis/server/events/models" "github.com/runatlantis/atlantis/server/logging" ) // NewInstrumentedGithubClient creates a client proxy responsible for gathering stats and logging -func NewInstrumentedGithubClient(client *GithubClient, statsScope stats.Scope, logger logging.SimpleLogging) IGithubClient { - scope := statsScope.Scope("github") +func NewInstrumentedGithubClient(client *GithubClient, statsScope tally.Scope, logger logging.SimpleLogging) IGithubClient { + scope := statsScope.SubScope("github") instrumentedGHClient := &InstrumentedClient{ Client: client, @@ -54,29 +54,29 @@ type IGithubClient interface { type InstrumentedGithubClient struct { *InstrumentedClient GhClient *GithubClient - StatsScope stats.Scope + StatsScope tally.Scope Logger logging.SimpleLogging } func (c *InstrumentedGithubClient) GetContents(owner, repo, branch, path string) ([]byte, error) { - scope := c.StatsScope.Scope("get_contents") + scope := c.StatsScope.SubScope("get_contents") logger := c.Logger.WithHistory([]interface{}{ "repository", fmt.Sprintf("%s/%s", owner, repo), }...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) contents, err := c.GhClient.GetContents(owner, repo, branch, path) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to get contents, error: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return contents, err @@ -88,69 +88,70 @@ func (c *InstrumentedGithubClient) GetPullRequest(repo models.Repo, pullNum int) } func (c *InstrumentedGithubClient) GetPullRequestFromName(repoName string, repoOwner string, pullNum int) (*github.PullRequest, error) { - scope := c.StatsScope.Scope("get_pull_request") + scope := c.StatsScope.SubScope("get_pull_request") logger := c.Logger.WithHistory([]interface{}{ "repository", fmt.Sprintf("%s/%s", repoOwner, repoName), "pull-num", strconv.Itoa(pullNum), }...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) pull, err := c.GhClient.GetPullRequestFromName(repoName, repoOwner, pullNum) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to get pull number for repo, error: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return pull, err } + func (c *InstrumentedGithubClient) GetRepoChecks(repo models.Repo, pull models.PullRequest) ([]*github.CheckRun, error) { - scope := c.StatsScope.Scope("get_repo_checks") + scope := c.StatsScope.SubScope("get_repo_checks") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) statuses, err := c.GhClient.GetRepoChecks(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to get repo status: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return statuses, err } func (c *InstrumentedGithubClient) GetRepoStatuses(repo models.Repo, pull models.PullRequest) ([]*github.RepoStatus, error) { - scope := c.StatsScope.Scope("get_repo_status") + scope := c.StatsScope.SubScope("get_repo_status") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) statuses, err := c.GhClient.GetRepoStatuses(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to get repo status: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return statuses, err @@ -158,151 +159,151 @@ func (c *InstrumentedGithubClient) GetRepoStatuses(repo models.Repo, pull models type InstrumentedClient struct { Client - StatsScope stats.Scope + StatsScope tally.Scope Logger logging.SimpleLogging } func (c *InstrumentedClient) GetModifiedFiles(repo models.Repo, pull models.PullRequest) ([]string, error) { - scope := c.StatsScope.Scope("get_modified_files") + scope := c.StatsScope.SubScope("get_modified_files") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) files, err := c.Client.GetModifiedFiles(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to get modified files, error: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return files, err } func (c *InstrumentedClient) CreateComment(repo models.Repo, pullNum int, comment string, command string) error { - scope := c.StatsScope.Scope("create_comment") + scope := c.StatsScope.SubScope("create_comment") logger := c.Logger.WithHistory(fmtLogSrc(repo, pullNum)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) if err := c.Client.CreateComment(repo, pullNum, comment, command); err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to create comment for command %s, error: %s", command, err.Error()) return err } - executionSuccess.Inc() + executionSuccess.Inc(1) return nil } func (c *InstrumentedClient) HidePrevCommandComments(repo models.Repo, pullNum int, command string) error { - scope := c.StatsScope.Scope("hide_prev_plan_comments") + scope := c.StatsScope.SubScope("hide_prev_plan_comments") logger := c.Logger.WithHistory(fmtLogSrc(repo, pullNum)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) if err := c.Client.HidePrevCommandComments(repo, pullNum, command); err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to hide previous %s comments, error: %s", command, err.Error()) return err } - executionSuccess.Inc() + executionSuccess.Inc(1) return nil } func (c *InstrumentedClient) PullIsApproved(repo models.Repo, pull models.PullRequest) (models.ApprovalStatus, error) { - scope := c.StatsScope.Scope("pull_is_approved") + scope := c.StatsScope.SubScope("pull_is_approved") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) approvalStatus, err := c.Client.PullIsApproved(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to check pull approval status, error: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return approvalStatus, err } func (c *InstrumentedClient) PullIsMergeable(repo models.Repo, pull models.PullRequest) (bool, error) { - scope := c.StatsScope.Scope("pull_is_mergeable") + scope := c.StatsScope.SubScope("pull_is_mergeable") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) mergeable, err := c.Client.PullIsMergeable(repo, pull) if err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to check pull mergeable status, error: %s", err.Error()) } else { - executionSuccess.Inc() + executionSuccess.Inc(1) } return mergeable, err } func (c *InstrumentedClient) UpdateStatus(repo models.Repo, pull models.PullRequest, state models.CommitStatus, src string, description string, url string) error { - scope := c.StatsScope.Scope("update_status") + scope := c.StatsScope.SubScope("update_status") logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) if err := c.Client.UpdateStatus(repo, pull, state, src, description, url); err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to update status at url: %s, error: %s", url, err.Error()) return err } - executionSuccess.Inc() + executionSuccess.Inc(1) return nil } func (c *InstrumentedClient) MergePull(pull models.PullRequest, pullOptions models.PullRequestOptions) error { - scope := c.StatsScope.Scope("merge_pull") + scope := c.StatsScope.SubScope("merge_pull") logger := c.Logger.WithHistory("pull-num", pull.Num) - executionTime := scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() - executionSuccess := scope.NewCounter(metrics.ExecutionSuccessMetric) - executionError := scope.NewCounter(metrics.ExecutionErrorMetric) + executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric) + executionError := scope.Counter(metrics.ExecutionErrorMetric) if err := c.Client.MergePull(pull, pullOptions); err != nil { - executionError.Inc() + executionError.Inc(1) logger.Err("Unable to merge pull, error: %s", err.Error()) } - executionSuccess.Inc() + executionSuccess.Inc(1) return nil } diff --git a/server/events/yaml/valid/global_cfg.go b/server/events/yaml/valid/global_cfg.go index fa46ffb9f..e2d028c44 100644 --- a/server/events/yaml/valid/global_cfg.go +++ b/server/events/yaml/valid/global_cfg.go @@ -35,6 +35,16 @@ type GlobalCfg struct { Repos []Repo Workflows map[string]Workflow PolicySets PolicySets + Metrics Metrics +} + +type Metrics struct { + Statsd *Statsd +} + +type Statsd struct { + Port string + Host string } // Repo is the final parsed version of server-side repo config. diff --git a/server/handlers/instrumented_project_command_output_handler.go b/server/handlers/instrumented_project_command_output_handler.go deleted file mode 100644 index 14ee571b1..000000000 --- a/server/handlers/instrumented_project_command_output_handler.go +++ /dev/null @@ -1,55 +0,0 @@ -package handlers - -import ( - "fmt" - - stats "github.com/lyft/gostats" - "github.com/runatlantis/atlantis/server/events/models" - "github.com/runatlantis/atlantis/server/logging" -) - -type InstrumentedProjectCommandOutputHandler struct { - ProjectCommandOutputHandler - numWSConnnections stats.Gauge - logger logging.SimpleLogging -} - -func NewInstrumentedProjectCommandOutputHandler(projectCmdOutput chan *models.ProjectCmdOutputLine, - projectStatusUpdater ProjectStatusUpdater, - projectJobURLGenerator ProjectJobURLGenerator, - logger logging.SimpleLogging, - scope stats.Scope) ProjectCommandOutputHandler { - prjCmdOutputHandler := NewAsyncProjectCommandOutputHandler( - projectCmdOutput, - projectStatusUpdater, - projectJobURLGenerator, - logger, - ) - return &InstrumentedProjectCommandOutputHandler{ - ProjectCommandOutputHandler: prjCmdOutputHandler, - numWSConnnections: scope.Scope("getprojectjobs").Scope("websocket").NewGauge("connections"), - logger: logger, - } -} - -func (p *InstrumentedProjectCommandOutputHandler) Register(projectInfo string, receiver chan string) { - p.numWSConnnections.Inc() - defer func() { - // Log message to ensure numWSConnnections gauge is being updated properly. - // [ORCA-955] TODO: Remove when removing the feature flag for log streaming. - p.logger.Info(fmt.Sprintf("Decreasing num of ws connections for project: %s", projectInfo)) - p.numWSConnnections.Dec() - }() - p.ProjectCommandOutputHandler.Register(projectInfo, receiver) -} - -func (p *InstrumentedProjectCommandOutputHandler) Deregister(projectInfo string, receiver chan string) { - p.numWSConnnections.Inc() - defer func() { - // Log message to ensure numWSConnnections gauge is being updated properly. - // [ORCA-955] TODO: Remove when removing the feature flag for log streaming. - p.logger.Info(fmt.Sprintf("Decreasing num of ws connections for project: %s", projectInfo)) - p.numWSConnnections.Dec() - }() - p.ProjectCommandOutputHandler.Deregister(projectInfo, receiver) -} diff --git a/server/lyft/aws/sns/writer.go b/server/lyft/aws/sns/writer.go index b11440c8f..1485bb214 100644 --- a/server/lyft/aws/sns/writer.go +++ b/server/lyft/aws/sns/writer.go @@ -5,8 +5,8 @@ import ( "github.com/aws/aws-sdk-go/aws/client" awsSns "github.com/aws/aws-sdk-go/service/sns" snsApi "github.com/aws/aws-sdk-go/service/sns/snsiface" - stats "github.com/lyft/gostats" "github.com/runatlantis/atlantis/server/events/metrics" + "github.com/uber-go/tally" ) //go:generate pegomock generate -m --use-experimental-model-gen --package mocks -o mocks/mock_writer.go Writer @@ -25,10 +25,10 @@ func NewNoopWriter() Writer { func NewWriterWithStats( session client.ConfigProvider, topicArn string, - scope stats.Scope, + scope tally.Scope, ) Writer { return &writerWithStats{ - scope: scope.Scope("aws.sns"), + scope: scope.SubScope("aws.sns"), Writer: &writer{ client: awsSns.New(session), topicArn: aws.String(topicArn), @@ -52,19 +52,19 @@ func (w *writer) Write(payload []byte) error { // writerWithStats decorator to track writing to sns topic type writerWithStats struct { Writer - scope stats.Scope + scope tally.Scope } func (w *writerWithStats) Write(payload []byte) error { - executionTime := w.scope.NewTimer(metrics.ExecutionTimeMetric).AllocateSpan() - defer executionTime.Complete() + executionTime := w.scope.Timer(metrics.ExecutionTimeMetric).Start() + defer executionTime.Stop() if err := w.Writer.Write(payload); err != nil { - w.scope.NewCounter(metrics.ExecutionErrorMetric) + w.scope.Counter(metrics.ExecutionErrorMetric).Inc(1) return err } - w.scope.NewCounter(metrics.ExecutionSuccessMetric) + w.scope.Counter(metrics.ExecutionSuccessMetric).Inc(1) return nil } diff --git a/server/lyft/decorators/audit_project_commands_wrapper_test.go b/server/lyft/decorators/audit_project_commands_wrapper_test.go index 1f4479d32..9650f20cc 100644 --- a/server/lyft/decorators/audit_project_commands_wrapper_test.go +++ b/server/lyft/decorators/audit_project_commands_wrapper_test.go @@ -5,7 +5,6 @@ import ( "errors" "testing" - stats "github.com/lyft/gostats" . "github.com/runatlantis/atlantis/testing" . "github.com/petergtz/pegomock" @@ -16,6 +15,7 @@ import ( "github.com/runatlantis/atlantis/server/logging" snsMocks "github.com/runatlantis/atlantis/server/lyft/aws/sns/mocks" "github.com/runatlantis/atlantis/server/lyft/decorators" + "github.com/runatlantis/atlantis/server/metrics" ) func TestAuditProjectCommandsWrapper(t *testing.T) { @@ -60,9 +60,13 @@ func TestAuditProjectCommandsWrapper(t *testing.T) { prjRslt.Failure = "oh-no" } + logger := logging.NewNoopLogger(t) + + scope, _, _ := metrics.NewLoggingScope(logger, "atlantis") + ctx := models.ProjectCommandContext{ - Scope: stats.NewStore(stats.NewNullSink(), false), - Log: logging.NewNoopLogger(t), + Scope: scope, + Log: logger, Steps: []valid.Step{}, ProjectName: "test-project", User: models.User{ diff --git a/server/lyft/scheduled/executor_service.go b/server/lyft/scheduled/executor_service.go index a05554926..f7fa59e1c 100644 --- a/server/lyft/scheduled/executor_service.go +++ b/server/lyft/scheduled/executor_service.go @@ -2,6 +2,12 @@ package scheduled import ( "context" + "github.com/runatlantis/atlantis/server/events" + "github.com/runatlantis/atlantis/server/events/metrics" + "github.com/runatlantis/atlantis/server/events/models" + "github.com/runatlantis/atlantis/server/events/vcs" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" "io" "os" "os/signal" @@ -10,36 +16,30 @@ import ( "syscall" "text/template" "time" - - stats "github.com/lyft/gostats" - "github.com/runatlantis/atlantis/server/events" - "github.com/runatlantis/atlantis/server/events/metrics" - "github.com/runatlantis/atlantis/server/events/models" - "github.com/runatlantis/atlantis/server/events/vcs" - "github.com/runatlantis/atlantis/server/logging" ) type ExecutorService struct { log logging.SimpleLogging // jobs - garbageCollector JobDefinition - rateLimitPublisher JobDefinition + garbageCollector JobDefinition + rateLimitPublisher JobDefinition + runtimeStatsPublisher JobDefinition } func NewExecutorService( workingDirIterator events.WorkDirIterator, - statsScope stats.Scope, + statsScope tally.Scope, log logging.SimpleLogging, closedPullCleaner events.PullCleaner, openPullCleaner events.PullCleaner, githubClient *vcs.GithubClient, ) *ExecutorService { - scheduledScope := statsScope.Scope("scheduled") + scheduledScope := statsScope.SubScope("scheduled") garbageCollector := &GarbageCollector{ workingDirIterator: workingDirIterator, - stats: scheduledScope.Scope("garbagecollector"), + stats: scheduledScope.SubScope("garbagecollector"), log: log, closedPullCleaner: closedPullCleaner, openPullCleaner: openPullCleaner, @@ -53,7 +53,7 @@ func NewExecutorService( rateLimitPublisher := &RateLimitStatsPublisher{ client: githubClient, - stats: scheduledScope.Scope("ratelimitpublisher"), + stats: scheduledScope.SubScope("ratelimitpublisher"), log: log, } @@ -64,10 +64,18 @@ func NewExecutorService( Period: 1 * time.Minute, } + runtimeStatsPublisher := NewRuntimeStats(scheduledScope) + + runtimeStatsPublisherJob := JobDefinition{ + Job: runtimeStatsPublisher, + Period: 10 * time.Second, + } + return &ExecutorService{ - log: log, - garbageCollector: garbageCollectorJob, - rateLimitPublisher: rateLimitPublisherJob, + log: log, + garbageCollector: garbageCollectorJob, + rateLimitPublisher: rateLimitPublisherJob, + runtimeStatsPublisher: runtimeStatsPublisherJob, } } @@ -85,6 +93,7 @@ func (s *ExecutorService) Run() { s.runScheduledJob(ctx, &wg, s.garbageCollector) s.runScheduledJob(ctx, &wg, s.rateLimitPublisher) + s.runScheduledJob(ctx, &wg, s.runtimeStatsPublisher) interrupt := make(chan os.Signal, 1) @@ -136,22 +145,22 @@ type Job interface { type RateLimitStatsPublisher struct { log logging.SimpleLogging - stats stats.Scope + stats tally.Scope client *vcs.GithubClient } func (r *RateLimitStatsPublisher) Run() { - errCounter := r.stats.NewCounter(metrics.ExecutionErrorMetric) - rateLimitRemainingCounter := r.stats.NewCounter("ratelimitremaining") + errCounter := r.stats.Counter(metrics.ExecutionErrorMetric) + rateLimitRemainingCounter := r.stats.Counter("ratelimitremaining") rateLimits, err := r.client.GetRateLimits() if err != nil { - errCounter.Inc() + errCounter.Inc(1) return } - rateLimitRemainingCounter.Add(uint64(rateLimits.GetCore().Remaining)) + rateLimitRemainingCounter.Inc(int64(rateLimits.GetCore().Remaining)) } var gcStaleClosedPullTemplate = template.Must(template.New("").Parse( @@ -186,26 +195,26 @@ func (t *GCStalePullTemplate) Execute(wr io.Writer, data interface{}) error { type GarbageCollector struct { workingDirIterator events.WorkDirIterator - stats stats.Scope + stats tally.Scope log logging.SimpleLogging closedPullCleaner events.PullCleaner openPullCleaner events.PullCleaner } func (g *GarbageCollector) Run() { - errCounter := g.stats.NewCounter(metrics.ExecutionErrorMetric) + errCounter := g.stats.Counter(metrics.ExecutionErrorMetric) pulls, err := g.workingDirIterator.ListCurrentWorkingDirPulls() if err != nil { g.log.Err("error listing pulls %s", err) - errCounter.Inc() + errCounter.Inc(1) } - openPullsCounter := g.stats.NewCounter("pulls.open") - updatedthirtyDaysAgoOpenPullsCounter := g.stats.NewCounter("pulls.open.updated.thirtydaysago") - closedPullsCounter := g.stats.NewCounter("pulls.closed") - fiveMinutesAgoClosedPullsCounter := g.stats.NewCounter("pulls.closed.fiveminutesago") + openPullsCounter := g.stats.Counter("pulls.open") + updatedthirtyDaysAgoOpenPullsCounter := g.stats.Counter("pulls.open.updated.thirtydaysago") + closedPullsCounter := g.stats.Counter("pulls.closed") + fiveMinutesAgoClosedPullsCounter := g.stats.Counter("pulls.closed.fiveminutesago") // we can make this shorter, but this allows us to see trends more clearly // to determine if there is an issue or not @@ -216,10 +225,10 @@ func (g *GarbageCollector) Run() { logger := g.log.With(fmtLogSrc(pull.BaseRepo, pull.Num)...) if pull.State == models.OpenPullState { - openPullsCounter.Inc() + openPullsCounter.Inc(1) if pull.UpdatedAt.Before(thirtyDaysAgo) { - updatedthirtyDaysAgoOpenPullsCounter.Inc() + updatedthirtyDaysAgoOpenPullsCounter.Inc(1) logger.Warn("Pull hasn't been updated for more than 30 days.") @@ -227,7 +236,7 @@ func (g *GarbageCollector) Run() { if err != nil { logger.Err("Error cleaning up open pulls that haven't been updated in 30 days %s", err) - errCounter.Inc() + errCounter.Inc(1) return } } @@ -235,12 +244,12 @@ func (g *GarbageCollector) Run() { } // assume only other state is closed - closedPullsCounter.Inc() + closedPullsCounter.Inc(1) // Let's clean up any closed pulls within 5 minutes of closing to ensure that // any locks are released. if pull.ClosedAt.Before(fiveMinutesAgo) { - fiveMinutesAgoClosedPullsCounter.Inc() + fiveMinutesAgoClosedPullsCounter.Inc(1) logger.Warn("Pull closed for more than 5 minutes but data still on disk") @@ -248,7 +257,7 @@ func (g *GarbageCollector) Run() { if err != nil { logger.Err("Error cleaning up 5 minutes old closed pulls %s", err) - errCounter.Inc() + errCounter.Inc(1) return } } diff --git a/server/lyft/scheduled/runtime_stats.go b/server/lyft/scheduled/runtime_stats.go new file mode 100644 index 000000000..78836267e --- /dev/null +++ b/server/lyft/scheduled/runtime_stats.go @@ -0,0 +1,128 @@ +package scheduled + +import ( + "runtime" + + "github.com/uber-go/tally" +) + +type RuntimeStatCollector struct { + runtimeMetrics runtimeMetrics +} + +type runtimeMetrics struct { + cpuGoroutines tally.Gauge + cpuCgoCalls tally.Gauge + + memoryAlloc tally.Gauge + memoryTotal tally.Gauge + memorySys tally.Gauge + memoryLookups tally.Gauge + memoryMalloc tally.Gauge + memoryFrees tally.Gauge + + memoryHeapAlloc tally.Gauge + memoryHeapSys tally.Gauge + memoryHeapIdle tally.Gauge + memoryHeapInuse tally.Gauge + memoryHeapReleased tally.Gauge + memoryHeapObjects tally.Gauge + + memoryStackInuse tally.Gauge + memoryStackSys tally.Gauge + memoryStackMSpanInuse tally.Gauge + memoryStackMSpanSys tally.Gauge + memoryStackMCacheInuse tally.Gauge + memoryStackMCacheSys tally.Gauge + + memoryOtherSys tally.Gauge + + memoryGCSys tally.Gauge + memoryGCNext tally.Gauge + memoryGCLast tally.Gauge + memoryGCPauseTotal tally.Gauge + memoryGCCount tally.Gauge +} + +func NewRuntimeStats(scope tally.Scope) *RuntimeStatCollector { + runtimeScope := scope.SubScope("runtime") + runtimeMetrics := runtimeMetrics{ + // cpu + cpuGoroutines: runtimeScope.Gauge("cpu.goroutines"), + cpuCgoCalls: runtimeScope.Gauge("cpu.cgo_calls"), + // memory + memoryAlloc: runtimeScope.Gauge("memory.alloc"), + memoryTotal: runtimeScope.Gauge("memory.total"), + memorySys: runtimeScope.Gauge("memory.sys"), + memoryLookups: runtimeScope.Gauge("memory.lookups"), + memoryMalloc: runtimeScope.Gauge("memory.malloc"), + memoryFrees: runtimeScope.Gauge("memory.frees"), + // heap + memoryHeapAlloc: runtimeScope.Gauge("memory.heap.alloc"), + memoryHeapSys: runtimeScope.Gauge("memory.heap.sys"), + memoryHeapIdle: runtimeScope.Gauge("memory.heap.idle"), + memoryHeapInuse: runtimeScope.Gauge("memory.heap.inuse"), + memoryHeapReleased: runtimeScope.Gauge("memory.heap.released"), + memoryHeapObjects: runtimeScope.Gauge("memory.heap.objects"), + // stack + memoryStackInuse: runtimeScope.Gauge("memory.stack.inuse"), + memoryStackSys: runtimeScope.Gauge("memory.stack.sys"), + memoryStackMSpanInuse: runtimeScope.Gauge("memory.stack.mspan_inuse"), + memoryStackMSpanSys: runtimeScope.Gauge("memory.stack.sys"), + memoryStackMCacheInuse: runtimeScope.Gauge("memory.stack.mcache_inuse"), + memoryStackMCacheSys: runtimeScope.Gauge("memory.stack.mcache_sys"), + memoryOtherSys: runtimeScope.Gauge("memory.othersys"), + // GC + memoryGCSys: runtimeScope.Gauge("memory.gc.sys"), + memoryGCNext: runtimeScope.Gauge("memory.gc.next"), + memoryGCLast: runtimeScope.Gauge("memory.gc.last"), + memoryGCPauseTotal: runtimeScope.Gauge("memory.gc.pause_total"), + memoryGCCount: runtimeScope.Gauge("memory.gc.count"), + } + + return &RuntimeStatCollector{ + runtimeMetrics: runtimeMetrics, + } +} + +func (r *RuntimeStatCollector) Run() { + // cpu stats + r.runtimeMetrics.cpuGoroutines.Update(float64(runtime.NumGoroutine())) + r.runtimeMetrics.cpuCgoCalls.Update(float64(runtime.NumCgoCall())) + + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + + // general + r.runtimeMetrics.memoryAlloc.Update(float64(memStats.Alloc)) + r.runtimeMetrics.memoryTotal.Update(float64(memStats.TotalAlloc)) + r.runtimeMetrics.memorySys.Update(float64(memStats.Sys)) + r.runtimeMetrics.memoryLookups.Update(float64(memStats.Lookups)) + r.runtimeMetrics.memoryMalloc.Update(float64(memStats.Mallocs)) + r.runtimeMetrics.memoryFrees.Update(float64(memStats.Frees)) + + // heap + r.runtimeMetrics.memoryHeapAlloc.Update(float64(memStats.HeapAlloc)) + r.runtimeMetrics.memoryHeapSys.Update(float64(memStats.HeapSys)) + r.runtimeMetrics.memoryHeapIdle.Update(float64(memStats.HeapIdle)) + r.runtimeMetrics.memoryHeapInuse.Update(float64(memStats.HeapInuse)) + r.runtimeMetrics.memoryHeapReleased.Update(float64(memStats.HeapReleased)) + r.runtimeMetrics.memoryHeapObjects.Update(float64(memStats.HeapObjects)) + + // stack + r.runtimeMetrics.memoryStackInuse.Update(float64(memStats.StackInuse)) + r.runtimeMetrics.memoryStackSys.Update(float64(memStats.StackSys)) + r.runtimeMetrics.memoryStackMSpanInuse.Update(float64(memStats.MSpanInuse)) + r.runtimeMetrics.memoryStackMSpanSys.Update(float64(memStats.MSpanSys)) + r.runtimeMetrics.memoryStackMCacheInuse.Update(float64(memStats.MCacheInuse)) + r.runtimeMetrics.memoryStackMCacheSys.Update(float64(memStats.MCacheSys)) + r.runtimeMetrics.memoryOtherSys.Update(float64(memStats.OtherSys)) + + // GC + r.runtimeMetrics.memoryGCSys.Update(float64(memStats.GCSys)) + r.runtimeMetrics.memoryGCNext.Update(float64(memStats.NextGC)) + r.runtimeMetrics.memoryGCLast.Update(float64(memStats.LastGC)) + r.runtimeMetrics.memoryGCPauseTotal.Update(float64(memStats.PauseTotalNs)) + r.runtimeMetrics.memoryGCCount.Update(float64(memStats.NumGC)) + +} diff --git a/server/metrics/debug.go b/server/metrics/debug.go new file mode 100644 index 000000000..10560755e --- /dev/null +++ b/server/metrics/debug.go @@ -0,0 +1,93 @@ +package metrics + +import ( + "time" + + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" +) + +// newLoggingReporter returns a tally reporter that logs to the provided logger at debug level. This is useful for +// local development where the usual sinks are not available. +func newLoggingReporter(logger logging.SimpleLogging) tally.StatsReporter { + return &debugReporter{log: logger} +} + +type debugReporter struct { + log logging.SimpleLogging +} + +// Capabilities interface. + +func (r *debugReporter) Reporting() bool { + return true +} + +func (r *debugReporter) Tagging() bool { + return true +} + +func (r *debugReporter) Capabilities() tally.Capabilities { + return r +} + +// Reporter interface. + +func (r *debugReporter) Flush() { + // Silence. +} + +func (r *debugReporter) ReportCounter(name string, tags map[string]string, value int64) { + log := r.log.With("name", name, "value", value, "tags", tags, "type", "counter") + log.Debug("counter") +} + +func (r *debugReporter) ReportGauge(name string, tags map[string]string, value float64) { + log := r.log.With("name", name, "value", value, "tags", tags, "type", "gauge") + log.Debug("gauge") +} + +func (r *debugReporter) ReportTimer(name string, tags map[string]string, interval time.Duration) { + log := r.log.With("name", name, "value", interval, "tags", tags, "type", "timer") + log.Debug("timer") +} + +func (r *debugReporter) ReportHistogramValueSamples( + name string, + tags map[string]string, + buckets tally.Buckets, + bucketLowerBound, + bucketUpperBound float64, + samples int64, +) { + log := r.log.With( + "name", name, + "buckets", buckets.AsValues(), + "bucketLowerBound", bucketLowerBound, + "bucketUpperBound", bucketUpperBound, + "samples", samples, + "tags", tags, + "type", "valueHistogram", + ) + log.Debug("histogram") +} + +func (r *debugReporter) ReportHistogramDurationSamples( + name string, + tags map[string]string, + buckets tally.Buckets, + bucketLowerBound, + bucketUpperBound time.Duration, + samples int64, +) { + log := r.log.With( + "name", name, + "buckets", buckets.AsValues(), + "bucketLowerBound", bucketLowerBound, + "bucketUpperBound", bucketUpperBound, + "samples", samples, + "tags", tags, + "type", "durationHistogram", + ) + log.Debug("histogram") +} diff --git a/server/metrics/scope.go b/server/metrics/scope.go new file mode 100644 index 000000000..d92c45e60 --- /dev/null +++ b/server/metrics/scope.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "io" + "strings" + "time" + + "github.com/cactus/go-statsd-client/statsd" + "github.com/pkg/errors" + "github.com/runatlantis/atlantis/server/events/yaml/valid" + "github.com/runatlantis/atlantis/server/logging" + "github.com/uber-go/tally" + tallystatsd "github.com/uber-go/tally/statsd" +) + +func NewLoggingScope(logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { + reporter, err := newReporter(valid.Metrics{}, logger) + + if err != nil { + return nil, nil, errors.Wrap(err, "initializing stats reporter") + } + + scope, closer := tally.NewRootScope(tally.ScopeOptions{ + Prefix: statsNamespace, + Reporter: reporter, + }, time.Second) + + return scope, closer, nil +} + +func NewScope(cfg valid.Metrics, logger logging.SimpleLogging, statsNamespace string) (tally.Scope, io.Closer, error) { + reporter, err := newReporter(cfg, logger) + + if err != nil { + return nil, nil, errors.Wrap(err, "initializing stats reporter") + } + + scope, closer := tally.NewRootScope(tally.ScopeOptions{ + Prefix: statsNamespace, + Reporter: reporter, + }, time.Second) + + return scope, closer, nil +} + +func newReporter(cfg valid.Metrics, logger logging.SimpleLogging) (tally.StatsReporter, error) { + if cfg.Statsd == nil { + // return logging reporter and proceed + return newLoggingReporter(logger), nil + } + + statsdCfg := cfg.Statsd + + client, err := statsd.NewClientWithConfig(&statsd.ClientConfig{ + Address: strings.Join([]string{statsdCfg.Host, statsdCfg.Port}, ":"), + }) + + if err != nil { + return nil, errors.Wrap(err, "initializing statsd client") + } + + return tallystatsd.NewReporter(client, tallystatsd.Options{}), nil +} diff --git a/server/server.go b/server/server.go index 9ab2a92b0..1314a994b 100644 --- a/server/server.go +++ b/server/server.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "io" "io/ioutil" "log" "net/http" @@ -40,10 +41,11 @@ import ( lyftDecorators "github.com/runatlantis/atlantis/server/lyft/decorators" "github.com/runatlantis/atlantis/server/lyft/feature" "github.com/runatlantis/atlantis/server/lyft/scheduled" + "github.com/runatlantis/atlantis/server/metrics" + "github.com/uber-go/tally" assetfs "github.com/elazarl/go-bindata-assetfs" "github.com/gorilla/mux" - stats "github.com/lyft/gostats" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/controllers" events_controllers "github.com/runatlantis/atlantis/server/controllers/events" @@ -95,7 +97,8 @@ type Server struct { PreWorkflowHooksCommandRunner *events.DefaultPreWorkflowHooksCommandRunner CommandRunner *events.DefaultCommandRunner Logger logging.SimpleLogging - StatsScope stats.Scope + StatsScope tally.Scope + StatsCloser io.Closer Locker locking.Locker ApplyLocker locking.ApplyLocker VCSEventsController *events_controllers.VCSEventsController @@ -149,9 +152,6 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { return nil, err } - statsScope := stats.NewDefaultStore().Scope(userConfig.StatsNamespace) - statsScope.Store().AddStatGenerator(stats.NewRuntimeStats(statsScope.Scope("go"))) - var supportedVCSHosts []models.VCSHostType // not to be used directly, currently this is just used @@ -174,6 +174,41 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { policyChecksEnabled = true } + validator := &yaml.ParserValidator{} + + globalCfg := valid.NewGlobalCfgFromArgs( + valid.GlobalCfgArgs{ + AllowRepoCfg: userConfig.AllowRepoConfig, + MergeableReq: userConfig.RequireMergeable, + ApprovedReq: userConfig.RequireApproval, + UnDivergedReq: userConfig.RequireUnDiverged, + SQUnLockedReq: userConfig.RequireSQUnlocked, + PolicyCheckEnabled: userConfig.EnablePolicyChecksFlag, + }) + if userConfig.RepoConfig != "" { + globalCfg, err = validator.ParseGlobalCfg(userConfig.RepoConfig, globalCfg) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s file", userConfig.RepoConfig) + } + } else if userConfig.RepoConfigJSON != "" { + globalCfg, err = validator.ParseGlobalCfgJSON(userConfig.RepoConfigJSON, globalCfg) + if err != nil { + return nil, errors.Wrapf(err, "parsing --%s", config.RepoConfigJSONFlag) + } + } + + // TODO: move this to yaml in a followup + globalCfg.Metrics.Statsd = &valid.Statsd{ + Host: "127.0.0.1", + Port: "8125", + } + + statsScope, closer, err := metrics.NewScope(globalCfg.Metrics, logger, userConfig.StatsNamespace) + + if err != nil { + return nil, errors.Wrapf(err, "instantiating metrics scope") + } + if userConfig.GithubUser != "" || userConfig.GithubAppID != 0 { supportedVCSHosts = append(supportedVCSHosts, models.Github) if userConfig.GithubUser != "" { @@ -346,12 +381,11 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { projectCmdOutputHandler = &handlers.NoopProjectOutputHandler{} } else { projectCmdOutput := make(chan *models.ProjectCmdOutputLine) - projectCmdOutputHandler = handlers.NewInstrumentedProjectCommandOutputHandler( + projectCmdOutputHandler = handlers.NewAsyncProjectCommandOutputHandler( projectCmdOutput, commitStatusUpdater, router, logger, - statsScope.Scope("api"), ) } @@ -423,29 +457,6 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { DB: boltdb, } - validator := &yaml.ParserValidator{} - - globalCfg := valid.NewGlobalCfgFromArgs( - valid.GlobalCfgArgs{ - AllowRepoCfg: userConfig.AllowRepoConfig, - MergeableReq: userConfig.RequireMergeable, - ApprovedReq: userConfig.RequireApproval, - UnDivergedReq: userConfig.RequireUnDiverged, - SQUnLockedReq: userConfig.RequireSQUnlocked, - PolicyCheckEnabled: userConfig.EnablePolicyChecksFlag, - }) - if userConfig.RepoConfig != "" { - globalCfg, err = validator.ParseGlobalCfg(userConfig.RepoConfig, globalCfg) - if err != nil { - return nil, errors.Wrapf(err, "parsing %s file", userConfig.RepoConfig) - } - } else if userConfig.RepoConfigJSON != "" { - globalCfg, err = validator.ParseGlobalCfgJSON(userConfig.RepoConfigJSON, globalCfg) - if err != nil { - return nil, errors.Wrapf(err, "parsing --%s", config.RepoConfigJSONFlag) - } - } - pullClosedExecutor := events.NewInstrumentedPullClosedExecutor( statsScope, logger, @@ -708,7 +719,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { EventParser: eventParser, Logger: logger, GlobalCfg: globalCfg, - StatsScope: statsScope.Scope("cmd"), + StatsScope: statsScope.SubScope("cmd"), AllowForkPRs: userConfig.AllowForkPRs, AllowForkPRsFlag: config.AllowForkPRsFlag, SilenceForkPRErrors: userConfig.SilenceForkPRErrors, @@ -758,7 +769,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { ProjectJobsErrorTemplate: templates.ProjectJobsErrorTemplate, Db: boltdb, WsMux: wsMux, - StatsScope: statsScope.Scope("api"), + StatsScope: statsScope.SubScope("api"), } eventsController := &events_controllers.VCSEventsController{ @@ -837,6 +848,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) { CommandRunner: commandRunner, Logger: logger, StatsScope: statsScope, + StatsCloser: closer, Locker: lockingClient, ApplyLocker: applyLockingClient, VCSEventsController: eventsController, @@ -917,7 +929,9 @@ func (s *Server) Start() error { s.waitForDrain() // flush stats before shutdown - s.StatsScope.Store().Flush() + if err := s.StatsCloser.Close(); err != nil { + s.Logger.Err(err.Error()) + } ctx, _ := context.WithTimeout(context.Background(), 5*time.Second) // nolint: vet if err := server.Shutdown(ctx); err != nil {