From 4d1d0e0701bd1495891dcc19243e7e1e7af7cf55 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 21 Jan 2020 16:20:15 +0530 Subject: [PATCH] Report GraphQL stats from alpha. (#4607) * Report GraphQL stats from alpha. * Remove old telemetry file. * Addressed comments. * Stop waiting for telemetry go routine when stopping dgraph. --- dgraph/cmd/alpha/run.go | 6 ++++ dgraph/cmd/zero/zero.go | 10 +++--- edgraph/server.go | 40 ++++++++++++++++++++- {dgraph/cmd/zero => telemetry}/telemetry.go | 33 ++++++++++++----- 4 files changed, 76 insertions(+), 13 deletions(-) rename {dgraph/cmd/zero => telemetry}/telemetry.go (74%) diff --git a/dgraph/cmd/alpha/run.go b/dgraph/cmd/alpha/run.go index cecee70d34b..f539022969e 100644 --- a/dgraph/cmd/alpha/run.go +++ b/dgraph/cmd/alpha/run.go @@ -164,6 +164,7 @@ they form a Raft group and provide synchronous replication. "Actual usage by the process would be more than specified here.") flag.String("mutations", "allow", "Set mutation mode to allow, disallow, or strict.") + flag.Bool("telemetry", true, "Send anonymous telemetry data to Dgraph devs.") // Useful for running multiple servers on the same machine. flag.IntP("port_offset", "o", 0, @@ -487,9 +488,14 @@ func setupServer(closer *y.Closer) { go serveGRPC(grpcListener, tlsCfg, &wg) go serveHTTP(httpListener, tlsCfg, &wg) + if Alpha.Conf.GetBool("telemetry") { + go edgraph.PeriodicallyPostTelemetry() + } + go func() { defer wg.Done() <-shutdownCh + // Stops grpc/http servers; Already accepted connections are not closed. if err := grpcListener.Close(); err != nil { glog.Warningf("Error while closing gRPC listener: %s", err) diff --git a/dgraph/cmd/zero/zero.go b/dgraph/cmd/zero/zero.go index 44e1abf7d80..59d663a7260 100644 --- a/dgraph/cmd/zero/zero.go +++ b/dgraph/cmd/zero/zero.go @@ -30,6 +30,7 @@ import ( "github.com/dgraph-io/dgo/v2/protos/api" "github.com/dgraph-io/dgraph/conn" "github.com/dgraph-io/dgraph/protos/pb" + "github.com/dgraph-io/dgraph/telemetry" "github.com/dgraph-io/dgraph/x" "github.com/gogo/protobuf/proto" "github.com/golang/glog" @@ -94,7 +95,7 @@ func (s *Server) Init() { } func (s *Server) periodicallyPostTelemetry() { - glog.V(2).Infof("Starting telemetry data collection...") + glog.V(2).Infof("Starting telemetry data collection for zero...") start := time.Now() ticker := time.NewTicker(time.Minute) @@ -109,17 +110,18 @@ func (s *Server) periodicallyPostTelemetry() { continue } ms := s.membershipState() - t := newTelemetry(ms) + t := telemetry.NewZero(ms) if t == nil { continue } t.SinceHours = int(time.Since(start).Hours()) glog.V(2).Infof("Posting Telemetry data: %+v", t) - err := t.post() - glog.V(2).Infof("Telemetry data posted with error: %v", err) + err := t.Post() if err == nil { lastPostedAt = time.Now() + } else { + glog.V(2).Infof("Telemetry couldn't be posted. Error: %v", err) } } } diff --git a/edgraph/server.go b/edgraph/server.go index c17639ea3b4..206cc03cdaa 100644 --- a/edgraph/server.go +++ b/edgraph/server.go @@ -23,6 +23,7 @@ import ( "sort" "strconv" "strings" + "sync/atomic" "time" "unicode" @@ -37,6 +38,7 @@ import ( "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/query" "github.com/dgraph-io/dgraph/schema" + "github.com/dgraph-io/dgraph/telemetry" "github.com/dgraph-io/dgraph/types" "github.com/dgraph-io/dgraph/types/facets" "github.com/dgraph-io/dgraph/worker" @@ -76,9 +78,44 @@ const ( isGraphQL key = iota ) +var ( + numQueries uint64 + numGraphQL uint64 +) + // Server implements protos.DgraphServer type Server struct{} +// PeriodicallyPostTelemetry periodically reports telemetry data for alpha. +func PeriodicallyPostTelemetry() { + glog.V(2).Infof("Starting telemetry data collection for alpha...") + + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + + var lastPostedAt time.Time + for range ticker.C { + if time.Since(lastPostedAt) < time.Hour { + continue + } + ms := worker.GetMembershipState() + t := telemetry.NewAlpha(ms) + t.NumQueries = atomic.SwapUint64(&numQueries, 0) + t.NumGraphQL = atomic.SwapUint64(&numGraphQL, 0) + t.SinceHours = int(time.Since(lastPostedAt).Hours()) + glog.V(2).Infof("Posting Telemetry data: %+v", t) + + err := t.Post() + if err == nil { + lastPostedAt = time.Now() + } else { + atomic.AddUint64(&numQueries, t.NumQueries) + atomic.AddUint64(&numGraphQL, t.NumGraphQL) + glog.V(2).Infof("Telemetry couldn't be posted. Error: %v", err) + } + } +} + // Alter handles requests to change the schema or remove parts or all of the data. func (s *Server) Alter(ctx context.Context, op *api.Operation) (*api.Payload, error) { ctx, span := otrace.StartSpan(ctx, "Server.Alter") @@ -666,17 +703,18 @@ func (s *Server) State(ctx context.Context) (*api.Response, error) { // Query handles queries or mutations func (s *Server) Query(ctx context.Context, req *api.Request) (*api.Response, error) { + atomic.AddUint64(&numGraphQL, 1) return s.doQuery(ctx, req, NeedAuthorize) } // QueryForGraphql handles queries or mutations func (s *Server) QueryForGraphql(ctx context.Context, req *api.Request) (*api.Response, error) { + atomic.AddUint64(&numQueries, 1) return s.doQuery(context.WithValue(ctx, isGraphQL, true), req, NeedAuthorize) } func (s *Server) doQuery(ctx context.Context, req *api.Request, authorize int) ( resp *api.Response, rerr error) { - if ctx.Err() != nil { return nil, ctx.Err() } diff --git a/dgraph/cmd/zero/telemetry.go b/telemetry/telemetry.go similarity index 74% rename from dgraph/cmd/zero/telemetry.go rename to telemetry/telemetry.go index 979327889a6..5c70c892f01 100644 --- a/dgraph/cmd/zero/telemetry.go +++ b/telemetry/telemetry.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package zero +package telemetry import ( "bytes" @@ -22,13 +22,14 @@ import ( "io/ioutil" "net/http" "runtime" + "time" "github.com/dgraph-io/dgraph/protos/pb" "github.com/dgraph-io/dgraph/x" "github.com/golang/glog" ) -// Telemetry holds information about the state of the zero server. +// Telemetry holds information about the state of the zero and alpha server. type Telemetry struct { Arch string Cid string @@ -41,11 +42,14 @@ type Telemetry struct { OS string SinceHours int Version string + NumQueries uint64 + NumGraphQL uint64 } -var keenURL = "https://ping.dgraph.io/3.0/projects/5b809dfac9e77c0001783ad0/events" +var url = "https://ping.dgraph.io/3.0/projects/5b809dfac9e77c0001783ad0/events" -func newTelemetry(ms *pb.MembershipState) *Telemetry { +// NewZero returns a Telemetry struct that holds information about the state of zero server. +func NewZero(ms *pb.MembershipState) *Telemetry { if len(ms.Cid) == 0 { glog.V(2).Infoln("No CID found yet") return nil @@ -70,14 +74,27 @@ func newTelemetry(ms *pb.MembershipState) *Telemetry { return t } -func (t *Telemetry) post() error { +// NewAlpha returns a Telemetry struct that holds information about the state of alpha server. +func NewAlpha(ms *pb.MembershipState) *Telemetry { + return &Telemetry{ + Cid: ms.Cid, + Version: x.Version(), + OS: runtime.GOOS, + Arch: runtime.GOARCH, + } +} + +// Post reports the Telemetry to the stats server. +func (t *Telemetry) Post() error { data, err := json.Marshal(t) if err != nil { return err } - url := keenURL + "/dev" + if len(t.Version) > 0 { - url = keenURL + "/pings" + url += "/pings" + } else { + url += "/dev" } req, err := http.NewRequest("POST", url, bytes.NewBuffer(data)) if err != nil { @@ -88,7 +105,7 @@ func (t *Telemetry) post() error { "97497CA758881BD7D56CC2355A2F36B4560102CBC3279AC7B27E5391372C36A31167EB0D06BF3764894AD20"+ "A0554BAFF14C292A40BC252BB9FF008736A0FD1D44E085") - client := &http.Client{} + client := &http.Client{Timeout: 10 * time.Second} resp, err := client.Do(req) if err != nil { return err