Skip to content

Commit

Permalink
Add support to show metrics (#4033)
Browse files Browse the repository at this point in the history 
* add num_uids metrics to extensions. num_uids shows number 
of uids processed by each attribute.
  • Loading branch information
@poonai
poonai authored Nov 7, 2019
1 parent adbf1f7 commit ab8984b
Show file tree
Hide file tree
Showing 9 changed files with 73 additions and 11 deletions.
18 changes: 9 additions & 9 deletions contrib/scripts/goldendata-queries.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ function run_index_test {
}

echo -e "Running some queries and checking count of results returned."
run_index_test basic name 138676
run_index_test allof_the name 25431
run_index_test allof_the_a name 367
run_index_test allof_the_first name 4383
run_index_test releasedate release_date 137858
run_index_test releasedate_sort release_date 137858
run_index_test releasedate_sort_first_offset release_date 2315
run_index_test releasedate_geq release_date 60991
run_index_test gen_anyof_good_bad name 1103
run_index_test basic name 138677
run_index_test allof_the name 25432
run_index_test allof_the_a name 368
run_index_test allof_the_first name 4384
run_index_test releasedate release_date 137859
run_index_test releasedate_sort release_date 137859
run_index_test releasedate_sort_first_offset release_date 2316
run_index_test releasedate_geq release_date 60992
run_index_test gen_anyof_good_bad name 1104

popd &> /dev/null

1 change: 1 addition & 0 deletions dgraph/cmd/alpha/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
e := query.Extensions{
Txn: resp.Txn,
Latency: resp.Latency,
Metrics: resp.Metrics,
}
js, err := json.Marshal(e)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions edgraph/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1064,6 +1064,9 @@ func (s *Server) doQuery(ctx context.Context, req *api.Request, authorize int) (
}

resp.Latency = gl
resp.Metrics = &api.Metrics{
NumUids: er.Metrics,
}
return resp, err
}

Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
github.com/dgryski/go-groupvarint v0.0.0-20190318181831-5ce5df8ca4e1
github.com/dustin/go-humanize v1.0.0
github.com/go-ini/ini v1.39.0 // indirect
github.com/go-sql-driver/mysql v0.0.0-20190330032241-c0f6b444ad8f
github.com/gogo/protobuf v1.2.0
github.com/gogo/protobuf v1.3.1
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/protobuf v1.3.2
Expand Down
6 changes: 6 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7 h1:sXNo2Xtte2hOGe
github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7/go.mod h1:V7oadAIwiqodXo5jT04lP+8ysOAuMFZO5x20VSI94Uc=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da h1:JAadBE19xm15ZsBZ9HOSuQvMiaCKu+NsV5O5Mfb/62M=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da/go.mod h1:R/MTZMGhTo60XSziuKpLXzI1OnVWQCZS5oJjxA8Q1bo=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b h1:p3gcEqQqQBp4jPIYO5x05sZbY0n8WVp+V1F/EnbJ1oo=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI=
github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30 h1:FkdGlqxPjfHKdVUzS5dOSMeOkGjEG7PnR1RLbcEqw88=
github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q=
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 h1:9G6fVccQriMJu4nXwpwLDoy9y31t/KUSLAbPcoBgv+4=
Expand Down Expand Up @@ -90,6 +92,8 @@ github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a h1:DG/Rx1VnnaqyPhKoPFuU61p4N7lkF5//weoP7QwddNs=
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a/go.mod h1:vgWZ7cu0fq0KY3PpEHsocXOWJpRtkcbKemU4IUw0M60=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
Expand Down Expand Up @@ -131,6 +135,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
Expand Down Expand Up @@ -302,6 +307,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
Expand Down
10 changes: 10 additions & 0 deletions query/common_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,16 @@ func processQueryNoErr(t *testing.T, query string) string {
return res
}

// processQueryForMetrics works like processQuery but returns metrics instead of response.
func processQueryForMetrics(t *testing.T, query string) *api.Metrics {
txn := client.NewTxn()
defer txn.Discard(context.Background())

res, err := txn.Query(context.Background(), query)
require.NoError(t, err)
return res.Metrics
}

func processQueryWithVars(t *testing.T, query string,
vars map[string]string) (string, error) {
txn := client.NewTxn()
Expand Down
1 change: 1 addition & 0 deletions query/outputnode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,7 @@ func processNodeUids(fj *fastJsonNode, sg *SubGraph) error {
type Extensions struct {
Latency *api.Latency `json:"server_latency,omitempty"`
Txn *api.TxnContext `json:"txn,omitempty"`
Metrics *api.Metrics `json:"metrics,omitempty"`
}

func (sg *SubGraph) toFastJSON(l *Latency) ([]byte, error) {
Expand Down
27 changes: 27 additions & 0 deletions query/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2699,6 +2699,7 @@ type ExecutionResult struct {
Subgraphs []*SubGraph
SchemaNode []*pb.SchemaNode
Types []*pb.TypeUpdate
Metrics map[string]uint64
}

// Process handles a query request.
Expand All @@ -2708,6 +2709,12 @@ func (req *Request) Process(ctx context.Context) (er ExecutionResult, err error)
return er, err
}
er.Subgraphs = req.Subgraphs
// calculate metrics.
metrics := make(map[string]uint64)
for _, sg := range er.Subgraphs {
calculateMetrics(sg, metrics)
}
er.Metrics = metrics

schemaProcessingStart := time.Now()
if req.GqlQuery.Schema != nil {
Expand All @@ -2733,3 +2740,23 @@ func StripBlankNode(mp map[string]uint64) map[string]uint64 {
}
return temp
}

// calculateMetrics populates the given map with the number of uids are gathered for each
// attributes.
func calculateMetrics(sg *SubGraph, metrics map[string]uint64) {

// skip internal nodes.
if !sg.IsInternal() {
// we'll calculate srcUid of the each attribute. because, these are number of uids
// processed by this attribute.
metrics[sg.Attr] = metrics[sg.Attr] + uint64(len(sg.SrcUIDs.GetUids()))
}
// add all the uids gathered by filters
for _, filter := range sg.Filters {
calculateMetrics(filter, metrics)
}
// calculate metrics for the children as well.
for _, child := range sg.Children {
calculateMetrics(child, metrics)
}
}
14 changes: 14 additions & 0 deletions query/query4_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1158,3 +1158,17 @@ func TestCountUIDNestedMultiple(t *testing.T) {
}
}`, js)
}

func TestNumUids(t *testing.T) {
query := `{
me(func:has(name), first:10){
name
friend{
name
}
}
}`
metrics := processQueryForMetrics(t, query)
require.Equal(t, metrics.NumUids["friend"], uint64(10))
require.Equal(t, metrics.NumUids["name"], uint64(16))
}

0 comments on commit ab8984b

Please sign in to comment.