Skip to content

Commit

Permalink
feat(datastore): SUM and AVG aggregations (#8307)
Browse files Browse the repository at this point in the history 
* feat(datastore): SUM and AVG aggregations

* feat(datastore): Fixing integration tests for SUM AVG

* feat(datastore): Fixing integration tests

* feat(datastore): Fixing integration tests

* feat(datastore): Updating protos

* feat(datastore): updating protos

* feat(datastore): Undo go.work.sum changes

* feat(datastore): Used new protos

* feat(datastore): Use latest protos

---------

Co-authored-by: meredithslota <meredithslota@google.com>
Co-authored-by: kolea2 <45548808+kolea2@users.noreply.github.com>
  • Loading branch information
@bhshkh @meredithslota @kolea2
3 people authored Aug 22, 2023
1 parent fa6e827 commit a9fff18
Show file tree
Hide file tree
Showing 5 changed files with 174 additions and 41 deletions.
15 changes: 8 additions & 7 deletions datastore/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,20 @@ module cloud.google.com/go/datastore
go 1.19

require (
cloud.google.com/go v0.110.2
cloud.google.com/go/longrunning v0.5.0
cloud.google.com/go v0.110.7
cloud.google.com/go/longrunning v0.5.1
github.com/golang/protobuf v1.5.3
github.com/google/go-cmp v0.5.9
github.com/googleapis/gax-go/v2 v2.12.0
google.golang.org/api v0.128.0
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc
google.golang.org/grpc v1.56.1
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5
google.golang.org/grpc v1.57.0
google.golang.org/protobuf v1.31.0
)

require (
cloud.google.com/go/compute v1.19.3 // indirect
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.4 // indirect
Expand All @@ -25,9 +25,10 @@ require (
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect
)
29 changes: 15 additions & 14 deletions datastore/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA=
cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw=
cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds=
cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI=
cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o=
cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI=
cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/longrunning v0.5.0 h1:DK8BH0+hS+DIvc9a2TPnteUievsTCH4ORMAASSb7JcQ=
cloud.google.com/go/longrunning v0.5.0/go.mod h1:0JNuqRShmscVAhIACGtskSAWtqtOoPkwP0YF1oVEchc=
cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI=
cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
Expand Down Expand Up @@ -118,6 +118,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -162,12 +163,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM=
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93 h1:zv6ieVm8jNcN33At1+APsRISkRgynuWUxUhv6G123jY=
google.golang.org/genproto v0.0.0-20230821184602-ccc8af3d0e93/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44=
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
Expand All @@ -176,8 +177,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
Expand Down
111 changes: 92 additions & 19 deletions datastore/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"
)

// TODO(djd): Make test entity clean up more robust: some test entities may
Expand All @@ -52,6 +53,7 @@ var suffix string
const (
replayFilename = "datastore.replay"
envDatabases = "GCLOUD_TESTS_GOLANG_DATASTORE_DATABASES"
keyPrefix = "TestIntegration_"
)

type replayInfo struct {
Expand Down Expand Up @@ -471,6 +473,8 @@ func TestIntegration_NilKey(t *testing.T) {
type SQChild struct {
I, J int
T, U int64
V float64
W string
}

type SQTestCase struct {
Expand Down Expand Up @@ -701,17 +705,17 @@ func TestIntegration_AggregationQueries(t *testing.T) {
client := newTestClient(ctx, t)
defer client.Close()

parent := NameKey("SQParent", "TestIntegration_Filters"+suffix, nil)
parent := NameKey("SQParent", keyPrefix+"AggregationQueries"+suffix, nil)
now := timeNow.Truncate(time.Millisecond).Unix()
children := []*SQChild{
{I: 0, T: now, U: now},
{I: 1, T: now, U: now},
{I: 2, T: now, U: now},
{I: 3, T: now, U: now},
{I: 4, T: now, U: now},
{I: 5, T: now, U: now},
{I: 6, T: now, U: now},
{I: 7, T: now, U: now},
{I: 0, T: now, U: now, V: 1.5, W: "str"},
{I: 1, T: now, U: now, V: 1.5, W: "str"},
{I: 2, T: now, U: now, V: 1.5, W: "str"},
{I: 3, T: now, U: now, V: 1.5, W: "str"},
{I: 4, T: now, U: now, V: 1.5, W: "str"},
{I: 5, T: now, U: now, V: 1.5, W: "str"},
{I: 6, T: now, U: now, V: 1.5, W: "str"},
{I: 7, T: now, U: now, V: 1.5, W: "str"},
}

keys := make([]*Key, len(children))
Expand All @@ -729,7 +733,6 @@ func TestIntegration_AggregationQueries(t *testing.T) {
}
}()

baseQuery := NewQuery("SQChild").Ancestor(parent)
testCases := []struct {
desc string
aggQuery *AggregationQuery
Expand All @@ -738,21 +741,91 @@ func TestIntegration_AggregationQueries(t *testing.T) {
wantAggResult AggregationResult
}{
{
desc: "Count Failure - Missing index",
aggQuery: baseQuery.Filter("T>=", now).NewAggregationQuery().WithCount("count"),
wantFailure: true,
wantErrMsg: "no matching index found",
wantAggResult: nil,
desc: "Count Failure - Missing index",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T>=", now).
NewAggregationQuery().
WithCount("count"),
wantFailure: true,
wantErrMsg: "no matching index found",
},
{
desc: "Count Success",
aggQuery: baseQuery.Filter("T=", now).Filter("I>=", 3).NewAggregationQuery().WithCount("count"),
wantFailure: false,
wantErrMsg: "",
desc: "Count Success",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Filter("I>=", 3).
NewAggregationQuery().
WithCount("count"),
wantAggResult: map[string]interface{}{
"count": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 5}},
},
},
{
desc: "Multiple aggregations",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
NewAggregationQuery().
WithSum("I", "i_sum").
WithAvg("I", "avg").
WithSum("V", "v_sum"),
wantAggResult: map[string]interface{}{
"i_sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}},
"v_sum": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 12}},
"avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}},
},
},
{
desc: "Multiple aggregations with limit ",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2).
NewAggregationQuery().
WithSum("I", "sum").
WithAvg("I", "avg"),
wantAggResult: map[string]interface{}{
"sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 1}},
"avg": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 0.5}},
},
},
{
desc: "Multiple aggregations on non-numeric field",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).Limit(2).
NewAggregationQuery().
WithSum("W", "sum").
WithAvg("W", "avg"),
wantAggResult: map[string]interface{}{
"sum": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: int64(0)}},
"avg": &pb.Value{ValueType: &pb.Value_NullValue{NullValue: structpb.NullValue_NULL_VALUE}},
},
},
{
desc: "Sum aggregation without alias",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
NewAggregationQuery().
WithSum("I", ""),
wantAggResult: map[string]interface{}{
"property_1": &pb.Value{ValueType: &pb.Value_IntegerValue{IntegerValue: 28}},
},
},
{
desc: "Average aggregation without alias",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
NewAggregationQuery().
WithAvg("I", ""),
wantAggResult: map[string]interface{}{
"property_1": &pb.Value{ValueType: &pb.Value_DoubleValue{DoubleValue: 3.5}},
},
},
{
desc: "Sum aggregation on '__key__'",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
NewAggregationQuery().
WithSum("__key__", ""),
wantFailure: true,
wantErrMsg: "Aggregations are not supported for the property",
},
{
desc: "Average aggregation on '__key__'",
aggQuery: NewQuery("SQChild").Ancestor(parent).Filter("T=", now).
NewAggregationQuery().
WithAvg("__key__", ""),
wantFailure: true,
wantErrMsg: "Aggregations are not supported for the property",
},
}

for _, testCase := range testCases {
Expand Down
44 changes: 44 additions & 0 deletions datastore/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1057,5 +1057,49 @@ func (aq *AggregationQuery) WithCount(alias string) *AggregationQuery {
return aq
}

// WithSum specifies that the aggregation query should provide a sum of the values
// of the provided field in the results returned by the underlying Query.
// The alias argument can be empty or a valid Datastore entity property name. It can be used
// as key in the AggregationResult to get the sum value. If alias is empty, Datastore
// will autogenerate a key.
func (aq *AggregationQuery) WithSum(fieldName string, alias string) *AggregationQuery {
aqpb := &pb.AggregationQuery_Aggregation{
Alias: alias,
Operator: &pb.AggregationQuery_Aggregation_Sum_{
Sum: &pb.AggregationQuery_Aggregation_Sum{
Property: &pb.PropertyReference{
Name: fieldName,
},
},
},
}

aq.aggregationQueries = append(aq.aggregationQueries, aqpb)

return aq
}

// WithAvg specifies that the aggregation query should provide an average of the values
// of the provided field in the results returned by the underlying Query.
// The alias argument can be empty or a valid Datastore entity property name. It can be used
// as key in the AggregationResult to get the sum value. If alias is empty, Datastore
// will autogenerate a key.
func (aq *AggregationQuery) WithAvg(fieldName string, alias string) *AggregationQuery {
aqpb := &pb.AggregationQuery_Aggregation{
Alias: alias,
Operator: &pb.AggregationQuery_Aggregation_Avg_{
Avg: &pb.AggregationQuery_Aggregation_Avg{
Property: &pb.PropertyReference{
Name: fieldName,
},
},
},
}

aq.aggregationQueries = append(aq.aggregationQueries, aqpb)

return aq
}

// AggregationResult contains the results of an aggregation query.
type AggregationResult map[string]interface{}
16 changes: 15 additions & 1 deletion datastore/testdata/index.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,18 @@ indexes:
properties:
- name: T
- name: J
- name: U
- name: U

- kind: SQChild
ancestor: yes
properties:
- name: T
- name: I
- name: V

- kind: SQChild
ancestor: yes
properties:
- name: T
- name: W

0 comments on commit a9fff18

Please sign in to comment.