From 979c7bab28bc35004edb2b5febcb7add5c29d011 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Thu, 5 Oct 2023 20:47:30 +0000 Subject: [PATCH 1/4] feat(bigquery): introduce query preview features This PR introduces the ability for users of the bigquery client library to indicate the library should enable preview features. The first preview feature enabled here is the stateless query feature, a mechanism for running small queries without corresponding job metadata. Ability to use this feature is governed by the service itself, and will simply create jobs in cases where the feature isn't enable in either the client or service, or a query cannot be satisfied in a stateless manner. --- bigquery/bigquery.go | 22 +++++++++++++++++-- bigquery/go.mod | 26 +++++++++++----------- bigquery/go.sum | 52 ++++++++++++++++++++++---------------------- bigquery/query.go | 16 +++++++++----- go.work.sum | 7 +++--- 5 files changed, 73 insertions(+), 50 deletions(-) diff --git a/bigquery/bigquery.go b/bigquery/bigquery.go index 94720a912784..c597679bc70b 100644 --- a/bigquery/bigquery.go +++ b/bigquery/bigquery.go @@ -21,6 +21,7 @@ import ( "io" "net/http" "net/url" + "os" "strings" "time" @@ -59,6 +60,9 @@ type Client struct { projectID string bqs *bq.Service rc *readClient + + // governs use of preview query features. + enableQueryPreview bool } // DetectProjectID is a sentinel value that instructs NewClient to detect the @@ -75,6 +79,12 @@ const DetectProjectID = "*detect-project-id*" // // If the project ID is set to DetectProjectID, NewClient will attempt to detect // the project ID from credentials. +// +// This client supports enabling query-related preview features via environmental +// variables. By setting the environment variable QUERY_PREVIEW_ENABLED to the string +// "TRUE", the client will enable preview features, though behavior may still be +// controlled via the bigquery service as well. Currently, the feature(s) in scope +// include: stateless queries (query execution without corresponding job metadata). func NewClient(ctx context.Context, projectID string, opts ...option.ClientOption) (*Client, error) { o := []option.ClientOption{ option.WithScopes(Scope), @@ -92,9 +102,17 @@ func NewClient(ctx context.Context, projectID string, opts ...option.ClientOptio return nil, err } + var preview bool + if v, ok := os.LookupEnv("QUERY_PREVIEW_ENABLED"); ok { + if strings.ToUpper(v) == "TRUE" { + preview = true + } + } + c := &Client{ - projectID: projectID, - bqs: bqs, + projectID: projectID, + bqs: bqs, + enableQueryPreview: preview, } return c, nil } diff --git a/bigquery/go.mod b/bigquery/go.mod index 18721ab6f760..c83e0e64c148 100644 --- a/bigquery/go.mod +++ b/bigquery/go.mod @@ -3,23 +3,23 @@ module cloud.google.com/go/bigquery go 1.19 require ( - cloud.google.com/go v0.110.6 + cloud.google.com/go v0.110.7 cloud.google.com/go/datacatalog v1.16.0 cloud.google.com/go/iam v1.1.1 cloud.google.com/go/longrunning v0.5.1 cloud.google.com/go/storage v1.30.1 github.com/apache/arrow/go/v12 v12.0.0 github.com/google/go-cmp v0.5.9 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/googleapis/gax-go/v2 v2.12.0 go.opencensus.io v0.24.0 golang.org/x/sync v0.3.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 - google.golang.org/api v0.139.0 - google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 - google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d - google.golang.org/grpc v1.57.0 + google.golang.org/api v0.145.0 + google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb + google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb + google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 + google.golang.org/grpc v1.58.2 google.golang.org/protobuf v1.31.0 ) @@ -35,7 +35,7 @@ require ( github.com/google/flatbuffers v2.0.8+incompatible // indirect github.com/google/martian/v3 v3.3.2 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.15.9 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect @@ -44,12 +44,12 @@ require ( github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/stretchr/testify v1.8.2 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - golang.org/x/crypto v0.12.0 // indirect + golang.org/x/crypto v0.13.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect ) diff --git a/bigquery/go.sum b/bigquery/go.sum index 9285923b7e10..25a54095ab68 100644 --- a/bigquery/go.sum +++ b/bigquery/go.sum @@ -1,6 +1,6 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= -cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= @@ -75,10 +75,10 @@ github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= +github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= @@ -113,8 +113,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -132,11 +132,11 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -146,13 +146,13 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -167,8 +167,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -google.golang.org/api v0.139.0 h1:A1TrCPgMmOiYu0AiNkvQIpIx+D8blHTDcJ5EogkP7LI= -google.golang.org/api v0.139.0/go.mod h1:CVagp6Eekz9CjGZ718Z+sloknzkDJE7Vc1Ckj9+viBk= +google.golang.org/api v0.145.0 h1:kBjvf1A3/m30kUvnUX9jZJxTu3lJrpGFt5V/1YZrjwg= +google.golang.org/api v0.145.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= @@ -176,20 +176,20 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 h1:nIgk/EEq3/YlnmVVXVnm14rC2oxgs1o0ong4sD/rd44= -google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5/go.mod h1:5DZzOUPCLYL3mNkQ0ms0F3EuUNZ7py1Bqeq6sxzI7/Q= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= +google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 h1:N3bU/SQDCDyD6R528GJ/PwW9KjYcJA3dgyH+MovAkIM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:KSqppvjFjtoCI+KGd4PELB0qLNxdJHRGqRI09mB6pQA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/bigquery/query.go b/bigquery/query.go index 7030cfd50c4f..a44e93227d72 100644 --- a/bigquery/query.go +++ b/bigquery/query.go @@ -393,11 +393,14 @@ func (q *Query) Read(ctx context.Context) (it *RowIterator, err error) { } // construct a minimal job for backing the row iterator. - minimalJob := &Job{ - c: q.client, - jobID: resp.JobReference.JobId, - location: resp.JobReference.Location, - projectID: resp.JobReference.ProjectId, + var minimalJob *Job + if resp.JobReference != nil { + minimalJob = &Job{ + c: q.client, + jobID: resp.JobReference.JobId, + location: resp.JobReference.Location, + projectID: resp.JobReference.ProjectId, + } } if resp.JobComplete { @@ -484,6 +487,9 @@ func (q *Query) probeFastPath() (*bq.QueryRequest, error) { DatasetId: q.QueryConfig.DefaultDatasetID, } } + if q.client.enableQueryPreview { + qRequest.JobCreationMode = "JOB_CREATION_OPTIONAL" + } return qRequest, nil } diff --git a/go.work.sum b/go.work.sum index e16396bdfd61..6b4eec19c54b 100644 --- a/go.work.sum +++ b/go.work.sum @@ -10,6 +10,7 @@ github.com/elazarl/goproxy v0.0.0-20221015165544-a0805db90819/go.mod h1:Ro8st/El github.com/envoyproxy/protoc-gen-validate v1.0.1/go.mod h1:0vj8bNkYbSTNS2PIyH87KZaeN4x9zpL9Qt8fQC7d+vs= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= +github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoGcuNJpa7jnd5pMGdkSaQpNDYvPlXWMcjXXThLlY= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/googleapis/gax-go/v2 v2.9.1/go.mod h1:4FG3gMrVZlyMp5itSYKMU9z/lBE7+SbnUOvzH2HqbEY= github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= @@ -23,9 +24,9 @@ golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.10.0 h1:tvDr/iQoUqNdohiYm0LmmKcBk+q86lb9EprIUFhHHGg= golang.org/x/tools v0.10.0/go.mod h1:UJwyiVBsOA2uwvK/e5OY3GTpDUJriEd+/YlqAwLPmyM= google.golang.org/api v0.123.0/go.mod h1:gcitW0lvnyWjSp9nKxAbdHKIZ6vF4aajGueeslZOyms= @@ -34,8 +35,6 @@ google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02Oq google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230629202037-9506855d4529/go.mod h1:ylj+BE99M198VPbBh6A8d9n3w8fChvyLK3wwBOjXBFA= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230720185612-659f7aaaa771/go.mod h1:3QoBVwTHkXbY1oRGzlhwhOykfcATQN43LJ6iT8Wy8kE= -google.golang.org/genproto/googleapis/bytestream v0.0.0-20230807174057-1744710a1577/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= google.golang.org/genproto/googleapis/bytestream v0.0.0-20230911183012-2d3300fd4832/go.mod h1:NjCQG/D8JandXxM57PZbAJL1DCNL6EypA0vPPwfsc7c= +google.golang.org/genproto/googleapis/bytestream v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:qDbnxtViX5J6CvFbxeNUSzKgVlDLJ/6L+caxye9+Flo= google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= From ce1bcf5e804c5c7c9046a3cb7d15b203f3b85a73 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Mon, 23 Oct 2023 17:05:46 +0000 Subject: [PATCH 2/4] augment testing --- bigquery/integration_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/bigquery/integration_test.go b/bigquery/integration_test.go index 64ea202a5923..785ef980224c 100644 --- a/bigquery/integration_test.go +++ b/bigquery/integration_test.go @@ -806,6 +806,12 @@ func TestIntegration_SimpleRowResults(t *testing.T) { query: "select count(*) from unnest(generate_array(1,1000000)), unnest(generate_array(1, 1000)) as foo", want: [][]Value{{int64(1000000000)}}, }, + { + // Query doesn't yield a result. + description: "DML", + query: fmt.Sprintf("CREATE OR REPLACE TABLE %s.%s (foo STRING, bar INT64)", dataset.DatasetID, tableIDs.New()), + want: [][]Value{}, + }, } for _, tc := range testCases { curCase := tc @@ -3284,21 +3290,28 @@ func checkReadAndTotalRows(t *testing.T, msg string, it *RowIterator, want [][]V func compareRead(it *RowIterator, want [][]Value, compareTotalRows bool) (msg string, ok bool) { got, _, totalRows, err := readAll(it) + jobStr := "" + if it.SourceJob() != nil { + jobStr = it.SourceJob().jobID + } + if jobStr != "" { + jobStr = fmt.Sprintf("(Job: %s)", jobStr) + } if err != nil { return err.Error(), false } if len(got) != len(want) { - return fmt.Sprintf("got %d rows, want %d", len(got), len(want)), false + return fmt.Sprintf("%s got %d rows, want %d", len(got), len(want)), jobStr, false } if compareTotalRows && len(got) != int(totalRows) { - return fmt.Sprintf("got %d rows, but totalRows = %d", len(got), totalRows), false + return fmt.Sprintf("%s got %d rows, but totalRows = %d", jobStr, len(got), totalRows), false } sort.Sort(byCol0(got)) for i, r := range got { gotRow := []Value(r) wantRow := want[i] if !testutil.Equal(gotRow, wantRow) { - return fmt.Sprintf("#%d: got %#v, want %#v", i, gotRow, wantRow), false + return fmt.Sprintf("%s #%d: got %#v, want %#v", jobStr, i, gotRow, wantRow), false } } return "", true From 1c129a0e1a5ac9afa9ea7c066d877542cde804d3 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Mon, 23 Oct 2023 17:11:22 +0000 Subject: [PATCH 3/4] fix --- bigquery/integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery/integration_test.go b/bigquery/integration_test.go index 785ef980224c..4f01ba80fe74 100644 --- a/bigquery/integration_test.go +++ b/bigquery/integration_test.go @@ -3301,7 +3301,7 @@ func compareRead(it *RowIterator, want [][]Value, compareTotalRows bool) (msg st return err.Error(), false } if len(got) != len(want) { - return fmt.Sprintf("%s got %d rows, want %d", len(got), len(want)), jobStr, false + return fmt.Sprintf("%s got %d rows, want %d", jobStr, len(got), len(want)), false } if compareTotalRows && len(got) != int(totalRows) { return fmt.Sprintf("%s got %d rows, but totalRows = %d", jobStr, len(got), totalRows), false From cbe6a0e115aa9bd5baed63d166275837eb8369e8 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Mon, 23 Oct 2023 20:08:56 +0000 Subject: [PATCH 4/4] widen testing, deal with empty stateless result --- bigquery/integration_test.go | 51 +++++++++++++++++++++++++++--------- bigquery/iterator.go | 6 ++++- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/bigquery/integration_test.go b/bigquery/integration_test.go index 4f01ba80fe74..232333383da9 100644 --- a/bigquery/integration_test.go +++ b/bigquery/integration_test.go @@ -775,6 +775,11 @@ func TestIntegration_SimpleRowResults(t *testing.T) { if client == nil { t.Skip("Integration tests skipped") } + beforePreview := client.enableQueryPreview + // ensure we restore the preview setting on test exit + defer func() { + client.enableQueryPreview = beforePreview + }() ctx := context.Background() testCases := []struct { @@ -797,7 +802,7 @@ func TestIntegration_SimpleRowResults(t *testing.T) { // in the job config, but switching to relying on jobs.getQueryResults allows the // service to decide the behavior. description: "ctas ddl", - query: fmt.Sprintf("CREATE TABLE %s.%s AS SELECT 17 as foo", dataset.DatasetID, tableIDs.New()), + query: fmt.Sprintf("CREATE OR REPLACE TABLE %s.%s AS SELECT 17 as foo", dataset.DatasetID, tableIDs.New()), want: nil, }, { @@ -813,18 +818,38 @@ func TestIntegration_SimpleRowResults(t *testing.T) { want: [][]Value{}, }, } - for _, tc := range testCases { - curCase := tc - t.Run(curCase.description, func(t *testing.T) { - t.Parallel() - q := client.Query(curCase.query) - it, err := q.Read(ctx) - if err != nil { - t.Fatalf("%s read error: %v", curCase.description, err) - } - checkReadAndTotalRows(t, curCase.description, it, curCase.want) - }) - } + + t.Run("nopreview_group", func(t *testing.T) { + client.enableQueryPreview = false + for _, tc := range testCases { + curCase := tc + t.Run(curCase.description, func(t *testing.T) { + t.Parallel() + q := client.Query(curCase.query) + it, err := q.Read(ctx) + if err != nil { + t.Fatalf("%s read error: %v", curCase.description, err) + } + checkReadAndTotalRows(t, curCase.description, it, curCase.want) + }) + } + }) + t.Run("preview_group", func(t *testing.T) { + client.enableQueryPreview = true + for _, tc := range testCases { + curCase := tc + t.Run(curCase.description, func(t *testing.T) { + t.Parallel() + q := client.Query(curCase.query) + it, err := q.Read(ctx) + if err != nil { + t.Fatalf("%s read error: %v", curCase.description, err) + } + checkReadAndTotalRows(t, curCase.description, it, curCase.want) + }) + } + }) + } func TestIntegration_QueryIterationPager(t *testing.T) { diff --git a/bigquery/iterator.go b/bigquery/iterator.go index 37b12dea84d4..6339bd52b541 100644 --- a/bigquery/iterator.go +++ b/bigquery/iterator.go @@ -240,7 +240,11 @@ func fetchPage(ctx context.Context, src *rowSource, schema Schema, startIndex ui if src.j != nil { return fetchJobResultPage(ctx, src, schema, startIndex, pageSize, pageToken) } - return fetchTableResultPage(ctx, src, schema, startIndex, pageSize, pageToken) + if src.t != nil { + return fetchTableResultPage(ctx, src, schema, startIndex, pageSize, pageToken) + } + // No rows, but no table or job reference. Return an empty result set. + return &fetchPageResult{}, nil } return result, nil }