Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add api_method parameter to Client.query to select INSERT or QUERY API #967

Merged
merged 25 commits into from
Dec 2, 2021
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
575966d
feat: add `api_method` parameter to `Client.query` to select `insert`…
tswast Sep 13, 2021
094e3cb
WIP: begin implementation of jobs.query usage
tswast Sep 13, 2021
17994f5
remove extra files
tswast Oct 5, 2021
c963e25
insert query with jobs.query
tswast Oct 6, 2021
5093378
fix merge between job config and query request
tswast Oct 6, 2021
e2e1c7d
add tests
tswast Oct 6, 2021
2e4af92
update todo with thoughts on future perf update
tswast Oct 7, 2021
d700df5
clarify TODO comment
tswast Oct 7, 2021
1e73a56
add placeholders for needed tests
tswast Oct 8, 2021
7435c8d
add schema property
tswast Oct 11, 2021
d383847
feat: add `QueryJob.schema` property for dry run queries
tswast Oct 11, 2021
8bc2458
add more job properties
tswast Oct 13, 2021
a2b4c2b
add tests for differences in API error behavior between jobs.query an…
tswast Oct 14, 2021
8b970f2
update docs to show differences
tswast Oct 14, 2021
e7e5e17
cover error conversion
tswast Oct 14, 2021
b572188
restore missing modules
tswast Oct 14, 2021
7bb1200
add unit tests
tswast Oct 15, 2021
7fbf966
Merge remote-tracking branch 'upstream/v3' into issue589-queries-endp…
tswast Nov 18, 2021
0598ace
adjust query job construction
tswast Nov 19, 2021
4f36bae
avoid conflicting table IDs
tswast Nov 19, 2021
3058498
mock query response
tswast Nov 19, 2021
ba785d9
fix unit test coverage
tswast Nov 19, 2021
b67ac5a
fix type errors
tswast Nov 19, 2021
a3223b1
fix docs formatting
tswast Nov 19, 2021
b257609
comments and additional unit tests
tswast Nov 22, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix merge between job config and query request
tswast committed Oct 7, 2021
commit 50933786417ce42b9b48b73b3bf22ed833cc02a3
5 changes: 1 addition & 4 deletions google/cloud/bigquery/_job_helpers.py
Original file line number Diff line number Diff line change
@@ -123,9 +123,7 @@ def _to_query_request(job_config: Optional[job.QueryJobConfig]) -> Dict[str, Any
"""
request_body = {}
job_config_resource = job_config.to_api_repr() if job_config else {}
query_config_resource = job_config_resource.get("configuration", {}).get(
"query", {}
)
query_config_resource = job_config_resource.get("query", {})

request_body.update(query_config_resource)

@@ -144,7 +142,6 @@ def _to_query_request(job_config: Optional[job.QueryJobConfig]) -> Dict[str, Any
def _to_query_job(
client: "Client", query: str, query_response: Dict[str, Any]
) -> job.QueryJob:
# TODO: check for errors?
job_ref_resource = query_response["jobReference"]
job_ref = job._JobReference._from_api_repr(job_ref_resource)
query_job = job.QueryJob(job_ref, query, client=client)
34 changes: 9 additions & 25 deletions google/cloud/bigquery/client.py
Original file line number Diff line number Diff line change
@@ -61,6 +61,7 @@
from google.cloud.bigquery.dataset import Dataset
from google.cloud.bigquery.dataset import DatasetListItem
from google.cloud.bigquery.dataset import DatasetReference
from google.cloud.bigquery import enums
from google.cloud.bigquery.enums import AutoRowIDs
from google.cloud.bigquery.opentelemetry_tracing import create_span
from google.cloud.bigquery import job
@@ -3124,7 +3125,7 @@ def query(
retry: retries.Retry = DEFAULT_RETRY,
timeout: float = DEFAULT_TIMEOUT,
job_retry: retries.Retry = DEFAULT_JOB_RETRY,
api_method: str = "insert",
api_method: enums.QueryApiMethod = enums.QueryApiMethod.INSERT,
) -> job.QueryJob:
"""Run a SQL query.

@@ -3177,19 +3178,7 @@ def query(
specified here becomes the default ``job_retry`` for
``result()``, where it can also be specified.
api_method:
One of ``'insert'`` or ``'query'``. Defaults to ``'insert'``.

When set to ``'insert'``, submit a query job by using the
`jobs.insert REST API method
<https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert>_`.
This supports all job configuration options.

When set to ``'query'``, submit a query job by using the
`jobs.query REST API method
<https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query>`_.
This API waits up to the specified timeout for the query to
finish. The ``job_id`` and ``job_id_prefix`` parameters cannot
be used with this API method.
Method with which to start the query job.

Returns:
google.cloud.bigquery.job.QueryJob: A new query job instance.
@@ -3213,15 +3202,9 @@ def query(
" provided."
)

if api_method not in {"insert", "query"}:
raise ValueError(
f"Got unexpected value for api_method: {repr(api_method)}"
" Expected one of {'insert', 'query'}."
)

if job_id_given and api_method == "query":
if job_id_given and api_method == enums.QueryApiMethod.QUERY:
raise TypeError(
"`job_id` was provided, but the 'query' `api_method` was requested."
"`job_id` was provided, but the 'QUERY' `api_method` was requested."
)

if project is None:
@@ -3253,12 +3236,11 @@ def query(

# Note that we haven't modified the original job_config (or
# _default_query_job_config) up to this point.
if api_method == "query":
# TODO: error if job_id or job_id_prefix set
if api_method == enums.QueryApiMethod.QUERY:
return _job_helpers.query_jobs_query(
self, query, job_config, location, project, retry, timeout, job_retry,
)
else:
elif api_method == enums.QueryApiMethod.INSERT:
return _job_helpers.query_jobs_insert(
self,
query,
@@ -3271,6 +3253,8 @@ def query(
timeout,
job_retry,
)
else:
raise ValueError(f"Got unexpected value for api_method: {repr(api_method)}")

def insert_rows(
self,
28 changes: 28 additions & 0 deletions google/cloud/bigquery/enums.py
Original file line number Diff line number Diff line change
@@ -122,6 +122,34 @@ class QueryPriority(object):
"""Specifies batch priority."""


class QueryApiMethod(str, enum.Enum):
"""API method used to start the query. The default value is
:attr:`INSERT`.
"""

INSERT = "INSERT"
"""Submit a query job by using the `jobs.insert REST API method
<https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/insert>_`.

This supports all job configuration options.
"""

QUERY = "QUERY"
"""Submit a query job by using the `jobs.query REST API method
<https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/query>`_.

This API blocks for up to a specified timeout for the query to finish. The
full job resource (including job statistics) may not be available if the
query finishes within the timeout. Call
:meth:`~google.cloud.bigquery.job.QueryJob.reload` or
:meth:`~google.cloud.bigquery.client.Client.get_job` to get full job
statistics.

Many parameters, including destination table and job ID cannot be used with
this API method.
"""


class SchemaUpdateOption(object):
"""Specifies an update to the destination table schema as a side effect of
a load job.
Loading