From c04be44d6603eb7a5ca5ed222351b5af8ef3c061 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Jul 2019 16:00:47 +0200 Subject: [PATCH 1/4] Add method signature compatibility reminders --- bigquery/google/cloud/bigquery/job.py | 4 ++++ bigquery/google/cloud/bigquery/table.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 442420a7191b..670cc54cc6b9 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -2896,6 +2896,8 @@ def result(self, timeout=None, page_size=None, retry=DEFAULT_RETRY): rows._preserve_order = _contains_order_by(self.query) return rows + # If changing the signature of this method, make sure to apply the same + # changes to table.RowIterator.to_arrow() def to_arrow(self, progress_bar_type=None): """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -2934,6 +2936,8 @@ def to_arrow(self, progress_bar_type=None): """ return self.result().to_arrow(progress_bar_type=progress_bar_type) + # If changing the signature of this method, make sure to apply the same + # changes to table.RowIterator.to_dataframe() def to_dataframe(self, bqstorage_client=None, dtypes=None, progress_bar_type=None): """Return a pandas DataFrame from a QueryJob diff --git a/bigquery/google/cloud/bigquery/table.py b/bigquery/google/cloud/bigquery/table.py index 3c8ce6cc39d7..5b1de345c068 100644 --- a/bigquery/google/cloud/bigquery/table.py +++ b/bigquery/google/cloud/bigquery/table.py @@ -1449,6 +1449,8 @@ def _to_arrow_iterable(self, bqstorage_client=None): bqstorage_client=bqstorage_client, ) + # If changing the signature of this method, make sure to apply the same + # changes to job.QueryJob.to_arrow() def to_arrow(self, progress_bar_type=None, bqstorage_client=None): """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -1552,6 +1554,8 @@ def _to_dataframe_iterable(self, bqstorage_client=None, dtypes=None): bqstorage_client=bqstorage_client, ) + # If changing the signature of this method, make sure to apply the same + # changes to job.QueryJob.to_dataframe() def to_dataframe(self, bqstorage_client=None, dtypes=None, progress_bar_type=None): """Create a pandas DataFrame by loading all pages of a query. From 7d2f3d5d8a93e9fed5191d833634c9d291e31ed5 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Jul 2019 16:27:31 +0200 Subject: [PATCH 2/4] Add bqstorage_client param to QueryJob.to_arrow() --- bigquery/google/cloud/bigquery/job.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/bigquery/google/cloud/bigquery/job.py b/bigquery/google/cloud/bigquery/job.py index 670cc54cc6b9..5cd90ada9637 100644 --- a/bigquery/google/cloud/bigquery/job.py +++ b/bigquery/google/cloud/bigquery/job.py @@ -2898,7 +2898,7 @@ def result(self, timeout=None, page_size=None, retry=DEFAULT_RETRY): # If changing the signature of this method, make sure to apply the same # changes to table.RowIterator.to_arrow() - def to_arrow(self, progress_bar_type=None): + def to_arrow(self, progress_bar_type=None, bqstorage_client=None): """[Beta] Create a class:`pyarrow.Table` by loading all pages of a table or query. @@ -2921,6 +2921,18 @@ def to_arrow(self, progress_bar_type=None): ``'tqdm_gui'`` Use the :func:`tqdm.tqdm_gui` function to display a progress bar as a graphical dialog box. + bqstorage_client ( \ + google.cloud.bigquery_storage_v1beta1.BigQueryStorageClient \ + ): + **Beta Feature** Optional. A BigQuery Storage API client. If + supplied, use the faster BigQuery Storage API to fetch rows + from BigQuery. This API is a billable API. + + This method requires the ``pyarrow`` and + ``google-cloud-bigquery-storage`` libraries. + + Reading from a specific partition or snapshot is not + currently supported by this method. Returns: pyarrow.Table @@ -2934,7 +2946,9 @@ def to_arrow(self, progress_bar_type=None): ..versionadded:: 1.17.0 """ - return self.result().to_arrow(progress_bar_type=progress_bar_type) + return self.result().to_arrow( + progress_bar_type=progress_bar_type, bqstorage_client=bqstorage_client + ) # If changing the signature of this method, make sure to apply the same # changes to table.RowIterator.to_dataframe() From a5d425eda7eb3269311f7248d7c96fd6bccc72d3 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Jul 2019 16:50:36 +0200 Subject: [PATCH 3/4] Add test for to_*() method signature compatibility The method signatures for to_arrow() and to_dataframe() methods in the job.QueryJob and table.RowIterator classes must match to present a consistent API for users. --- .../unit/test_signature_compatibility.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 bigquery/tests/unit/test_signature_compatibility.py diff --git a/bigquery/tests/unit/test_signature_compatibility.py b/bigquery/tests/unit/test_signature_compatibility.py new file mode 100644 index 000000000000..e5016b0e59bc --- /dev/null +++ b/bigquery/tests/unit/test_signature_compatibility.py @@ -0,0 +1,43 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect + +import pytest + + +@pytest.fixture +def query_job_class(): + from google.cloud.bigquery.job import QueryJob + + return QueryJob + + +@pytest.fixture +def row_iterator_class(): + from google.cloud.bigquery.table import RowIterator + + return RowIterator + + +def test_to_arrow_method_signatures_match(query_job_class, row_iterator_class): + sig = inspect.signature(query_job_class.to_arrow) + sig2 = inspect.signature(row_iterator_class.to_arrow) + assert sig == sig2 + + +def test_to_dataframe_method_signatures_match(query_job_class, row_iterator_class): + sig = inspect.signature(query_job_class.to_dataframe) + sig2 = inspect.signature(row_iterator_class.to_dataframe) + assert sig == sig2 From 535e1dd2bbcc55135fce2dd77e733d827068fb51 Mon Sep 17 00:00:00 2001 From: Peter Lamut Date: Wed, 17 Jul 2019 17:19:49 +0200 Subject: [PATCH 4/4] Skip method signature test in old Pythons inspect.signature() method is only available in older Python versions --- bigquery/tests/unit/test_signature_compatibility.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bigquery/tests/unit/test_signature_compatibility.py b/bigquery/tests/unit/test_signature_compatibility.py index e5016b0e59bc..6002ae3e87c9 100644 --- a/bigquery/tests/unit/test_signature_compatibility.py +++ b/bigquery/tests/unit/test_signature_compatibility.py @@ -31,12 +31,20 @@ def row_iterator_class(): return RowIterator +@pytest.mark.skipif( + not hasattr(inspect, "signature"), + reason="inspect.signature() is not availalbe in older Python versions", +) def test_to_arrow_method_signatures_match(query_job_class, row_iterator_class): sig = inspect.signature(query_job_class.to_arrow) sig2 = inspect.signature(row_iterator_class.to_arrow) assert sig == sig2 +@pytest.mark.skipif( + not hasattr(inspect, "signature"), + reason="inspect.signature() is not availalbe in older Python versions", +) def test_to_dataframe_method_signatures_match(query_job_class, row_iterator_class): sig = inspect.signature(query_job_class.to_dataframe) sig2 = inspect.signature(row_iterator_class.to_dataframe)