From 1f4ebb1eca4f9380a31172fc8cb2fae125f8c5a2 Mon Sep 17 00:00:00 2001 From: Kira Date: Tue, 28 Nov 2023 13:47:06 -0800 Subject: [PATCH] feat: removed pkg_resources from all test files and moved importlib into pandas extra (#1726) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Introduce compatibility with native namespace packages * Update copyright year * removed pkg_resources from all test files and moved importlib into pandas extra * feat: removed pkg_resources from all test files and moved importlib into pandas extra * Adding no cover tag to test code * reformatted with black * undo revert * perf: use the first page a results when `query(api_method="QUERY")` (#1723) * perf: use the first page a results when `query(api_method="QUERY")` * add tests * respect max_results with cached page * respect page_size, also avoid bqstorage if almost fully downloaded * skip true test if bqstorage not installed * coverage * fix: ensure query job retry has longer deadline than API request deadline (#1734) In cases where we can't disambiguate API failure from job failure, this ensures we can still retry the job at least once. * fix: `load_table_from_dataframe` now assumes there may be local null values (#1735) Even if the remote schema is REQUIRED Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #1692 🦕 * chore: standardize samples directory - delete unneeded dependencies (#1732) * chore: standardize samples directory = delete unneeded dependencies * Removed unused import for linter * fix: move grpc, proto-plus and protobuf packages to extras (#1721) * chore: move grpc, proto-plus and protobuff packages to extras * formatted with black * feat: add `job_timeout_ms` to job configuration classes (#1675) * fix: adds new property and tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * updates docs to correct a sphinx failure * Updates formatting * Update tests/system/test_query.py * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update google/cloud/bigquery/job/base.py * updates one test and uses int_or_none * Update tests/system/test_query.py testing something. * Update tests/system/test_query.py * testing coverage feature * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * minor edits * tweaks to noxfile for testing purposes * add new test to base as experiment * adds a test, updates import statements * add another test * edit to tests * formatting fixes * update noxfile to correct debug code * removes unneeded comments. --------- Co-authored-by: Owl Bot --------- Co-authored-by: Chalmer Lowe Co-authored-by: Owl Bot Co-authored-by: Tim Swast * remove unnecessary version checks * undo bad commit, remove unneeded version checks * Revert "undo bad commit, remove unneeded version checks" This reverts commit 5c82dcf85f5fd73ad37ab7805b85b88a65351801. * Revert "remove unnecessary version checks" This reverts commit 9331a7e034de1c651c5ebc454f38f602d612a03d. * revert bad changes, remove pkg_resources from file * after clarification, reimplement changes and ignore 3.12 tests * reformatted with black * removed minimum check * updated pandas installed version check --------- Co-authored-by: Anthonios Partheniou Co-authored-by: Tim Swast Co-authored-by: Chalmer Lowe Co-authored-by: Owl Bot --- .coveragerc | 5 ---- google/__init__.py | 24 ------------------- google/cloud/__init__.py | 24 ------------------- noxfile.py | 5 ++-- setup.py | 10 ++------ tests/system/test_pandas.py | 23 ++++++++---------- tests/unit/job/test_query_pandas.py | 14 ++++++----- tests/unit/test__pandas_helpers.py | 32 ++++++++----------------- tests/unit/test_client.py | 23 +++++++----------- tests/unit/test_packaging.py | 37 +++++++++++++++++++++++++++++ tests/unit/test_table.py | 18 +++++++------- tests/unit/test_table_pandas.py | 14 ++++++----- 12 files changed, 96 insertions(+), 133 deletions(-) delete mode 100644 google/__init__.py delete mode 100644 google/cloud/__init__.py create mode 100644 tests/unit/test_packaging.py diff --git a/.coveragerc b/.coveragerc index 1ed1a9704..04092257a 100644 --- a/.coveragerc +++ b/.coveragerc @@ -12,8 +12,3 @@ exclude_lines = pragma: NO COVER # Ignore debug-only repr def __repr__ - # Ignore pkg_resources exceptions. - # This is added at the module level as a safeguard for if someone - # generates the code and tries to run it without pip installing. This - # makes it virtually impossible to test properly. - except pkg_resources.DistributionNotFound diff --git a/google/__init__.py b/google/__init__.py deleted file mode 100644 index 8e60d8439..000000000 --- a/google/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import pkg_resources - - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - - __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/google/cloud/__init__.py b/google/cloud/__init__.py deleted file mode 100644 index 8e60d8439..000000000 --- a/google/cloud/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import pkg_resources - - pkg_resources.declare_namespace(__name__) -except ImportError: - import pkgutil - - __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/noxfile.py b/noxfile.py index 9ccbdd30c..246952728 100644 --- a/noxfile.py +++ b/noxfile.py @@ -137,7 +137,7 @@ def mypy(session): "types-requests", "types-setuptools", ) - session.run("mypy", "google/cloud", "--show-traceback") + session.run("mypy", "-p", "google", "--show-traceback") @nox.session(python=DEFAULT_PYTHON_VERSION) @@ -149,7 +149,8 @@ def pytype(session): session.install("attrs==20.3.0") session.install("-e", ".[all]") session.install(PYTYPE_VERSION) - session.run("pytype") + # See https://github.com/google/pytype/issues/464 + session.run("pytype", "-P", ".", "google/cloud/bigquery") @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) diff --git a/setup.py b/setup.py index ead602e12..f21bb586d 100644 --- a/setup.py +++ b/setup.py @@ -62,6 +62,7 @@ "pandas>=1.1.0", pyarrow_dependency, "db-dtypes>=0.3.0,<2.0.0dev", + "importlib_metadata>=1.0.0; python_version<'3.8'", ], "ipywidgets": [ "ipywidgets>=7.7.0", @@ -108,16 +109,10 @@ # benchmarks, etc. packages = [ package - for package in setuptools.PEP420PackageFinder.find() + for package in setuptools.find_namespace_packages() if package.startswith("google") ] -# Determine which namespaces are needed. -namespaces = ["google"] -if "google.cloud" in packages: - namespaces.append("google.cloud") - - setuptools.setup( name=name, version=version, @@ -143,7 +138,6 @@ ], platforms="Posix; MacOS X; Windows", packages=packages, - namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, python_requires=">=3.7", diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index 9f7fc242e..e93f245c0 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -23,9 +23,13 @@ import warnings import google.api_core.retry -import pkg_resources import pytest +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata + from google.cloud import bigquery from google.cloud.bigquery import enums @@ -42,11 +46,9 @@ ) if pandas is not None: - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") - -PANDAS_INT64_VERSION = pkg_resources.parse_version("1.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" class MissingDataError(Exception): @@ -310,10 +312,7 @@ def test_load_table_from_dataframe_w_automatic_schema(bigquery_client, dataset_i ] -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION, - reason="Only `pandas version >=1.0.0` is supported", -) +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_load_table_from_dataframe_w_nullable_int64_datatype( bigquery_client, dataset_id ): @@ -342,7 +341,7 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype( @pytest.mark.skipif( - PANDAS_INSTALLED_VERSION < PANDAS_INT64_VERSION, + PANDAS_INSTALLED_VERSION[0:2].startswith("0."), reason="Only `pandas version >=1.0.0` is supported", ) def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema( @@ -1043,9 +1042,7 @@ def test_list_rows_max_results_w_bqstorage(bigquery_client): assert len(dataframe.index) == 100 -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") @pytest.mark.parametrize( ("max_results",), ( diff --git a/tests/unit/job/test_query_pandas.py b/tests/unit/job/test_query_pandas.py index 0accae0a2..6189830ff 100644 --- a/tests/unit/job/test_query_pandas.py +++ b/tests/unit/job/test_query_pandas.py @@ -17,7 +17,6 @@ import json import mock -import pkg_resources import pytest @@ -45,14 +44,19 @@ except (ImportError, AttributeError): # pragma: NO COVER tqdm = None +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata + from ..helpers import make_connection from .helpers import _make_client from .helpers import _make_job_resource if pandas is not None: - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" pandas = pytest.importorskip("pandas") @@ -656,9 +660,7 @@ def test_to_dataframe_bqstorage_no_pyarrow_compression(): ) -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") @pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`") def test_to_dataframe_column_dtypes(): from google.cloud.bigquery.job import QueryJob as target_class diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index 1f1b4eeb3..ad40a6da6 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -19,7 +19,11 @@ import operator import queue import warnings -import pkg_resources + +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata import mock @@ -57,13 +61,10 @@ bigquery_storage = _versions_helpers.BQ_STORAGE_VERSIONS.try_import() -PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0") - if pandas is not None: - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: - # Set to less than MIN version. - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" skip_if_no_bignumeric = pytest.mark.skipif( @@ -542,9 +543,7 @@ def test_bq_to_arrow_array_w_nullable_scalars(module_under_test, bq_type, rows): ], ) @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows): rows = [pandas.Timestamp(row) for row in rows] @@ -806,10 +805,7 @@ def test_list_columns_and_indexes_with_named_index_same_as_column_name( assert columns_and_indexes == expected -@pytest.mark.skipif( - pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, - reason="Requires `pandas version >= 1.0.0` which introduces pandas.NA", -) +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_dataframe_to_json_generator(module_under_test): utcnow = datetime.datetime.utcnow() df_data = collections.OrderedDict( @@ -837,16 +833,8 @@ def test_dataframe_to_json_generator(module_under_test): assert list(rows) == expected +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_dataframe_to_json_generator_repeated_field(module_under_test): - pytest.importorskip( - "pandas", - minversion=str(PANDAS_MINIUM_VERSION), - reason=( - f"Requires `pandas version >= {PANDAS_MINIUM_VERSION}` " - "which introduces pandas.NA" - ), - ) - df_data = [ collections.OrderedDict( [("repeated_col", [pandas.NA, 2, None, 4]), ("not_repeated_col", "first")] diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index af61ceb42..ff4c40f48 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -30,7 +30,11 @@ import requests import packaging import pytest -import pkg_resources + +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata try: import pandas @@ -76,13 +80,10 @@ from test_utils.imports import maybe_fail_import from tests.unit.helpers import make_connection -PANDAS_MINIUM_VERSION = pkg_resources.parse_version("1.0.0") - if pandas is not None: - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: - # Set to less than MIN version. - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" def _make_credentials(): @@ -8145,10 +8146,7 @@ def test_load_table_from_dataframe_unknown_table(self): timeout=DEFAULT_TIMEOUT, ) - @unittest.skipIf( - pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, - "Only `pandas version >=1.0.0` supported", - ) + @unittest.skipIf(pandas is None, "Requires `pandas`") @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_load_table_from_dataframe_w_nullable_int64_datatype(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES @@ -8193,10 +8191,7 @@ def test_load_table_from_dataframe_w_nullable_int64_datatype(self): SchemaField("x", "INT64", "NULLABLE", None), ) - @unittest.skipIf( - pandas is None or PANDAS_INSTALLED_VERSION < PANDAS_MINIUM_VERSION, - "Only `pandas version >=1.0.0` supported", - ) + @unittest.skipIf(pandas is None, "Requires `pandas`") # @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") def test_load_table_from_dataframe_w_nullable_int64_datatype_automatic_schema(self): from google.cloud.bigquery.client import _DEFAULT_NUM_RETRIES diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py new file mode 100644 index 000000000..6f1b16c66 --- /dev/null +++ b/tests/unit/test_packaging.py @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + + +def test_namespace_package_compat(tmp_path): + # The ``google`` namespace package should not be masked + # by the presence of ``google-cloud-bigquery``. + google = tmp_path / "google" + google.mkdir() + google.joinpath("othermod.py").write_text("") + env = dict(os.environ, PYTHONPATH=str(tmp_path)) + cmd = [sys.executable, "-m", "google.othermod"] + subprocess.check_call(cmd, env=env) + + # The ``google.cloud`` namespace package should not be masked + # by the presence of ``google-cloud-bigquery``. + google_cloud = tmp_path / "google" / "cloud" + google_cloud.mkdir() + google_cloud.joinpath("othermod.py").write_text("") + env = dict(os.environ, PYTHONPATH=str(tmp_path)) + cmd = [sys.executable, "-m", "google.cloud.othermod"] + subprocess.check_call(cmd, env=env) diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 05ad8de6e..85f335dd1 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -22,9 +22,13 @@ import warnings import mock -import pkg_resources import pytest +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata + import google.api_core.exceptions from test_utils.imports import maybe_fail_import @@ -71,9 +75,9 @@ tqdm = None if pandas is not None: - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" def _mock_client(): @@ -3793,9 +3797,7 @@ def test_to_dataframe_w_dtypes_mapper(self): self.assertEqual(df.timestamp.dtype.name, "object") @unittest.skipIf(pandas is None, "Requires `pandas`") - @pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" - ) + @pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") def test_to_dataframe_w_none_dtypes_mapper(self): from google.cloud.bigquery.schema import SchemaField @@ -3908,9 +3910,7 @@ def test_to_dataframe_w_unsupported_dtypes_mapper(self): ) @unittest.skipIf(pandas is None, "Requires `pandas`") - @pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" - ) + @pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") def test_to_dataframe_column_dtypes(self): from google.cloud.bigquery.schema import SchemaField diff --git a/tests/unit/test_table_pandas.py b/tests/unit/test_table_pandas.py index 6970d9d65..b38568561 100644 --- a/tests/unit/test_table_pandas.py +++ b/tests/unit/test_table_pandas.py @@ -15,7 +15,11 @@ import datetime import decimal from unittest import mock -import pkg_resources + +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata import pytest @@ -28,9 +32,9 @@ TEST_PATH = "/v1/project/test-proj/dataset/test-dset/table/test-tbl/data" if pandas is not None: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = pkg_resources.get_distribution("pandas").parsed_version + PANDAS_INSTALLED_VERSION = metadata.version("pandas") else: # pragma: NO COVER - PANDAS_INSTALLED_VERSION = pkg_resources.parse_version("0.0.0") + PANDAS_INSTALLED_VERSION = "0.0.0" @pytest.fixture @@ -40,9 +44,7 @@ def class_under_test(): return RowIterator -@pytest.mark.skipif( - PANDAS_INSTALLED_VERSION >= pkg_resources.parse_version("2.0.0"), reason="" -) +@pytest.mark.skipif(PANDAS_INSTALLED_VERSION[0:2] not in ["0.", "1."], reason="") def test_to_dataframe_nullable_scalars(monkeypatch, class_under_test): # See tests/system/test_arrow.py for the actual types we get from the API. arrow_schema = pyarrow.schema(